@aiwayds/dsh-topics-memory 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/README.zh.md +8 -0
- package/lib/config.d.ts +19 -1
- package/lib/config.js +18 -0
- package/lib/index.d.ts +4 -0
- package/lib/index.js +33 -11
- package/lib/jev/backends.d.ts +3 -0
- package/lib/jev/client.d.ts +7 -0
- package/lib/jev/client.js +18 -4
- package/lib/jev/dual.d.ts +48 -0
- package/lib/jev/dual.js +47 -0
- package/lib/jev/log.d.ts +8 -0
- package/lib/jev/log.js +2 -0
- package/lib/quality.js +89 -29
- package/package.json +2 -2
- package/skills/dsh-topics-memory-laya/SKILL.md +84 -0
- package/src/config.ts +18 -0
- package/src/index.ts +34 -11
- package/src/jev/backends.ts +3 -0
- package/src/jev/client.ts +22 -4
- package/src/jev/dual.ts +73 -0
- package/src/jev/log.ts +10 -0
- package/src/quality.ts +103 -40
package/README.md
CHANGED
|
@@ -94,6 +94,10 @@ Adopt / record / fallback bands (calibrated offline on a 383-case gold corpus; t
|
|
|
94
94
|
| Consolidation prefilter | The cluster goes to the LLM in full, exactly as today | none |
|
|
95
95
|
| Fast-lane shadow | Records the outcome only; zero behavior impact | none |
|
|
96
96
|
|
|
97
|
+
**Setting up laya**: the bundled skill `dsh-topics-memory-laya` walks through install (venv, mirror acceleration for CN networks, checkpoint download), starting laya-serve, wiring `jevLayaFallback`, and verifying via decisions.jsonl. Ask the agent about laya, or read `skills/dsh-topics-memory-laya/SKILL.md` directly.
|
|
98
|
+
|
|
99
|
+
**Local laya pace-maker (experimental, off by default).** With `jevLayaFallback: true` and a local [laya](https://github.com/NandhaKishorM/laya) serve running, every jev call fires a parallel local laya request: the primary answer drives the decision, and the laya answer is logged alongside it — a permanently running laya-vs-jev comparison on identical questions. laya is TELEMETRY ONLY: on real queries its within-batch ranking agreed with the primary 0/14 and its negative scores sit at 0.63-0.76 (absolute scores unusable), so a primary failure falls open to the legacy path exactly as without the pace-maker — the comparison rows are the value, not a degraded takeover. The consolidation prefilter deliberately does not ride the pace-maker. laya not running costs a refused connection in milliseconds.
|
|
100
|
+
|
|
97
101
|
Every call and verdict lands in `~/.dsh/topics/meta/decisions.jsonl` — local-only and redacted: slugs, pair hashes, question types, probabilities, latency and token counts; never conversation text or conclusion bodies. It has no config key (it stops together with `jevEnabled: false`); `/topics status` shows a 30-day summary line.
|
|
98
102
|
|
|
99
103
|
|
|
@@ -203,6 +207,8 @@ First-time setup belongs to `/topics onboard`; day-to-day tuning is `/topics set
|
|
|
203
207
|
| `jevModel` | empty (per backend: `jev-1.13-free` / `jev-1.13.0` / `typesafe/jev-1.13`) | Version-pinned decision model; upgrading is an explicit action |
|
|
204
208
|
| `jevTimeoutMs` | `3000` | Per-request decision timeout; single attempt, no retry |
|
|
205
209
|
| `jevSecretFile` | none | External secret list for the outbound secret gate; re-read when hot-changed |
|
|
210
|
+
| `jevLayaFallback` | `false` | Local laya pace-maker: fire a parallel laya call on every jev request; its answer takes over (degraded, relative-ranking only) when the primary fails |
|
|
211
|
+
| `jevLayaUrl` | `http://127.0.0.1:8000/v1/systemone` | laya-serve endpoint for the pace-maker |
|
|
206
212
|
|
|
207
213
|
## Acknowledgements
|
|
208
214
|
|
package/README.zh.md
CHANGED
|
@@ -193,8 +193,16 @@ dsh plugin --profile <name> remove @aiwayds/dsh-topics-memory
|
|
|
193
193
|
| `jevModel` | 空(按 backend:`jev-1.13-free` / `jev-1.13.0` / `typesafe/jev-1.13`) | 版本钉定的决策模型;升级是显式动作 |
|
|
194
194
|
| `jevTimeoutMs` | `3000` | 单次决策请求超时;单发不重试 |
|
|
195
195
|
| `jevSecretFile` | 无 | 出网 secret gate 的外部清单路径;热更换路径即重读 |
|
|
196
|
+
| `jevLayaFallback` | `false` | 本地 laya 陪跑:每次 jev 调用并发一路 laya 请求;laya 为纯遥测对照(不接管决策,真实负载排序一致率 0/14) |
|
|
197
|
+
| `jevLayaUrl` | `http://127.0.0.1:8000/v1/systemone` | 陪跑用的 laya-serve 端点 |
|
|
196
198
|
| `jevDebug` | 关 | 诊断日志开关(走宿主 logger,默认静默) |
|
|
197
199
|
|
|
200
|
+
### laya 本地陪跑(实验,默认关)
|
|
201
|
+
|
|
202
|
+
`jevLayaFallback: true` 且本地跑着 [laya](https://github.com/NandhaKishorM/laya) serve 时,每次 jev 调用并发一路本地 laya 请求:主路答案驱动决策,laya 答案并排落账——一组永久运行的 laya vs jev 同题对照。laya 是**纯遥测**(真实负载实测批内排序与主路一致率 0/14、绝对分不可用),主路失败照旧回退旧路径;laya 没起 = 毫秒级连接拒绝。
|
|
203
|
+
|
|
204
|
+
**laya 安装配置**:随包 skill `dsh-topics-memory-laya` 完整覆盖安装(venv / 国内镜像加速 / checkpoint 下载)、启动 laya-serve、配置 `jevLayaFallback`、用 decisions.jsonl 验证生效。对 agent 说 laya 即可触发,或直接读 `skills/dsh-topics-memory-laya/SKILL.md`。
|
|
205
|
+
|
|
198
206
|
## Acknowledgements
|
|
199
207
|
|
|
200
208
|
本项目的形态直接受以下项目的启发与支撑:
|
package/lib/config.d.ts
CHANGED
|
@@ -89,6 +89,14 @@ export declare const TopicsConfig: z<Schemastery.ObjectS<NoInfer<{
|
|
|
89
89
|
/** External secret list for the outbound secret gate (JEV_SECRET_FILE
|
|
90
90
|
* semantics, design §3.3); swapping the path hot reloads the list. */
|
|
91
91
|
jevSecretFile: z<string, string, "volatile-defined">;
|
|
92
|
+
/** Local laya pace-maker (design §3.4, 09-27): fire a parallel laya call on
|
|
93
|
+
* every jev request; its verdict is logged for comparison and takes over
|
|
94
|
+
* ONLY when the primary backend fails (degraded). Default off — zero
|
|
95
|
+
* behavior change, and laya-serve not running costs a refused connection. */
|
|
96
|
+
jevLayaFallback: z<boolean, boolean, "volatile-defined">;
|
|
97
|
+
/** laya-serve systemone endpoint for the pace-maker (default: the local
|
|
98
|
+
* laya-serve bind from the laya repo's serve mode). */
|
|
99
|
+
jevLayaUrl: z<string, string, "volatile-defined">;
|
|
92
100
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
93
101
|
/** GitHub repo `owner/name`; empty = local-only mode (ADR 0008). */
|
|
94
102
|
repo: z<string, string, "volatile-defined">;
|
|
@@ -173,6 +181,14 @@ export declare const TopicsConfig: z<Schemastery.ObjectS<NoInfer<{
|
|
|
173
181
|
/** External secret list for the outbound secret gate (JEV_SECRET_FILE
|
|
174
182
|
* semantics, design §3.3); swapping the path hot reloads the list. */
|
|
175
183
|
jevSecretFile: z<string, string, "volatile-defined">;
|
|
184
|
+
/** Local laya pace-maker (design §3.4, 09-27): fire a parallel laya call on
|
|
185
|
+
* every jev request; its verdict is logged for comparison and takes over
|
|
186
|
+
* ONLY when the primary backend fails (degraded). Default off — zero
|
|
187
|
+
* behavior change, and laya-serve not running costs a refused connection. */
|
|
188
|
+
jevLayaFallback: z<boolean, boolean, "volatile-defined">;
|
|
189
|
+
/** laya-serve systemone endpoint for the pace-maker (default: the local
|
|
190
|
+
* laya-serve bind from the laya repo's serve mode). */
|
|
191
|
+
jevLayaUrl: z<string, string, "volatile-defined">;
|
|
176
192
|
}>>, "plain">;
|
|
177
193
|
export type TopicsConfigValue = {
|
|
178
194
|
repo: string;
|
|
@@ -210,8 +226,10 @@ export type TopicsConfigValue = {
|
|
|
210
226
|
jevModel?: string;
|
|
211
227
|
jevTimeoutMs?: number;
|
|
212
228
|
jevSecretFile?: string;
|
|
229
|
+
jevLayaFallback?: boolean;
|
|
230
|
+
jevLayaUrl?: string;
|
|
213
231
|
};
|
|
214
|
-
export declare const CONFIG_KEYS: readonly ["repo", "autoInject", "injectDedup", "suppressEcho", "topK", "perTopicBudget", "totalBudget", "matchThreshold", "tagBoost", "injectMode", "qualityLane", "graphDepth", "recencyWindowDays", "autoObserve", "includeSubagents", "observationMaxChars", "distillEveryTurns", "distillOnSessionEnd", "distillProvider", "distillModel", "distillBatchSize", "distillMaxModelCalls", "consolidateCadence", "deprecatedTtlDays", "usageBoost", "pushDebounceSeconds", "jevEnabled", "jevBackend", "jevModel", "jevTimeoutMs", "jevSecretFile"];
|
|
232
|
+
export declare const CONFIG_KEYS: readonly ["repo", "autoInject", "injectDedup", "suppressEcho", "topK", "perTopicBudget", "totalBudget", "matchThreshold", "tagBoost", "injectMode", "qualityLane", "graphDepth", "recencyWindowDays", "autoObserve", "includeSubagents", "observationMaxChars", "distillEveryTurns", "distillOnSessionEnd", "distillProvider", "distillModel", "distillBatchSize", "distillMaxModelCalls", "consolidateCadence", "deprecatedTtlDays", "usageBoost", "pushDebounceSeconds", "jevEnabled", "jevBackend", "jevModel", "jevTimeoutMs", "jevSecretFile", "jevLayaFallback", "jevLayaUrl"];
|
|
215
233
|
export type ConfigKey = (typeof CONFIG_KEYS)[number];
|
|
216
234
|
/** CamelCase → dash-display (topK → top-k) for command output. */
|
|
217
235
|
export declare function displayKey(key: string): string;
|
package/lib/config.js
CHANGED
|
@@ -89,6 +89,14 @@ export const TopicsConfig = z.object({
|
|
|
89
89
|
/** External secret list for the outbound secret gate (JEV_SECRET_FILE
|
|
90
90
|
* semantics, design §3.3); swapping the path hot reloads the list. */
|
|
91
91
|
jevSecretFile: z.string().default('').volatile(),
|
|
92
|
+
/** Local laya pace-maker (design §3.4, 09-27): fire a parallel laya call on
|
|
93
|
+
* every jev request; its verdict is logged for comparison and takes over
|
|
94
|
+
* ONLY when the primary backend fails (degraded). Default off — zero
|
|
95
|
+
* behavior change, and laya-serve not running costs a refused connection. */
|
|
96
|
+
jevLayaFallback: z.boolean().default(false).volatile(),
|
|
97
|
+
/** laya-serve systemone endpoint for the pace-maker (default: the local
|
|
98
|
+
* laya-serve bind from the laya repo's serve mode). */
|
|
99
|
+
jevLayaUrl: z.string().default('http://127.0.0.1:8000/v1/systemone').volatile(),
|
|
92
100
|
});
|
|
93
101
|
export const CONFIG_KEYS = [
|
|
94
102
|
'repo',
|
|
@@ -122,6 +130,8 @@ export const CONFIG_KEYS = [
|
|
|
122
130
|
'jevModel',
|
|
123
131
|
'jevTimeoutMs',
|
|
124
132
|
'jevSecretFile',
|
|
133
|
+
'jevLayaFallback',
|
|
134
|
+
'jevLayaUrl',
|
|
125
135
|
];
|
|
126
136
|
/** CamelCase → dash-display (topK → top-k) for command output. */
|
|
127
137
|
export function displayKey(key) {
|
|
@@ -198,6 +208,14 @@ export function parseConfigValue(key, raw) {
|
|
|
198
208
|
case 'distillModel':
|
|
199
209
|
case 'jevModel':
|
|
200
210
|
case 'jevSecretFile':
|
|
211
|
+
case 'jevLayaUrl':
|
|
201
212
|
return raw;
|
|
213
|
+
case 'jevLayaFallback': {
|
|
214
|
+
if (raw === 'on' || raw === 'true')
|
|
215
|
+
return true;
|
|
216
|
+
if (raw === 'off' || raw === 'false')
|
|
217
|
+
return false;
|
|
218
|
+
return { error: 'jev-laya-fallback 取值 on|off' };
|
|
219
|
+
}
|
|
202
220
|
}
|
|
203
221
|
}
|
package/lib/index.d.ts
CHANGED
|
@@ -118,6 +118,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
118
118
|
jevModel: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
119
119
|
jevTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
120
120
|
jevSecretFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
121
|
+
jevLayaFallback: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
122
|
+
jevLayaUrl: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
121
123
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
122
124
|
repo: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
123
125
|
autoInject: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
@@ -150,6 +152,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
150
152
|
jevModel: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
151
153
|
jevTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
152
154
|
jevSecretFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
155
|
+
jevLayaFallback: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
156
|
+
jevLayaUrl: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
153
157
|
}>>, "plain">;
|
|
154
158
|
/** Live config reference as a 0.1.7 host passes it (cron / mcp-adapter
|
|
155
159
|
* spelling): `.get()` snapshots the current value — the host swaps the
|
package/lib/index.js
CHANGED
|
@@ -174,13 +174,18 @@ function readConfigValue(slot) {
|
|
|
174
174
|
// --- Bundled skill -----------------------------------------------------------
|
|
175
175
|
/** Provider name under `ctx.skills`; doubles as the skill name. */
|
|
176
176
|
const SKILL_PROVIDER_NAME = 'dsh-topics-memory-config';
|
|
177
|
-
/** Packaged skill
|
|
177
|
+
/** Packaged skill bodies; `../skills/` resolves to the package root from both lib/ and src/. */
|
|
178
178
|
const SKILL_BODY_URL = new URL('../skills/dsh-topics-memory-config/SKILL.md', import.meta.url);
|
|
179
|
-
|
|
179
|
+
const LAYA_SKILL_BODY_URL = new URL('../skills/dsh-topics-memory-laya/SKILL.md', import.meta.url);
|
|
180
|
+
/** Resource bases served with the skills so their relative links resolve. */
|
|
180
181
|
const SKILL_RESOURCE_BASE = {
|
|
181
182
|
kind: 'directory',
|
|
182
183
|
path: fileURLToPath(new URL('../skills/dsh-topics-memory-config/', import.meta.url)),
|
|
183
184
|
};
|
|
185
|
+
const LAYA_SKILL_RESOURCE_BASE = {
|
|
186
|
+
kind: 'directory',
|
|
187
|
+
path: fileURLToPath(new URL('../skills/dsh-topics-memory-laya/', import.meta.url)),
|
|
188
|
+
};
|
|
184
189
|
const SKILL_INVOCATION = { modelInvocable: true, userInvocable: true };
|
|
185
190
|
/** Routing description; must stay identical to the SKILL.md frontmatter (asserted in tests). */
|
|
186
191
|
const SKILL_DESCRIPTION = 'dsh 记忆插件(@aiwayds/dsh-topics-memory)使用与配置指南。凡涉及 dsh 记忆/话题库/GitHub 同步/蒸馏/整理,或要配置 topics 时先读本指南:设置页 `dsh-topics-memory` 条目全部键(repo/autoInject/topK/注入预算/蒸馏/整理/观察/图游走等)、/topics 命令族(onboard/status/distill/consolidate/stats/list/show/history/graph/sync/config/set)、首次配置 ask_user_question 向导(local-only 或绑 GitHub 仓、蒸馏模型路由、注入档位、自动观察)、注入形态 pointer/digest。触发词:topics、记忆、topic、蒸馏、distill、整理、consolidate、合并重复、deprecatedTtl、usageBoost、使用加成、autoInject、记忆库、include-subagents。';
|
|
@@ -194,18 +199,35 @@ const SKILL_CANDIDATE = {
|
|
|
194
199
|
rank: BUNDLED_SKILL_RANK,
|
|
195
200
|
locator: SKILL_BODY_URL,
|
|
196
201
|
};
|
|
202
|
+
/** The laya setup skill rides the same provider: installing/configuring the
|
|
203
|
+
* local pace-maker is part of operating this plugin's decision layer. */
|
|
204
|
+
const LAYA_SKILL_NAME = 'dsh-topics-memory-laya';
|
|
205
|
+
const LAYA_SKILL_DESCRIPTION = '为 dsh-topics-memory 的 jev 决策层安装并配置本地 laya 陪跑(laya-serve)。当用户要开启 jevLayaFallback、安装 laya、启动 laya-serve、配置 laya 端点、排查陪跑不生效或 decisions.jsonl 没有 laya 行时读本指南:venv 创建、国内三镜像加速(清华 PyPI / hf-mirror / 禁 Xet)、checkpoint 下载、laya-serve 启动、jevLayaFallback/jevLayaUrl 两键、真机验证(decisions.jsonl 出现 backend=laya 行)。触发词:laya、laya-serve、陪跑、本地决策、jevLaya、离线、8000 端口。';
|
|
206
|
+
const LAYA_SKILL_CANDIDATE = {
|
|
207
|
+
name: LAYA_SKILL_NAME,
|
|
208
|
+
description: LAYA_SKILL_DESCRIPTION,
|
|
209
|
+
invocation: SKILL_INVOCATION,
|
|
210
|
+
provider: SKILL_PROVIDER_NAME,
|
|
211
|
+
source: 'bundled',
|
|
212
|
+
resourceBase: LAYA_SKILL_RESOURCE_BASE,
|
|
213
|
+
rank: BUNDLED_SKILL_RANK,
|
|
214
|
+
locator: LAYA_SKILL_BODY_URL,
|
|
215
|
+
};
|
|
197
216
|
const skillProvider = {
|
|
198
217
|
name: SKILL_PROVIDER_NAME,
|
|
199
|
-
list: () => Promise.resolve([SKILL_CANDIDATE]),
|
|
200
|
-
async get(
|
|
218
|
+
list: () => Promise.resolve([SKILL_CANDIDATE, LAYA_SKILL_CANDIDATE]),
|
|
219
|
+
async get(candidate) {
|
|
220
|
+
const isLaya = candidate.name === LAYA_SKILL_NAME;
|
|
221
|
+
const bodyUrl = isLaya ? LAYA_SKILL_BODY_URL : SKILL_BODY_URL;
|
|
222
|
+
const resourceBase = isLaya ? LAYA_SKILL_RESOURCE_BASE : SKILL_RESOURCE_BASE;
|
|
201
223
|
return {
|
|
202
|
-
name:
|
|
203
|
-
description:
|
|
204
|
-
invocation:
|
|
205
|
-
provider:
|
|
206
|
-
source:
|
|
207
|
-
resourceBase
|
|
208
|
-
content: stripFrontmatter(await readFile(
|
|
224
|
+
name: candidate.name,
|
|
225
|
+
description: candidate.description,
|
|
226
|
+
invocation: candidate.invocation,
|
|
227
|
+
provider: SKILL_PROVIDER_NAME,
|
|
228
|
+
source: candidate.source,
|
|
229
|
+
resourceBase,
|
|
230
|
+
content: stripFrontmatter(await readFile(bodyUrl, 'utf8')),
|
|
209
231
|
};
|
|
210
232
|
},
|
|
211
233
|
};
|
package/lib/jev/backends.d.ts
CHANGED
|
@@ -9,6 +9,9 @@
|
|
|
9
9
|
* @module jev/backends
|
|
10
10
|
*/
|
|
11
11
|
export type JevBackendName = 'zen' | 'native' | 'openrouter';
|
|
12
|
+
/** The local laya pace-maker writes its own call-layer rows; not a config
|
|
13
|
+
* backend (no entry in BACKENDS) — it rides jevAsk's overrides hook. */
|
|
14
|
+
export type JevPaceMakerName = 'laya';
|
|
12
15
|
export type JevFamily = 'systemone' | 'decisions';
|
|
13
16
|
export interface JevBackend {
|
|
14
17
|
readonly name: JevBackendName;
|
package/lib/jev/client.d.ts
CHANGED
|
@@ -41,6 +41,13 @@ export interface JevAskArgs {
|
|
|
41
41
|
fallback?: boolean;
|
|
42
42
|
/** Host logger (optional, warn-only, best-effort). */
|
|
43
43
|
logger?: GateLogger;
|
|
44
|
+
/** Dual-run pace-maker hook (jev/dual.ts): target this endpoint/model with
|
|
45
|
+
* no auth header and tag the call-layer row backend='laya'. Never set by
|
|
46
|
+
* user config — internal to the dual wrapper. */
|
|
47
|
+
overrides?: {
|
|
48
|
+
endpoint: string;
|
|
49
|
+
model: string;
|
|
50
|
+
};
|
|
44
51
|
}
|
|
45
52
|
export type JevAnswer = Record<string, unknown>;
|
|
46
53
|
export type JevAskOk = {
|
package/lib/jev/client.js
CHANGED
|
@@ -119,9 +119,22 @@ export function normalizeQuestions(questions, family) {
|
|
|
119
119
|
* resolves to a structured failure (never throws) so the calling lane can
|
|
120
120
|
* fail open. Writes the decisions.jsonl call-layer row on EVERY path. */
|
|
121
121
|
export async function jevAsk(args) {
|
|
122
|
-
|
|
122
|
+
// Pace-maker override: a synthetic laya backend (no key — the auth header
|
|
123
|
+
// carries a placeholder the local server ignores; the secret gate and the
|
|
124
|
+
// call-layer row still run, tagged backend='laya').
|
|
125
|
+
const backend = args.overrides
|
|
126
|
+
? {
|
|
127
|
+
name: 'laya',
|
|
128
|
+
family: 'systemone',
|
|
129
|
+
endpoint: args.overrides.endpoint,
|
|
130
|
+
defaultModel: args.overrides.model,
|
|
131
|
+
keyEnv: '',
|
|
132
|
+
keychainService: '',
|
|
133
|
+
keychainByDefault: false,
|
|
134
|
+
}
|
|
135
|
+
: resolveBackend(args.config.jevBackend);
|
|
123
136
|
// '' is the sentinel; undefined (bare harnesses) resolves to the default too
|
|
124
|
-
const model = args.config.jevModel ? args.config.jevModel : backend.defaultModel;
|
|
137
|
+
const model = args.overrides ? args.overrides.model : args.config.jevModel ? args.config.jevModel : backend.defaultModel;
|
|
125
138
|
const fallback = args.fallback === true; // absent => false: the column is call-time-invariant (fail-open = outcome !== ok)
|
|
126
139
|
const state = typeof args.state === 'string' ? args.state : '';
|
|
127
140
|
const started = Date.now();
|
|
@@ -157,8 +170,9 @@ export async function jevAsk(args) {
|
|
|
157
170
|
if (hit !== null) {
|
|
158
171
|
return refuse('secret_gate_blocked', `outbound payload matched secret pattern '${hit.pattern}' at offset ${hit.offset}; NOTHING was sent. Remove the secret from the request body (model/state/questions).`);
|
|
159
172
|
}
|
|
160
|
-
// 3. API key — absent key fails before any network activity.
|
|
161
|
-
|
|
173
|
+
// 3. API key — absent key fails before any network activity. The laya
|
|
174
|
+
// pace-maker skips resolution entirely (local, no auth).
|
|
175
|
+
const key = args.overrides ? 'laya-local' : await resolveApiKey(backend);
|
|
162
176
|
if (key === '') {
|
|
163
177
|
const hint = backend.keychainByDefault
|
|
164
178
|
? ` (set ${backend.keyEnv}, or store the key in the macOS keychain service '${backend.keychainService}')`
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dual-run pace-maker (design §3.4, added 09-27): fire the configured
|
|
3
|
+
* backend AND the local laya-serve endpoint in parallel on every jev call.
|
|
4
|
+
*
|
|
5
|
+
* - the PRIMARY answer drives the decision whenever it is ok;
|
|
6
|
+
* - the laya answer is always logged (its own call-layer row tagged
|
|
7
|
+
* backend='laya' plus verdict rows) — a free, permanently running
|
|
8
|
+
* laya-vs-jev comparison on identical questions;
|
|
9
|
+
* - TELEMETRY-ONLY: laya NEVER drives a decision. On real data its
|
|
10
|
+
* within-batch top-1 agreed with the primary 0/14 and its negatives sit in
|
|
11
|
+
* the same band as its positives (0.63-0.76), so both its relative ranking
|
|
12
|
+
* and its absolute scores are unusable (4-way bench, 09-27). A primary
|
|
13
|
+
* FAILURE (cold-start spike, timeout, 5xx…) is therefore a failure even
|
|
14
|
+
* when laya answered: the result stays ok:false and the caller fails open
|
|
15
|
+
* (ADR 0018). `layaResult` still carries the comparison answers so the
|
|
16
|
+
* caller can log the laya verdict rows;
|
|
17
|
+
* - laya not running = a refused connection in milliseconds — the switch
|
|
18
|
+
* can stay on unconditionally.
|
|
19
|
+
*
|
|
20
|
+
* The consolidation prefilter deliberately does NOT ride this wrapper: laya's
|
|
21
|
+
* Chinese noul is the weak spot, and a wrong veto silently skips real
|
|
22
|
+
* consolidation work. Prefilter keeps its plain fail-open.
|
|
23
|
+
*
|
|
24
|
+
* @module jev/dual
|
|
25
|
+
*/
|
|
26
|
+
import type { JevAskArgs, JevAskConfig, JevAskResult } from './client.ts';
|
|
27
|
+
/** Default local laya-serve endpoint (laya repo: `laya[serve]` binds 8000). */
|
|
28
|
+
export declare const DEFAULT_LAYA_URL = "http://127.0.0.1:8000/v1/systemone";
|
|
29
|
+
export interface JevDualConfig extends JevAskConfig {
|
|
30
|
+
/** Pace-maker switch (default OFF — zero extra requests when false). */
|
|
31
|
+
jevLayaFallback: boolean;
|
|
32
|
+
/** Local laya-serve systemone endpoint. */
|
|
33
|
+
jevLayaUrl: string;
|
|
34
|
+
}
|
|
35
|
+
export interface JevDualArgs extends Omit<JevAskArgs, 'config'> {
|
|
36
|
+
config: JevDualConfig;
|
|
37
|
+
}
|
|
38
|
+
export type JevDualSource = 'primary' | 'laya';
|
|
39
|
+
export type JevDualResult = JevAskResult & {
|
|
40
|
+
/** Which backend ANSWERED for the record: 'primary' when the primary
|
|
41
|
+
* answered (the only case that may drive), 'laya' when it failed and the
|
|
42
|
+
* pace-maker covered for telemetry only. */
|
|
43
|
+
source: JevDualSource;
|
|
44
|
+
/** The pace-maker's own result when it ran (null when the switch is off),
|
|
45
|
+
* always logged by jevAsk via the overrides hook. */
|
|
46
|
+
layaResult: JevAskResult | null;
|
|
47
|
+
};
|
|
48
|
+
export declare function jevAskDual(args: JevDualArgs): Promise<JevDualResult>;
|
package/lib/jev/dual.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dual-run pace-maker (design §3.4, added 09-27): fire the configured
|
|
3
|
+
* backend AND the local laya-serve endpoint in parallel on every jev call.
|
|
4
|
+
*
|
|
5
|
+
* - the PRIMARY answer drives the decision whenever it is ok;
|
|
6
|
+
* - the laya answer is always logged (its own call-layer row tagged
|
|
7
|
+
* backend='laya' plus verdict rows) — a free, permanently running
|
|
8
|
+
* laya-vs-jev comparison on identical questions;
|
|
9
|
+
* - TELEMETRY-ONLY: laya NEVER drives a decision. On real data its
|
|
10
|
+
* within-batch top-1 agreed with the primary 0/14 and its negatives sit in
|
|
11
|
+
* the same band as its positives (0.63-0.76), so both its relative ranking
|
|
12
|
+
* and its absolute scores are unusable (4-way bench, 09-27). A primary
|
|
13
|
+
* FAILURE (cold-start spike, timeout, 5xx…) is therefore a failure even
|
|
14
|
+
* when laya answered: the result stays ok:false and the caller fails open
|
|
15
|
+
* (ADR 0018). `layaResult` still carries the comparison answers so the
|
|
16
|
+
* caller can log the laya verdict rows;
|
|
17
|
+
* - laya not running = a refused connection in milliseconds — the switch
|
|
18
|
+
* can stay on unconditionally.
|
|
19
|
+
*
|
|
20
|
+
* The consolidation prefilter deliberately does NOT ride this wrapper: laya's
|
|
21
|
+
* Chinese noul is the weak spot, and a wrong veto silently skips real
|
|
22
|
+
* consolidation work. Prefilter keeps its plain fail-open.
|
|
23
|
+
*
|
|
24
|
+
* @module jev/dual
|
|
25
|
+
*/
|
|
26
|
+
import { jevAsk } from "./client.js";
|
|
27
|
+
/** Default local laya-serve endpoint (laya repo: `laya[serve]` binds 8000). */
|
|
28
|
+
export const DEFAULT_LAYA_URL = 'http://127.0.0.1:8000/v1/systemone';
|
|
29
|
+
export async function jevAskDual(args) {
|
|
30
|
+
if (args.config.jevLayaFallback !== true) {
|
|
31
|
+
const r = await jevAsk(args);
|
|
32
|
+
return { ...r, source: 'primary', layaResult: null };
|
|
33
|
+
}
|
|
34
|
+
const layaArgs = {
|
|
35
|
+
...args,
|
|
36
|
+
config: { ...args.config, jevModel: 'laya-rl-agent' },
|
|
37
|
+
overrides: { endpoint: args.config.jevLayaUrl || DEFAULT_LAYA_URL, model: 'laya-rl-agent' },
|
|
38
|
+
};
|
|
39
|
+
const [primary, laya] = await Promise.all([jevAsk(args), jevAsk(layaArgs)]);
|
|
40
|
+
if (primary.ok)
|
|
41
|
+
return { ...primary, source: 'primary', layaResult: laya };
|
|
42
|
+
// REVISED 09-27 (real-data evidence: within-batch top-1 agreement 0/14):
|
|
43
|
+
// a primary failure is a FAILURE even when the pace-maker answered — laya
|
|
44
|
+
// is telemetry-only. The caller fails open (ADR 0018); layaResult carries
|
|
45
|
+
// the comparison answers so the caller can still log laya verdict rows.
|
|
46
|
+
return { ok: false, outcome: primary.outcome, message: primary.message, latencyMs: primary.latencyMs, source: 'laya', layaResult: laya };
|
|
47
|
+
}
|
package/lib/jev/log.d.ts
CHANGED
|
@@ -52,6 +52,14 @@ export interface JevVerdictRecord {
|
|
|
52
52
|
probability: number;
|
|
53
53
|
band: JevBand;
|
|
54
54
|
agree: 'hit' | 'nearFloor' | 'gate-blocked' | 'wouldBlock' | 'n/a';
|
|
55
|
+
/** Which backend produced this verdict: 'primary' (default — the configured
|
|
56
|
+
* backend) or 'laya' (the local pace-maker). Design §3.4. */
|
|
57
|
+
backend?: string;
|
|
58
|
+
/** True when this row's source DROVE the decision because the primary
|
|
59
|
+
* backend was unavailable (degraded mode). Comparison rows from the
|
|
60
|
+
* pace-maker are degraded=false; laya rows are excluded from ECE either
|
|
61
|
+
* way (uncalibrated probabilities). */
|
|
62
|
+
degraded?: boolean;
|
|
55
63
|
}
|
|
56
64
|
export declare function decisionsFile(root: string): string;
|
|
57
65
|
/** digest(state, questionId, instructions): `h1:` + first 16 hex of
|
package/lib/jev/log.js
CHANGED
package/lib/quality.js
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* @module quality
|
|
23
23
|
*/
|
|
24
24
|
import { searchTopics } from "./retrieval.js";
|
|
25
|
-
import {
|
|
25
|
+
import { jevAskDual } from "./jev/dual.js";
|
|
26
26
|
import { band as jevBand } from "./jev/thresholds.js";
|
|
27
27
|
import { digestFor, logVerdicts } from "./jev/log.js";
|
|
28
28
|
export const PENDING_TTL_MS = 10 * 60_000;
|
|
@@ -88,6 +88,33 @@ function noulProbability(answer) {
|
|
|
88
88
|
const p = answer.noul;
|
|
89
89
|
return typeof p === 'number' && Number.isFinite(p) && p >= 0 && p <= 1 ? p : undefined;
|
|
90
90
|
}
|
|
91
|
+
/** Pace-maker comparison rows for the slow rerank (design §3.4, REVISED
|
|
92
|
+
* 09-27): laya is TELEMETRY ONLY, so its rows are recorded band='record'
|
|
93
|
+
* (its absolute scores never claim adopt) and degraded=false whether or not
|
|
94
|
+
* the primary answered — landing them costs no behavior. An unparseable
|
|
95
|
+
* answer gets no fabricated row. */
|
|
96
|
+
function layaRerankRows(at, state, metas, answers) {
|
|
97
|
+
const recs = [];
|
|
98
|
+
for (const m of metas) {
|
|
99
|
+
const probability = noulProbability(answers?.[m.qid]);
|
|
100
|
+
if (probability === undefined)
|
|
101
|
+
continue;
|
|
102
|
+
recs.push({
|
|
103
|
+
at,
|
|
104
|
+
lane: 'slowlane-rerank',
|
|
105
|
+
questionId: m.qid,
|
|
106
|
+
qtype: 'noul',
|
|
107
|
+
ref: `slug:${m.slug}`,
|
|
108
|
+
digest: digestFor(state, m.qid, m.instructions),
|
|
109
|
+
probability,
|
|
110
|
+
band: 'record',
|
|
111
|
+
agree: 'n/a',
|
|
112
|
+
backend: 'laya',
|
|
113
|
+
degraded: false,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return recs;
|
|
117
|
+
}
|
|
91
118
|
/** The config slice jevAsk needs, tolerating bare harnesses (§4: consumers
|
|
92
119
|
* treat undefined as the documented default). */
|
|
93
120
|
function jevConfigOf(cfg) {
|
|
@@ -96,6 +123,8 @@ function jevConfigOf(cfg) {
|
|
|
96
123
|
jevModel: cfg.jevModel ?? '',
|
|
97
124
|
jevTimeoutMs: typeof cfg.jevTimeoutMs === 'number' && cfg.jevTimeoutMs > 0 ? cfg.jevTimeoutMs : 3000,
|
|
98
125
|
jevSecretFile: cfg.jevSecretFile ?? '',
|
|
126
|
+
jevLayaFallback: cfg.jevLayaFallback === true,
|
|
127
|
+
jevLayaUrl: cfg.jevLayaUrl ?? 'http://127.0.0.1:8000/v1/systemone',
|
|
99
128
|
};
|
|
100
129
|
}
|
|
101
130
|
/** Race `p` against a timeout; `onTimeout` fires so callers can abort upstream work. */
|
|
@@ -242,37 +271,48 @@ export class SlowLane {
|
|
|
242
271
|
}
|
|
243
272
|
if (metas.length === 0)
|
|
244
273
|
return;
|
|
245
|
-
const
|
|
274
|
+
const dual = await jevAskDual({
|
|
246
275
|
state,
|
|
247
276
|
questions,
|
|
248
277
|
config: jevConfigOf(this.service.cfg),
|
|
249
278
|
lane: 'fastgate-shadow',
|
|
250
279
|
fallback: false, // pure shadow — no behavior to fall back
|
|
251
280
|
});
|
|
252
|
-
if (!result.ok)
|
|
253
|
-
return;
|
|
254
281
|
const at = new Date().toISOString();
|
|
282
|
+
// Verdict rows for BOTH sources when they answered: the primary row set
|
|
283
|
+
// and the laya pace-maker comparison set (backend='laya', excluded from
|
|
284
|
+
// ECE, §6.2). Pure shadow — nothing gates here, so the laya rows land
|
|
285
|
+
// whenever laya answered, primary success or not.
|
|
286
|
+
const sources = [];
|
|
287
|
+
if (dual.ok)
|
|
288
|
+
sources.push({ answers: dual.answers, backend: 'primary', degraded: false });
|
|
289
|
+
if (dual.layaResult?.ok)
|
|
290
|
+
sources.push({ answers: dual.layaResult.answers, backend: 'laya', degraded: false });
|
|
255
291
|
const recs = [];
|
|
256
|
-
for (const
|
|
257
|
-
const
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
292
|
+
for (const src of sources) {
|
|
293
|
+
for (const m of metas) {
|
|
294
|
+
const probability = noulProbability(src.answers?.[m.qid]);
|
|
295
|
+
if (probability === undefined)
|
|
296
|
+
continue; // unparseable answer: no fabricated row
|
|
297
|
+
const b = jevBand(probability, 'rerank');
|
|
298
|
+
// agree reconciliation (§6.2): the ilog disposition, with wouldBlock for
|
|
299
|
+
// the reverse disagreement (lexical let it through, jev lands in the veto
|
|
300
|
+
// band — the other half of the calibration goldmine).
|
|
301
|
+
const agree = m.disposition === 'hit' && b === 'fallback' ? 'wouldBlock' : m.disposition;
|
|
302
|
+
recs.push({
|
|
303
|
+
at,
|
|
304
|
+
lane: 'fastgate-shadow',
|
|
305
|
+
questionId: m.qid,
|
|
306
|
+
qtype: 'noul',
|
|
307
|
+
ref: `slug:${m.slug}`,
|
|
308
|
+
digest: digestFor(state, m.qid, m.instructions),
|
|
309
|
+
probability,
|
|
310
|
+
band: b,
|
|
311
|
+
agree,
|
|
312
|
+
backend: src.backend,
|
|
313
|
+
degraded: src.degraded,
|
|
314
|
+
});
|
|
315
|
+
}
|
|
276
316
|
}
|
|
277
317
|
await logVerdicts(recs);
|
|
278
318
|
}
|
|
@@ -477,7 +517,7 @@ export class SlowLane {
|
|
|
477
517
|
questions[qid] = { type: 'noul', instructions, criteria: candidateCriteria(c) };
|
|
478
518
|
metas.push({ qid, slug: c.slug, instructions });
|
|
479
519
|
}
|
|
480
|
-
const
|
|
520
|
+
const dual = await jevAskDual({
|
|
481
521
|
state,
|
|
482
522
|
questions,
|
|
483
523
|
config: jevConfigOf(this.service.cfg),
|
|
@@ -488,17 +528,28 @@ export class SlowLane {
|
|
|
488
528
|
// stat. Fail-open occurrence = outcome !== 'ok' (this lane's fallback
|
|
489
529
|
// to the legacy LLM rerank is deterministic per ADR 0018).
|
|
490
530
|
});
|
|
491
|
-
|
|
531
|
+
// REVISED 09-27 (real-data evidence): the laya pace-maker is TELEMETRY
|
|
532
|
+
// ONLY. Its within-batch top-1 agreed with the primary 0/14 on real
|
|
533
|
+
// queries (and its negatives sit at 0.63-0.76 — absolute scores unusable),
|
|
534
|
+
// so the earlier degraded-ranking takeover is REVERTED: on primary
|
|
535
|
+
// failure the lane falls open to the legacy LLM rerank exactly as
|
|
536
|
+
// ADR 0018 says. The pace-maker still answers, so its comparison rows
|
|
537
|
+
// land FIRST here (pure telemetry), then the legacy rerank takes over.
|
|
538
|
+
if (!dual.ok) {
|
|
539
|
+
if (dual.layaResult?.ok) {
|
|
540
|
+
await logVerdicts(layaRerankRows(new Date().toISOString(), state, metas, dual.layaResult.answers));
|
|
541
|
+
}
|
|
492
542
|
return undefined;
|
|
543
|
+
}
|
|
493
544
|
const scored = [];
|
|
494
545
|
for (const m of metas) {
|
|
495
|
-
const probability = noulProbability(
|
|
546
|
+
const probability = noulProbability(dual.answers[m.qid]);
|
|
496
547
|
if (probability === undefined)
|
|
497
548
|
return undefined; // 坏答案 → fail-open (§1)
|
|
498
549
|
scored.push({ ...m, probability, band: jevBand(probability, 'rerank') });
|
|
499
550
|
}
|
|
500
551
|
const at = new Date().toISOString();
|
|
501
|
-
|
|
552
|
+
const recs = scored.map((m) => ({
|
|
502
553
|
at,
|
|
503
554
|
lane: 'slowlane-rerank',
|
|
504
555
|
questionId: m.qid,
|
|
@@ -508,7 +559,16 @@ export class SlowLane {
|
|
|
508
559
|
probability: m.probability,
|
|
509
560
|
band: m.band,
|
|
510
561
|
agree: 'n/a',
|
|
511
|
-
|
|
562
|
+
backend: dual.source,
|
|
563
|
+
degraded: false,
|
|
564
|
+
}));
|
|
565
|
+
// Pace-maker comparison rows (design §3.4): when the primary drove and
|
|
566
|
+
// laya also answered, log laya's verdicts alongside — the permanently
|
|
567
|
+
// running laya-vs-jev comparison. band='record': laya's absolute scores
|
|
568
|
+
// never claim adopt.
|
|
569
|
+
if (dual.layaResult?.ok)
|
|
570
|
+
recs.push(...layaRerankRows(at, state, metas, dual.layaResult.answers));
|
|
571
|
+
await logVerdicts(recs);
|
|
512
572
|
return scored
|
|
513
573
|
.filter((m) => m.band === 'adopt')
|
|
514
574
|
.sort((a, b) => b.probability - a.probability)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aiwayds/dsh-topics-memory",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.20.0",
|
|
4
4
|
"description": "dsh-plugin: quality-first topic memory for dsh — a topic keeps the starting question, conclusion, impact and dependencies, not the process; local-first git-traced OKF bundle, budgeted LLM-free injection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"build": "tsc -p tsconfig.json",
|
|
43
43
|
"check": "tsc --noEmit -p tsconfig.json",
|
|
44
44
|
"pretest": "tsc -p tsconfig.json",
|
|
45
|
-
"test": "node --test test/yaml.test.mjs test/okf.test.mjs test/retrieval.test.mjs test/digest.test.mjs test/ilog.test.mjs test/store.test.mjs test/paths.test.mjs test/git.test.mjs test/sync.test.mjs test/tools.test.mjs test/dedup.test.mjs test/commands.test.mjs test/onboard.test.mjs test/delegation.test.mjs test/observer.test.mjs test/distill.test.mjs test/consolidate.test.mjs test/viz.test.mjs test/corpus.test.mjs test/llm-scope.test.mjs test/quality.test.mjs test/replay-gate.test.mjs test/depends-repair.test.mjs test/skill.test.mjs test/fast-lane.test.mjs test/legacy-import.test.mjs test/jev-config.test.mjs test/jev-gate.test.mjs test/jev-client.test.mjs test/jev-log.test.mjs test/jev-fastgate.test.mjs",
|
|
45
|
+
"test": "node --test test/yaml.test.mjs test/okf.test.mjs test/retrieval.test.mjs test/digest.test.mjs test/ilog.test.mjs test/store.test.mjs test/paths.test.mjs test/git.test.mjs test/sync.test.mjs test/tools.test.mjs test/dedup.test.mjs test/commands.test.mjs test/onboard.test.mjs test/delegation.test.mjs test/observer.test.mjs test/distill.test.mjs test/consolidate.test.mjs test/viz.test.mjs test/corpus.test.mjs test/llm-scope.test.mjs test/quality.test.mjs test/replay-gate.test.mjs test/depends-repair.test.mjs test/skill.test.mjs test/fast-lane.test.mjs test/legacy-import.test.mjs test/jev-config.test.mjs test/jev-gate.test.mjs test/jev-client.test.mjs test/jev-log.test.mjs test/jev-dual.test.mjs test/jev-fastgate.test.mjs",
|
|
46
46
|
"smoke": "node scripts/smoke-boot.mjs"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {},
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: dsh-topics-memory-laya
|
|
3
|
+
description: "为 dsh-topics-memory 的 jev 决策层安装并配置本地 laya 陪跑(laya-serve)。当用户要开启 jevLayaFallback、安装 laya、启动 laya-serve、配置 laya 端点、排查 laya 连接失败/陪跑不生效/decisions.jsonl 没有 laya 行时读本指南。覆盖:venv 创建、国内三镜像加速(清华 PyPI / hf-mirror / 禁 Xet)、checkpoint 下载、laya-serve 启动与自启、jevLayaFallback/jevLayaUrl 两键配置、真机验证(decisions.jsonl 出现 backend=laya 行)。触发词:laya、laya-serve、陪跑、本地决策、jevLaya、离线、8000 端口。"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# laya 本地陪跑安装与配置
|
|
7
|
+
|
|
8
|
+
> laya 是开源的 System One 决策引擎(本地推理,jev 的同范式替身)。topics-memory 的
|
|
9
|
+
> `jevLayaFallback: true` 会把每次 jev 调用并发一路本地 laya 请求作对照遥测。本指南把
|
|
10
|
+
> 安装配置一次讲完;全程不涉及付费 API。
|
|
11
|
+
|
|
12
|
+
## 前置检查(先判断要不要装)
|
|
13
|
+
|
|
14
|
+
1. `curl -s --max-time 3 -o /dev/null -w "%{http_code}" -X POST http://127.0.0.1:8000/v1/systemone -H "content-type: application/json" -d '{"state":"w","questions":{"q":{"type":"noul","instructions":"w","criteria":{"true":"t","false":"f"}}}}'`
|
|
15
|
+
返回 200 = laya-serve 已在跑,**跳到「插件配置」**;连接拒绝 = 继续安装。
|
|
16
|
+
2. `python3 --version` 需要 ≥ 3.10(3.13 以下优先,太新的版本 torch 轮子可能缺)。
|
|
17
|
+
3. 磁盘 ≥ 5GB(torch + 两个 checkpoint)。
|
|
18
|
+
|
|
19
|
+
## 安装(国内网络三件套必带,否则要么 100KB/s 要么挂死)
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
mkdir -p ~/laya-tryout && cd ~/laya-tryout
|
|
23
|
+
uv venv --python 3.12 .venv # 没有 uv: brew install uv;或 python3 -m venv .venv
|
|
24
|
+
uv pip install --python .venv/bin/python \
|
|
25
|
+
--index-url https://pypi.tuna.tsinghua.edu.cn/simple "laya[serve]" hf_transfer
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
下载 checkpoint(两个共 ~1.5GB)——三个环境变量一个都不能少:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
export HF_ENDPOINT=https://hf-mirror.com # HF 直连国内常 <50KB/s
|
|
32
|
+
export HF_HUB_DISABLE_XET=1 # 新版 huggingface_hub 默认 Xet 协议,镜像不支持,会 0 字节挂死
|
|
33
|
+
export HF_HUB_ENABLE_HF_TRANSFER=1 # 并行分块,断流自动续传
|
|
34
|
+
.venv/bin/python -c "
|
|
35
|
+
from laya import Router
|
|
36
|
+
r = Router()
|
|
37
|
+
r.preload(['english', 'multilingual'])
|
|
38
|
+
print('checkpoints ok')
|
|
39
|
+
"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## 启动 laya-serve
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
LAYA_PRELOAD=1 LAYA_MODELS=english,multilingual ~/laya-tryout/.venv/bin/laya-serve
|
|
46
|
+
# binds 0.0.0.0:8000;进程前台跑,挂 nohup/tmux/launchd 由用户自选
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
- `LAYA_MODELS` 不设会尝试预载第三个 checkpoint(typed-decisions,再 +1.7GB)——陪跑用不到,别下。
|
|
50
|
+
- 自启(macOS launchd)可选:写 `~/Library/LaunchAgents/com.laya.serve.plist` 指向上面命令;不开自启就要接受「重启后陪跑静默失效」(无 laya = 毫秒级连接拒绝,无害但没对照数据)。
|
|
51
|
+
|
|
52
|
+
## 插件配置(在 profile patch 的 `dsh-topics-memory` 条目)
|
|
53
|
+
|
|
54
|
+
```yaml
|
|
55
|
+
config:
|
|
56
|
+
jevEnabled: true # 决策层总开关(本身默认关)
|
|
57
|
+
jevLayaFallback: true # laya 陪跑开关
|
|
58
|
+
# jevLayaUrl 默认就是 http://127.0.0.1:8000/v1/systemone,不改端口不用写
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
前置:`jevEnabled: true` 且已按主配置指南配好 jev 后端 key(zen 免费档默认;key 走 patch `env:` 块或 `JEV_KEYCHAIN`——dsh 会清洗 ambient KEY 变量)。
|
|
62
|
+
|
|
63
|
+
## 验证(三步)
|
|
64
|
+
|
|
65
|
+
1. `curl -s -X POST http://127.0.0.1:8000/v1/systemone -H "content-type: application/json" -d '{"state":"w","model":"laya-rl-agent","questions":{"q":{"type":"noul","instructions":"w","criteria":{"true":"t","false":"f"}}}}' | jq .answers` — 返回 noul 概率即 laya 侧通。
|
|
66
|
+
2. 跑一轮真实会话(任意问题),然后:
|
|
67
|
+
`jq -r 'select(.backend=="laya") | [.ref,.probability] | @tsv' ~/.dsh/topics/meta/decisions.jsonl | head` — 出现 `backend:"laya"` 的判定行 = 陪跑生效。
|
|
68
|
+
3. `~/.dsh/topics/meta/decisions.jsonl` 里同一 `digest` 应有成对的 primary/laya 行——这就是永久运行的 laya-vs-jev 对照数据。
|
|
69
|
+
|
|
70
|
+
## 语义边界(必须向用户讲清)
|
|
71
|
+
|
|
72
|
+
- laya 是**纯遥测对照,不参与决策**:真实负载实测其批内排序与 jev 一致率 0/14、负例分 0.63-0.76(绝对分不可用),主路失败照旧回退旧路径。
|
|
73
|
+
- 判定质量上 zen/typesafe 的 jev 系远胜 laya;laya 的价值是免费对照数据流 + 离线可用。
|
|
74
|
+
- laya 换语言(中→英)首次调用会触发 checkpoint 重载(数百 ms),陪跑场景无感。
|
|
75
|
+
|
|
76
|
+
## 常见坑速查
|
|
77
|
+
|
|
78
|
+
| 症状 | 原因 | 解法 |
|
|
79
|
+
|---|---|---|
|
|
80
|
+
| 下载 0 字节挂死 | Xet 协议 + 镜像不兼容 | `HF_HUB_DISABLE_XET=1` |
|
|
81
|
+
| checkpoint 下载 <100KB/s | HF 直连 | `HF_ENDPOINT=https://hf-mirror.com` |
|
|
82
|
+
| `z.string().volatile is not a function` 类插件报错 | (与 laya 无关)dsh 版本漂移 | 钉 0.1.7-rc 线,勿裸解析 |
|
|
83
|
+
| decisions.jsonl 无 backend=laya 行 | laya-serve 没起 / jevLayaFallback 没开 / jevEnabled 没开 | 按「验证」步骤逐层查 |
|
|
84
|
+
| 首次调用慢(秒级) | checkpoint 冷加载 | 预热一发即稳态(32-300ms) |
|
package/src/config.ts
CHANGED
|
@@ -91,6 +91,14 @@ export const TopicsConfig = z.object({
|
|
|
91
91
|
/** External secret list for the outbound secret gate (JEV_SECRET_FILE
|
|
92
92
|
* semantics, design §3.3); swapping the path hot reloads the list. */
|
|
93
93
|
jevSecretFile: z.string().default('').volatile(),
|
|
94
|
+
/** Local laya pace-maker (design §3.4, 09-27): fire a parallel laya call on
|
|
95
|
+
* every jev request; its verdict is logged for comparison and takes over
|
|
96
|
+
* ONLY when the primary backend fails (degraded). Default off — zero
|
|
97
|
+
* behavior change, and laya-serve not running costs a refused connection. */
|
|
98
|
+
jevLayaFallback: z.boolean().default(false).volatile(),
|
|
99
|
+
/** laya-serve systemone endpoint for the pace-maker (default: the local
|
|
100
|
+
* laya-serve bind from the laya repo's serve mode). */
|
|
101
|
+
jevLayaUrl: z.string().default('http://127.0.0.1:8000/v1/systemone').volatile(),
|
|
94
102
|
})
|
|
95
103
|
|
|
96
104
|
export type TopicsConfigValue = {
|
|
@@ -129,6 +137,8 @@ export type TopicsConfigValue = {
|
|
|
129
137
|
jevModel?: string
|
|
130
138
|
jevTimeoutMs?: number
|
|
131
139
|
jevSecretFile?: string
|
|
140
|
+
jevLayaFallback?: boolean
|
|
141
|
+
jevLayaUrl?: string
|
|
132
142
|
}
|
|
133
143
|
|
|
134
144
|
export const CONFIG_KEYS = [
|
|
@@ -163,6 +173,8 @@ export const CONFIG_KEYS = [
|
|
|
163
173
|
'jevModel',
|
|
164
174
|
'jevTimeoutMs',
|
|
165
175
|
'jevSecretFile',
|
|
176
|
+
'jevLayaFallback',
|
|
177
|
+
'jevLayaUrl',
|
|
166
178
|
] as const
|
|
167
179
|
|
|
168
180
|
export type ConfigKey = (typeof CONFIG_KEYS)[number]
|
|
@@ -235,6 +247,12 @@ export function parseConfigValue(key: ConfigKey, raw: string): boolean | number
|
|
|
235
247
|
case 'distillModel':
|
|
236
248
|
case 'jevModel':
|
|
237
249
|
case 'jevSecretFile':
|
|
250
|
+
case 'jevLayaUrl':
|
|
238
251
|
return raw
|
|
252
|
+
case 'jevLayaFallback': {
|
|
253
|
+
if (raw === 'on' || raw === 'true') return true
|
|
254
|
+
if (raw === 'off' || raw === 'false') return false
|
|
255
|
+
return { error: 'jev-laya-fallback 取值 on|off' }
|
|
256
|
+
}
|
|
239
257
|
}
|
|
240
258
|
}
|
package/src/index.ts
CHANGED
|
@@ -288,14 +288,19 @@ interface UserMessageData {
|
|
|
288
288
|
/** Provider name under `ctx.skills`; doubles as the skill name. */
|
|
289
289
|
const SKILL_PROVIDER_NAME = 'dsh-topics-memory-config'
|
|
290
290
|
|
|
291
|
-
/** Packaged skill
|
|
291
|
+
/** Packaged skill bodies; `../skills/` resolves to the package root from both lib/ and src/. */
|
|
292
292
|
const SKILL_BODY_URL = new URL('../skills/dsh-topics-memory-config/SKILL.md', import.meta.url)
|
|
293
|
+
const LAYA_SKILL_BODY_URL = new URL('../skills/dsh-topics-memory-laya/SKILL.md', import.meta.url)
|
|
293
294
|
|
|
294
|
-
/** Resource
|
|
295
|
+
/** Resource bases served with the skills so their relative links resolve. */
|
|
295
296
|
const SKILL_RESOURCE_BASE = {
|
|
296
297
|
kind: 'directory',
|
|
297
298
|
path: fileURLToPath(new URL('../skills/dsh-topics-memory-config/', import.meta.url)),
|
|
298
299
|
} as const
|
|
300
|
+
const LAYA_SKILL_RESOURCE_BASE = {
|
|
301
|
+
kind: 'directory',
|
|
302
|
+
path: fileURLToPath(new URL('../skills/dsh-topics-memory-laya/', import.meta.url)),
|
|
303
|
+
} as const
|
|
299
304
|
|
|
300
305
|
const SKILL_INVOCATION = { modelInvocable: true, userInvocable: true } as const
|
|
301
306
|
|
|
@@ -313,18 +318,36 @@ const SKILL_CANDIDATE: SkillCandidate = {
|
|
|
313
318
|
locator: SKILL_BODY_URL,
|
|
314
319
|
}
|
|
315
320
|
|
|
321
|
+
/** The laya setup skill rides the same provider: installing/configuring the
|
|
322
|
+
* local pace-maker is part of operating this plugin's decision layer. */
|
|
323
|
+
const LAYA_SKILL_NAME = 'dsh-topics-memory-laya'
|
|
324
|
+
const LAYA_SKILL_DESCRIPTION = '为 dsh-topics-memory 的 jev 决策层安装并配置本地 laya 陪跑(laya-serve)。当用户要开启 jevLayaFallback、安装 laya、启动 laya-serve、配置 laya 端点、排查陪跑不生效或 decisions.jsonl 没有 laya 行时读本指南:venv 创建、国内三镜像加速(清华 PyPI / hf-mirror / 禁 Xet)、checkpoint 下载、laya-serve 启动、jevLayaFallback/jevLayaUrl 两键、真机验证(decisions.jsonl 出现 backend=laya 行)。触发词:laya、laya-serve、陪跑、本地决策、jevLaya、离线、8000 端口。'
|
|
325
|
+
const LAYA_SKILL_CANDIDATE: SkillCandidate = {
|
|
326
|
+
name: LAYA_SKILL_NAME,
|
|
327
|
+
description: LAYA_SKILL_DESCRIPTION,
|
|
328
|
+
invocation: SKILL_INVOCATION,
|
|
329
|
+
provider: SKILL_PROVIDER_NAME,
|
|
330
|
+
source: 'bundled',
|
|
331
|
+
resourceBase: LAYA_SKILL_RESOURCE_BASE,
|
|
332
|
+
rank: BUNDLED_SKILL_RANK,
|
|
333
|
+
locator: LAYA_SKILL_BODY_URL,
|
|
334
|
+
}
|
|
335
|
+
|
|
316
336
|
const skillProvider: SkillProvider = {
|
|
317
337
|
name: SKILL_PROVIDER_NAME,
|
|
318
|
-
list: () => Promise.resolve([SKILL_CANDIDATE]),
|
|
319
|
-
async get(
|
|
338
|
+
list: () => Promise.resolve([SKILL_CANDIDATE, LAYA_SKILL_CANDIDATE]),
|
|
339
|
+
async get(candidate): Promise<SkillDefinition> {
|
|
340
|
+
const isLaya = candidate.name === LAYA_SKILL_NAME
|
|
341
|
+
const bodyUrl = isLaya ? LAYA_SKILL_BODY_URL : SKILL_BODY_URL
|
|
342
|
+
const resourceBase = isLaya ? LAYA_SKILL_RESOURCE_BASE : SKILL_RESOURCE_BASE
|
|
320
343
|
return {
|
|
321
|
-
name:
|
|
322
|
-
description:
|
|
323
|
-
invocation:
|
|
324
|
-
provider:
|
|
325
|
-
source:
|
|
326
|
-
resourceBase
|
|
327
|
-
content: stripFrontmatter(await readFile(
|
|
344
|
+
name: candidate.name,
|
|
345
|
+
description: candidate.description,
|
|
346
|
+
invocation: candidate.invocation,
|
|
347
|
+
provider: SKILL_PROVIDER_NAME,
|
|
348
|
+
source: candidate.source,
|
|
349
|
+
resourceBase,
|
|
350
|
+
content: stripFrontmatter(await readFile(bodyUrl, 'utf8')),
|
|
328
351
|
}
|
|
329
352
|
},
|
|
330
353
|
}
|
package/src/jev/backends.ts
CHANGED
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
export type JevBackendName = 'zen' | 'native' | 'openrouter'
|
|
13
|
+
/** The local laya pace-maker writes its own call-layer rows; not a config
|
|
14
|
+
* backend (no entry in BACKENDS) — it rides jevAsk's overrides hook. */
|
|
15
|
+
export type JevPaceMakerName = 'laya'
|
|
13
16
|
export type JevFamily = 'systemone' | 'decisions'
|
|
14
17
|
|
|
15
18
|
export interface JevBackend {
|
package/src/jev/client.ts
CHANGED
|
@@ -51,6 +51,10 @@ export interface JevAskArgs {
|
|
|
51
51
|
fallback?: boolean
|
|
52
52
|
/** Host logger (optional, warn-only, best-effort). */
|
|
53
53
|
logger?: GateLogger
|
|
54
|
+
/** Dual-run pace-maker hook (jev/dual.ts): target this endpoint/model with
|
|
55
|
+
* no auth header and tag the call-layer row backend='laya'. Never set by
|
|
56
|
+
* user config — internal to the dual wrapper. */
|
|
57
|
+
overrides?: { endpoint: string; model: string }
|
|
54
58
|
}
|
|
55
59
|
|
|
56
60
|
export type JevAnswer = Record<string, unknown>
|
|
@@ -214,9 +218,22 @@ export function normalizeQuestions(questions: Record<string, JevQuestion>, famil
|
|
|
214
218
|
* resolves to a structured failure (never throws) so the calling lane can
|
|
215
219
|
* fail open. Writes the decisions.jsonl call-layer row on EVERY path. */
|
|
216
220
|
export async function jevAsk(args: JevAskArgs): Promise<JevAskResult> {
|
|
217
|
-
|
|
221
|
+
// Pace-maker override: a synthetic laya backend (no key — the auth header
|
|
222
|
+
// carries a placeholder the local server ignores; the secret gate and the
|
|
223
|
+
// call-layer row still run, tagged backend='laya').
|
|
224
|
+
const backend = args.overrides
|
|
225
|
+
? ({
|
|
226
|
+
name: 'laya',
|
|
227
|
+
family: 'systemone',
|
|
228
|
+
endpoint: args.overrides.endpoint,
|
|
229
|
+
defaultModel: args.overrides.model,
|
|
230
|
+
keyEnv: '',
|
|
231
|
+
keychainService: '',
|
|
232
|
+
keychainByDefault: false,
|
|
233
|
+
} as unknown as ReturnType<typeof resolveBackend>)
|
|
234
|
+
: resolveBackend(args.config.jevBackend)
|
|
218
235
|
// '' is the sentinel; undefined (bare harnesses) resolves to the default too
|
|
219
|
-
const model = args.config.jevModel ? args.config.jevModel : backend.defaultModel
|
|
236
|
+
const model = args.overrides ? args.overrides.model : args.config.jevModel ? args.config.jevModel : backend.defaultModel
|
|
220
237
|
const fallback = args.fallback === true // absent => false: the column is call-time-invariant (fail-open = outcome !== ok)
|
|
221
238
|
const state = typeof args.state === 'string' ? args.state : ''
|
|
222
239
|
const started = Date.now()
|
|
@@ -259,8 +276,9 @@ export async function jevAsk(args: JevAskArgs): Promise<JevAskResult> {
|
|
|
259
276
|
)
|
|
260
277
|
}
|
|
261
278
|
|
|
262
|
-
// 3. API key — absent key fails before any network activity.
|
|
263
|
-
|
|
279
|
+
// 3. API key — absent key fails before any network activity. The laya
|
|
280
|
+
// pace-maker skips resolution entirely (local, no auth).
|
|
281
|
+
const key = args.overrides ? 'laya-local' : await resolveApiKey(backend)
|
|
264
282
|
if (key === '') {
|
|
265
283
|
const hint =
|
|
266
284
|
backend.keychainByDefault
|
package/src/jev/dual.ts
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dual-run pace-maker (design §3.4, added 09-27): fire the configured
|
|
3
|
+
* backend AND the local laya-serve endpoint in parallel on every jev call.
|
|
4
|
+
*
|
|
5
|
+
* - the PRIMARY answer drives the decision whenever it is ok;
|
|
6
|
+
* - the laya answer is always logged (its own call-layer row tagged
|
|
7
|
+
* backend='laya' plus verdict rows) — a free, permanently running
|
|
8
|
+
* laya-vs-jev comparison on identical questions;
|
|
9
|
+
* - TELEMETRY-ONLY: laya NEVER drives a decision. On real data its
|
|
10
|
+
* within-batch top-1 agreed with the primary 0/14 and its negatives sit in
|
|
11
|
+
* the same band as its positives (0.63-0.76), so both its relative ranking
|
|
12
|
+
* and its absolute scores are unusable (4-way bench, 09-27). A primary
|
|
13
|
+
* FAILURE (cold-start spike, timeout, 5xx…) is therefore a failure even
|
|
14
|
+
* when laya answered: the result stays ok:false and the caller fails open
|
|
15
|
+
* (ADR 0018). `layaResult` still carries the comparison answers so the
|
|
16
|
+
* caller can log the laya verdict rows;
|
|
17
|
+
* - laya not running = a refused connection in milliseconds — the switch
|
|
18
|
+
* can stay on unconditionally.
|
|
19
|
+
*
|
|
20
|
+
* The consolidation prefilter deliberately does NOT ride this wrapper: laya's
|
|
21
|
+
* Chinese noul is the weak spot, and a wrong veto silently skips real
|
|
22
|
+
* consolidation work. Prefilter keeps its plain fail-open.
|
|
23
|
+
*
|
|
24
|
+
* @module jev/dual
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { jevAsk } from './client.ts'
|
|
28
|
+
import type { JevAskArgs, JevAskConfig, JevAskResult } from './client.ts'
|
|
29
|
+
|
|
30
|
+
/** Default local laya-serve endpoint (laya repo: `laya[serve]` binds 8000). */
|
|
31
|
+
export const DEFAULT_LAYA_URL = 'http://127.0.0.1:8000/v1/systemone'
|
|
32
|
+
|
|
33
|
+
export interface JevDualConfig extends JevAskConfig {
|
|
34
|
+
/** Pace-maker switch (default OFF — zero extra requests when false). */
|
|
35
|
+
jevLayaFallback: boolean
|
|
36
|
+
/** Local laya-serve systemone endpoint. */
|
|
37
|
+
jevLayaUrl: string
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface JevDualArgs extends Omit<JevAskArgs, 'config'> {
|
|
41
|
+
config: JevDualConfig
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type JevDualSource = 'primary' | 'laya'
|
|
45
|
+
|
|
46
|
+
export type JevDualResult = JevAskResult & {
|
|
47
|
+
/** Which backend ANSWERED for the record: 'primary' when the primary
|
|
48
|
+
* answered (the only case that may drive), 'laya' when it failed and the
|
|
49
|
+
* pace-maker covered for telemetry only. */
|
|
50
|
+
source: JevDualSource
|
|
51
|
+
/** The pace-maker's own result when it ran (null when the switch is off),
|
|
52
|
+
* always logged by jevAsk via the overrides hook. */
|
|
53
|
+
layaResult: JevAskResult | null
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export async function jevAskDual(args: JevDualArgs): Promise<JevDualResult> {
|
|
57
|
+
if (args.config.jevLayaFallback !== true) {
|
|
58
|
+
const r = await jevAsk(args as JevAskArgs)
|
|
59
|
+
return { ...r, source: 'primary', layaResult: null }
|
|
60
|
+
}
|
|
61
|
+
const layaArgs: JevAskArgs = {
|
|
62
|
+
...args,
|
|
63
|
+
config: { ...args.config, jevModel: 'laya-rl-agent' },
|
|
64
|
+
overrides: { endpoint: args.config.jevLayaUrl || DEFAULT_LAYA_URL, model: 'laya-rl-agent' },
|
|
65
|
+
}
|
|
66
|
+
const [primary, laya] = await Promise.all([jevAsk(args as JevAskArgs), jevAsk(layaArgs)])
|
|
67
|
+
if (primary.ok) return { ...primary, source: 'primary', layaResult: laya }
|
|
68
|
+
// REVISED 09-27 (real-data evidence: within-batch top-1 agreement 0/14):
|
|
69
|
+
// a primary failure is a FAILURE even when the pace-maker answered — laya
|
|
70
|
+
// is telemetry-only. The caller fails open (ADR 0018); layaResult carries
|
|
71
|
+
// the comparison answers so the caller can still log laya verdict rows.
|
|
72
|
+
return { ok: false, outcome: primary.outcome, message: primary.message, latencyMs: primary.latencyMs, source: 'laya', layaResult: laya }
|
|
73
|
+
}
|
package/src/jev/log.ts
CHANGED
|
@@ -70,6 +70,14 @@ export interface JevVerdictRecord {
|
|
|
70
70
|
probability: number
|
|
71
71
|
band: JevBand
|
|
72
72
|
agree: 'hit' | 'nearFloor' | 'gate-blocked' | 'wouldBlock' | 'n/a'
|
|
73
|
+
/** Which backend produced this verdict: 'primary' (default — the configured
|
|
74
|
+
* backend) or 'laya' (the local pace-maker). Design §3.4. */
|
|
75
|
+
backend?: string
|
|
76
|
+
/** True when this row's source DROVE the decision because the primary
|
|
77
|
+
* backend was unavailable (degraded mode). Comparison rows from the
|
|
78
|
+
* pace-maker are degraded=false; laya rows are excluded from ECE either
|
|
79
|
+
* way (uncalibrated probabilities). */
|
|
80
|
+
degraded?: boolean
|
|
73
81
|
}
|
|
74
82
|
|
|
75
83
|
const COMPACT_LIMIT = 512 * 1024 // ~512KB cap, same as the ilog sidecars
|
|
@@ -178,6 +186,8 @@ export async function logVerdicts(recs: readonly JevVerdictRecord[]): Promise<vo
|
|
|
178
186
|
probability: rec.probability,
|
|
179
187
|
band: rec.band,
|
|
180
188
|
agree: rec.agree,
|
|
189
|
+
backend: rec.backend ?? 'primary',
|
|
190
|
+
degraded: rec.degraded === true,
|
|
181
191
|
}),
|
|
182
192
|
),
|
|
183
193
|
)
|
package/src/quality.ts
CHANGED
|
@@ -28,8 +28,9 @@ import { searchTopics } from './retrieval.ts'
|
|
|
28
28
|
import type { RingEntry } from './observer.ts'
|
|
29
29
|
import type { QueryBuildShape } from './ilog.ts'
|
|
30
30
|
import type { TopicsConfigValue } from './config.ts'
|
|
31
|
-
import {
|
|
32
|
-
import type {
|
|
31
|
+
import { jevAskDual } from './jev/dual.ts'
|
|
32
|
+
import type { JevQuestion } from './jev/client.ts'
|
|
33
|
+
import type { JevDualConfig, JevDualResult } from './jev/dual.ts'
|
|
33
34
|
import { band as jevBand } from './jev/thresholds.ts'
|
|
34
35
|
import type { JevBand } from './jev/thresholds.ts'
|
|
35
36
|
import { digestFor, logVerdicts } from './jev/log.ts'
|
|
@@ -125,14 +126,48 @@ function noulProbability(answer: unknown): number | undefined {
|
|
|
125
126
|
return typeof p === 'number' && Number.isFinite(p) && p >= 0 && p <= 1 ? p : undefined
|
|
126
127
|
}
|
|
127
128
|
|
|
129
|
+
/** Pace-maker comparison rows for the slow rerank (design §3.4, REVISED
|
|
130
|
+
* 09-27): laya is TELEMETRY ONLY, so its rows are recorded band='record'
|
|
131
|
+
* (its absolute scores never claim adopt) and degraded=false whether or not
|
|
132
|
+
* the primary answered — landing them costs no behavior. An unparseable
|
|
133
|
+
* answer gets no fabricated row. */
|
|
134
|
+
function layaRerankRows(
|
|
135
|
+
at: string,
|
|
136
|
+
state: string,
|
|
137
|
+
metas: { qid: string; slug: string; instructions: string }[],
|
|
138
|
+
answers: Record<string, unknown> | undefined,
|
|
139
|
+
): JevVerdictRecord[] {
|
|
140
|
+
const recs: JevVerdictRecord[] = []
|
|
141
|
+
for (const m of metas) {
|
|
142
|
+
const probability = noulProbability(answers?.[m.qid])
|
|
143
|
+
if (probability === undefined) continue
|
|
144
|
+
recs.push({
|
|
145
|
+
at,
|
|
146
|
+
lane: 'slowlane-rerank',
|
|
147
|
+
questionId: m.qid,
|
|
148
|
+
qtype: 'noul',
|
|
149
|
+
ref: `slug:${m.slug}`,
|
|
150
|
+
digest: digestFor(state, m.qid, m.instructions),
|
|
151
|
+
probability,
|
|
152
|
+
band: 'record',
|
|
153
|
+
agree: 'n/a',
|
|
154
|
+
backend: 'laya',
|
|
155
|
+
degraded: false,
|
|
156
|
+
})
|
|
157
|
+
}
|
|
158
|
+
return recs
|
|
159
|
+
}
|
|
160
|
+
|
|
128
161
|
/** The config slice jevAsk needs, tolerating bare harnesses (§4: consumers
|
|
129
162
|
* treat undefined as the documented default). */
|
|
130
|
-
function jevConfigOf(cfg: TopicsConfigValue):
|
|
163
|
+
function jevConfigOf(cfg: TopicsConfigValue): JevDualConfig {
|
|
131
164
|
return {
|
|
132
165
|
jevBackend: cfg.jevBackend ?? 'zen',
|
|
133
166
|
jevModel: cfg.jevModel ?? '',
|
|
134
167
|
jevTimeoutMs: typeof cfg.jevTimeoutMs === 'number' && cfg.jevTimeoutMs > 0 ? cfg.jevTimeoutMs : 3000,
|
|
135
168
|
jevSecretFile: cfg.jevSecretFile ?? '',
|
|
169
|
+
jevLayaFallback: cfg.jevLayaFallback === true,
|
|
170
|
+
jevLayaUrl: cfg.jevLayaUrl ?? 'http://127.0.0.1:8000/v1/systemone',
|
|
136
171
|
}
|
|
137
172
|
}
|
|
138
173
|
|
|
@@ -320,35 +355,45 @@ export class SlowLane {
|
|
|
320
355
|
metas.push({ qid, slug: cand.slug, instructions, disposition: cand.disposition })
|
|
321
356
|
}
|
|
322
357
|
if (metas.length === 0) return
|
|
323
|
-
const
|
|
358
|
+
const dual = await jevAskDual({
|
|
324
359
|
state,
|
|
325
360
|
questions,
|
|
326
361
|
config: jevConfigOf(this.service.cfg),
|
|
327
362
|
lane: 'fastgate-shadow',
|
|
328
363
|
fallback: false, // pure shadow — no behavior to fall back
|
|
329
364
|
})
|
|
330
|
-
if (!result.ok) return
|
|
331
365
|
const at = new Date().toISOString()
|
|
366
|
+
// Verdict rows for BOTH sources when they answered: the primary row set
|
|
367
|
+
// and the laya pace-maker comparison set (backend='laya', excluded from
|
|
368
|
+
// ECE, §6.2). Pure shadow — nothing gates here, so the laya rows land
|
|
369
|
+
// whenever laya answered, primary success or not.
|
|
370
|
+
const sources: { answers: Record<string, unknown> | undefined; backend: string; degraded: boolean }[] = []
|
|
371
|
+
if (dual.ok) sources.push({ answers: dual.answers, backend: 'primary', degraded: false })
|
|
372
|
+
if (dual.layaResult?.ok) sources.push({ answers: dual.layaResult.answers, backend: 'laya', degraded: false })
|
|
332
373
|
const recs: JevVerdictRecord[] = []
|
|
333
|
-
for (const
|
|
334
|
-
const
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
374
|
+
for (const src of sources) {
|
|
375
|
+
for (const m of metas) {
|
|
376
|
+
const probability = noulProbability(src.answers?.[m.qid])
|
|
377
|
+
if (probability === undefined) continue // unparseable answer: no fabricated row
|
|
378
|
+
const b = jevBand(probability, 'rerank')
|
|
379
|
+
// agree reconciliation (§6.2): the ilog disposition, with wouldBlock for
|
|
380
|
+
// the reverse disagreement (lexical let it through, jev lands in the veto
|
|
381
|
+
// band — the other half of the calibration goldmine).
|
|
382
|
+
const agree: JevVerdictRecord['agree'] = m.disposition === 'hit' && b === 'fallback' ? 'wouldBlock' : m.disposition
|
|
383
|
+
recs.push({
|
|
384
|
+
at,
|
|
385
|
+
lane: 'fastgate-shadow',
|
|
386
|
+
questionId: m.qid,
|
|
387
|
+
qtype: 'noul',
|
|
388
|
+
ref: `slug:${m.slug}`,
|
|
389
|
+
digest: digestFor(state, m.qid, m.instructions),
|
|
390
|
+
probability,
|
|
391
|
+
band: b,
|
|
392
|
+
agree,
|
|
393
|
+
backend: src.backend,
|
|
394
|
+
degraded: src.degraded,
|
|
395
|
+
})
|
|
396
|
+
}
|
|
352
397
|
}
|
|
353
398
|
await logVerdicts(recs)
|
|
354
399
|
}
|
|
@@ -544,7 +589,7 @@ export class SlowLane {
|
|
|
544
589
|
questions[qid] = { type: 'noul', instructions, criteria: candidateCriteria(c) }
|
|
545
590
|
metas.push({ qid, slug: c.slug, instructions })
|
|
546
591
|
}
|
|
547
|
-
const
|
|
592
|
+
const dual = await jevAskDual({
|
|
548
593
|
state,
|
|
549
594
|
questions,
|
|
550
595
|
config: jevConfigOf(this.service.cfg),
|
|
@@ -555,27 +600,45 @@ export class SlowLane {
|
|
|
555
600
|
// stat. Fail-open occurrence = outcome !== 'ok' (this lane's fallback
|
|
556
601
|
// to the legacy LLM rerank is deterministic per ADR 0018).
|
|
557
602
|
})
|
|
558
|
-
|
|
603
|
+
// REVISED 09-27 (real-data evidence): the laya pace-maker is TELEMETRY
|
|
604
|
+
// ONLY. Its within-batch top-1 agreed with the primary 0/14 on real
|
|
605
|
+
// queries (and its negatives sit at 0.63-0.76 — absolute scores unusable),
|
|
606
|
+
// so the earlier degraded-ranking takeover is REVERTED: on primary
|
|
607
|
+
// failure the lane falls open to the legacy LLM rerank exactly as
|
|
608
|
+
// ADR 0018 says. The pace-maker still answers, so its comparison rows
|
|
609
|
+
// land FIRST here (pure telemetry), then the legacy rerank takes over.
|
|
610
|
+
if (!dual.ok) {
|
|
611
|
+
if (dual.layaResult?.ok) {
|
|
612
|
+
await logVerdicts(layaRerankRows(new Date().toISOString(), state, metas, dual.layaResult.answers))
|
|
613
|
+
}
|
|
614
|
+
return undefined
|
|
615
|
+
}
|
|
559
616
|
const scored: { qid: string; slug: string; instructions: string; probability: number; band: JevBand }[] = []
|
|
560
617
|
for (const m of metas) {
|
|
561
|
-
const probability = noulProbability(
|
|
618
|
+
const probability = noulProbability(dual.answers[m.qid])
|
|
562
619
|
if (probability === undefined) return undefined // 坏答案 → fail-open (§1)
|
|
563
620
|
scored.push({ ...m, probability, band: jevBand(probability, 'rerank') })
|
|
564
621
|
}
|
|
565
622
|
const at = new Date().toISOString()
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
)
|
|
623
|
+
const recs: JevVerdictRecord[] = scored.map((m) => ({
|
|
624
|
+
at,
|
|
625
|
+
lane: 'slowlane-rerank' as const,
|
|
626
|
+
questionId: m.qid,
|
|
627
|
+
qtype: 'noul' as const,
|
|
628
|
+
ref: `slug:${m.slug}`,
|
|
629
|
+
digest: digestFor(state, m.qid, m.instructions),
|
|
630
|
+
probability: m.probability,
|
|
631
|
+
band: m.band,
|
|
632
|
+
agree: 'n/a' as const,
|
|
633
|
+
backend: dual.source,
|
|
634
|
+
degraded: false,
|
|
635
|
+
}))
|
|
636
|
+
// Pace-maker comparison rows (design §3.4): when the primary drove and
|
|
637
|
+
// laya also answered, log laya's verdicts alongside — the permanently
|
|
638
|
+
// running laya-vs-jev comparison. band='record': laya's absolute scores
|
|
639
|
+
// never claim adopt.
|
|
640
|
+
if (dual.layaResult?.ok) recs.push(...layaRerankRows(at, state, metas, dual.layaResult.answers))
|
|
641
|
+
await logVerdicts(recs)
|
|
579
642
|
return scored
|
|
580
643
|
.filter((m) => m.band === 'adopt')
|
|
581
644
|
.sort((a, b) => b.probability - a.probability)
|