@aiwayds/dsh-topics-memory 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +38 -1
- package/lib/config.js +29 -0
- package/lib/digest.d.ts +73 -0
- package/lib/digest.js +384 -0
- package/lib/distill.d.ts +39 -0
- package/lib/distill.js +250 -15
- package/lib/embedding.d.ts +113 -0
- package/lib/embedding.js +208 -0
- package/lib/ilog.d.ts +16 -0
- package/lib/index.d.ts +6 -0
- package/lib/index.js +23 -6
- package/lib/pipeline-events.d.ts +36 -0
- package/lib/pipeline-events.js +121 -0
- package/lib/quality.d.ts +47 -7
- package/lib/quality.js +124 -20
- package/lib/retrieval.d.ts +106 -0
- package/lib/retrieval.js +280 -0
- package/lib/service.d.ts +17 -1
- package/lib/service.js +202 -11
- package/lib/store.d.ts +9 -0
- package/lib/store.js +32 -1
- package/package.json +2 -2
- package/src/config.ts +34 -0
- package/src/digest.ts +416 -0
- package/src/distill.ts +275 -14
- package/src/embedding.ts +294 -0
- package/src/ilog.ts +17 -0
- package/src/index.ts +23 -5
- package/src/pipeline-events.ts +132 -0
- package/src/quality.ts +145 -21
- package/src/retrieval.ts +335 -0
- package/src/service.ts +199 -12
- package/src/store.ts +32 -2
package/lib/embedding.js
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local embedding client (ollama) — the semantic layer behind the nearFloor
|
|
3
|
+
* reranker (A4 决议 2, spec §3.1).
|
|
4
|
+
*
|
|
5
|
+
* Semantics (mirror of jev/client's outcome mode):
|
|
6
|
+
* - every failure returns as a structured {ok:false, outcome} result, NEVER
|
|
7
|
+
* a throw — the caller degrades explicitly to the pure-lexical path;
|
|
8
|
+
* - one AbortSignal.timeout per request, NO retry (query 2s / batch 30s);
|
|
9
|
+
* - localhost only in v1 (ADR 0017 local exception). Remote providers do
|
|
10
|
+
* not exist here by design: a remote embedding may only ever enter a
|
|
11
|
+
* slow-lane/offline path, never the fast lane (red line ①).
|
|
12
|
+
*
|
|
13
|
+
* @module embedding
|
|
14
|
+
*/
|
|
15
|
+
import { execFileSync } from 'node:child_process';
|
|
16
|
+
/** Default ollama bind (the plugin's own default; OLLAMA_HOST overrides). */
|
|
17
|
+
export const DEFAULT_EMBED_HOST = 'http://127.0.0.1:11434';
|
|
18
|
+
/** Provider default model when embeddingModel is the '' sentinel
|
|
19
|
+
* (spec §7: ollama → nomic-embed-text, 768 dims, A1 §5 measured). */
|
|
20
|
+
export const DEFAULT_EMBED_MODEL = 'nomic-embed-text';
|
|
21
|
+
/** Query-side timeout — the reranker sits in the spliced hot path; a slow
|
|
22
|
+
* ollama must lose the race, not the injection. */
|
|
23
|
+
export const EMBED_QUERY_TIMEOUT_MS = 2000;
|
|
24
|
+
/** Batch-side timeout (index rebuild: A1 measured ~9.5s for 137 topics). */
|
|
25
|
+
export const EMBED_BATCH_TIMEOUT_MS = 30_000;
|
|
26
|
+
/** Per-text byte cap before anything leaves the process (query端 p50=153 tok
|
|
27
|
+
* ~ well under this; the cap only bites on paste pollution). */
|
|
28
|
+
export const MAX_TEXT_BYTES = 2048;
|
|
29
|
+
/** The only provider v1 speaks. Anything else is an explicit
|
|
30
|
+
* unsupported_provider outcome, not a silent attempt. */
|
|
31
|
+
export const SUPPORTED_PROVIDER = 'ollama';
|
|
32
|
+
/** Resolve the ollama base URL: explicit arg wins, then OLLAMA_HOST, then
|
|
33
|
+
* the localhost default. Bare `host:port` values get the http:// prefix
|
|
34
|
+
* (ollama's own OLLAMA_HOST convention); trailing slashes are trimmed. */
|
|
35
|
+
export function resolveEmbedHost(explicit) {
|
|
36
|
+
const raw = (explicit ?? process.env.OLLAMA_HOST ?? '').trim();
|
|
37
|
+
const withScheme = raw === '' ? DEFAULT_EMBED_HOST : /^https?:\/\//.test(raw) ? raw : `http://${raw.replace(/^tcp:\/\//, '')}`;
|
|
38
|
+
return withScheme.replace(/\/+$/, '');
|
|
39
|
+
}
|
|
40
|
+
/** '' sentinel → provider default model. */
|
|
41
|
+
export function resolveEmbedModel(model) {
|
|
42
|
+
const m = (model ?? '').trim();
|
|
43
|
+
return m === '' ? DEFAULT_EMBED_MODEL : m;
|
|
44
|
+
}
|
|
45
|
+
/** Truncate a string to ≤ maxBytes UTF-8 bytes WITHOUT splitting a multibyte
|
|
46
|
+
* character: cut at the byte cap, then back off over any continuation bytes
|
|
47
|
+
* (0b10xxxxxx) so the decoded tail is never a replacement char. */
|
|
48
|
+
export function truncateUtf8(text, maxBytes = MAX_TEXT_BYTES) {
|
|
49
|
+
if (maxBytes <= 0)
|
|
50
|
+
return '';
|
|
51
|
+
const buf = Buffer.from(text, 'utf8');
|
|
52
|
+
if (buf.length <= maxBytes)
|
|
53
|
+
return text;
|
|
54
|
+
let end = maxBytes;
|
|
55
|
+
while (end > 0 && (buf[end] & 0xc0) === 0x80)
|
|
56
|
+
end -= 1;
|
|
57
|
+
return buf.subarray(0, end).toString('utf8');
|
|
58
|
+
}
|
|
59
|
+
function isPlainObject(v) {
|
|
60
|
+
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
61
|
+
}
|
|
62
|
+
/** Shared request path: POST {host}/api/embed, outcome-mode failures only. */
|
|
63
|
+
async function embedRequest(texts, config, timeoutMs) {
|
|
64
|
+
const provider = (config.provider ?? '').trim();
|
|
65
|
+
if (provider === '') {
|
|
66
|
+
return { ok: false, outcome: 'provider_off', message: "embeddingProvider is '' — the semantic layer is off (pure lexical).", host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
67
|
+
}
|
|
68
|
+
if (provider !== SUPPORTED_PROVIDER) {
|
|
69
|
+
return { ok: false, outcome: 'unsupported_provider', message: `embeddingProvider '${provider}' is not supported in v1 (only '${SUPPORTED_PROVIDER}'); remote providers may not enter the fast lane.`, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
70
|
+
}
|
|
71
|
+
if (texts.length === 0) {
|
|
72
|
+
return { ok: false, outcome: 'empty_input', message: 'no texts to embed.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
73
|
+
}
|
|
74
|
+
const host = resolveEmbedHost(config.host);
|
|
75
|
+
const model = resolveEmbedModel(config.model);
|
|
76
|
+
const started = Date.now();
|
|
77
|
+
let res;
|
|
78
|
+
try {
|
|
79
|
+
res = await fetch(`${host}/api/embed`, {
|
|
80
|
+
method: 'POST',
|
|
81
|
+
headers: { 'content-type': 'application/json' },
|
|
82
|
+
body: JSON.stringify({ model, input: texts.map((t) => truncateUtf8(t)) }),
|
|
83
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
const outcome = err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError') ? 'timeout' : 'network';
|
|
88
|
+
return { ok: false, outcome, message: `${outcome}: ${err instanceof Error ? err.message : String(err)}`, host, model, latencyMs: Date.now() - started };
|
|
89
|
+
}
|
|
90
|
+
if (!res.ok) {
|
|
91
|
+
const text = (await res.text().catch(() => '')).slice(0, 300);
|
|
92
|
+
// ollama answers 404 {"error":"model \"…\" not found, try pulling it first"}
|
|
93
|
+
const outcome = res.status === 404 ? 'model_missing' : res.status >= 500 ? 'http_5xx' : 'http_4xx';
|
|
94
|
+
return { ok: false, outcome, message: `ollama responded ${res.status}: ${text}`, host, model, latencyMs: Date.now() - started };
|
|
95
|
+
}
|
|
96
|
+
let json;
|
|
97
|
+
try {
|
|
98
|
+
json = await res.json();
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
return { ok: false, outcome: 'bad_json', message: `non-JSON body: ${err instanceof Error ? err.message : String(err)}`, host, model, latencyMs: Date.now() - started };
|
|
102
|
+
}
|
|
103
|
+
const vectors = isPlainObject(json) ? json.embeddings : undefined;
|
|
104
|
+
if (!Array.isArray(vectors) ||
|
|
105
|
+
vectors.length !== texts.length ||
|
|
106
|
+
!vectors.every((v) => Array.isArray(v) && v.length > 0 && v.every((n) => typeof n === 'number' && Number.isFinite(n))) ||
|
|
107
|
+
new Set(vectors.map((v) => v.length)).size !== 1) {
|
|
108
|
+
return { ok: false, outcome: 'bad_shape', message: 'embeddings must be a rectangular [texts.length][dim] array of finite numbers.', host, model, latencyMs: Date.now() - started };
|
|
109
|
+
}
|
|
110
|
+
return { ok: true, vectors: vectors, model, dim: vectors[0].length, latencyMs: Date.now() - started };
|
|
111
|
+
}
|
|
112
|
+
/** Batch-embed texts (index build/rebuild path). Vectors align 1:1 with the
|
|
113
|
+
* input order; each text is truncated to MAX_TEXT_BYTES first. */
|
|
114
|
+
export async function embedTexts(texts, config = {}, opts = {}) {
|
|
115
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_BATCH_TIMEOUT_MS;
|
|
116
|
+
return embedRequest(texts, config, timeoutMs);
|
|
117
|
+
}
|
|
118
|
+
/** Embed one query (reranker hot path, 2s budget). Empty/whitespace queries
|
|
119
|
+
* fail locally with 'empty_input' — nothing leaves the process. */
|
|
120
|
+
export async function embedQuery(query, config = {}, opts = {}) {
|
|
121
|
+
if ((query ?? '').trim() === '') {
|
|
122
|
+
return { ok: false, outcome: 'empty_input', message: 'empty query.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
123
|
+
}
|
|
124
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_QUERY_TIMEOUT_MS;
|
|
125
|
+
const r = await embedRequest([query], config, timeoutMs);
|
|
126
|
+
if (!r.ok)
|
|
127
|
+
return r;
|
|
128
|
+
return { ok: true, vector: r.vectors[0], model: r.model, dim: r.dim, latencyMs: r.latencyMs };
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Probe: is the server there, is the model pulled? One micro embed call —
|
|
132
|
+
* unlike a /api/tags listing it validates the FULL path (server + model +
|
|
133
|
+
* response shape) and yields the vector dim for index validation, at the
|
|
134
|
+
* same 15-30ms cost (A1 §5). Structured result; never throws.
|
|
135
|
+
*/
|
|
136
|
+
export async function probeEmbedding(config = {}, opts = {}) {
|
|
137
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_QUERY_TIMEOUT_MS;
|
|
138
|
+
const r = await embedRequest(['dsh-topics-memory semantic layer probe'], config, timeoutMs);
|
|
139
|
+
if (!r.ok)
|
|
140
|
+
return r;
|
|
141
|
+
return { ok: true, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), dim: r.dim, latencyMs: r.latencyMs };
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Build an embedder handle for retrieval's `ensureIndexFresh` — a callable
|
|
145
|
+
* embedTexts bound to this config, carrying the RESOLVED model identity the
|
|
146
|
+
* index must match. ensureIndexFresh reads `.model` only; the actual
|
|
147
|
+
* rebuild stays with the caller.
|
|
148
|
+
*/
|
|
149
|
+
export function embedderWith(config) {
|
|
150
|
+
const fn = (texts) => embedTexts(texts, config);
|
|
151
|
+
return Object.assign(fn, { model: resolveEmbedModel(config.model) });
|
|
152
|
+
}
|
|
153
|
+
// --- Sync query embed (Phase-2 wiring) --------------------------------------
|
|
154
|
+
// The spliced hot path is synchronous (the chancelu lesson): retrieveSync
|
|
155
|
+
// cannot await. A local ollama query is 15-30ms steady state (A1 §5), so a
|
|
156
|
+
// bounded SYNC subprocess call keeps the reranker on the hot lane inside the
|
|
157
|
+
// ADR 0017 local exception — hard-capped so a wedged ollama loses the race,
|
|
158
|
+
// never the injection. curl ships with the host OS; node has no sync HTTP.
|
|
159
|
+
/** Hard sync budget for the hot-lane query embed. Generous over the measured
|
|
160
|
+
* 15-30ms, merciless for a wedged server. */
|
|
161
|
+
export const EMBED_QUERY_SYNC_TIMEOUT_MS = 250;
|
|
162
|
+
export function embedQuerySync(query, config = {}) {
|
|
163
|
+
const provider = (config.provider ?? '').trim();
|
|
164
|
+
if (provider === '') {
|
|
165
|
+
return { ok: false, outcome: 'provider_off', message: "embeddingProvider is '' — the semantic layer is off.", host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
166
|
+
}
|
|
167
|
+
if (provider !== SUPPORTED_PROVIDER) {
|
|
168
|
+
return { ok: false, outcome: 'unsupported_provider', message: `embeddingProvider '${provider}' is not supported in v1.`, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
169
|
+
}
|
|
170
|
+
if ((query ?? '').trim() === '') {
|
|
171
|
+
return { ok: false, outcome: 'empty_input', message: 'empty query.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
172
|
+
}
|
|
173
|
+
const host = resolveEmbedHost(config.host);
|
|
174
|
+
const model = resolveEmbedModel(config.model);
|
|
175
|
+
const started = Date.now();
|
|
176
|
+
let raw;
|
|
177
|
+
try {
|
|
178
|
+
raw = execFileSync('curl', [
|
|
179
|
+
'-sS',
|
|
180
|
+
'--max-time', String(Math.ceil(EMBED_QUERY_SYNC_TIMEOUT_MS / 1000)),
|
|
181
|
+
'-X', 'POST', `${host}/api/embed`,
|
|
182
|
+
'-H', 'content-type: application/json',
|
|
183
|
+
'-d', JSON.stringify({ model, input: [truncateUtf8(query)] }),
|
|
184
|
+
], { timeout: EMBED_QUERY_SYNC_TIMEOUT_MS, encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] });
|
|
185
|
+
}
|
|
186
|
+
catch (err) {
|
|
187
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
188
|
+
const outcome = /timed out|ETIMEDOUT|maxTime/i.test(message) ? 'timeout' : 'network';
|
|
189
|
+
return { ok: false, outcome, message: `${outcome}: ${message}`, host, model, latencyMs: Date.now() - started };
|
|
190
|
+
}
|
|
191
|
+
let json;
|
|
192
|
+
try {
|
|
193
|
+
json = JSON.parse(raw);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return { ok: false, outcome: 'bad_json', message: 'ollama returned non-JSON (sync path).', host, model, latencyMs: Date.now() - started };
|
|
197
|
+
}
|
|
198
|
+
const embeddings = isPlainObject(json) ? json.embeddings : undefined;
|
|
199
|
+
const first = Array.isArray(embeddings) && embeddings.length > 0 ? embeddings[0] : undefined;
|
|
200
|
+
if (!Array.isArray(first)) {
|
|
201
|
+
return { ok: false, outcome: 'bad_shape', message: 'ollama response has no embeddings[0] array (sync path).', host, model, latencyMs: Date.now() - started };
|
|
202
|
+
}
|
|
203
|
+
const vector = first.filter((n) => typeof n === 'number');
|
|
204
|
+
if (vector.length === 0 || vector.length !== first.length) {
|
|
205
|
+
return { ok: false, outcome: 'bad_shape', message: 'embeddings[0] malformed (sync path).', host, model, latencyMs: Date.now() - started };
|
|
206
|
+
}
|
|
207
|
+
return { ok: true, vector, model, dim: vector.length, latencyMs: Date.now() - started };
|
|
208
|
+
}
|
package/lib/ilog.d.ts
CHANGED
|
@@ -23,6 +23,13 @@ export interface SlowItem {
|
|
|
23
23
|
slug: string;
|
|
24
24
|
why: string;
|
|
25
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Slow-lane full-lifecycle outcome (A4 决议 3, spec §3.2): where a slow-lane
|
|
28
|
+
* run (or round with a slow-lane stake) ended. Precedent = decisions.jsonl's
|
|
29
|
+
* outcome enum — a closed vocabulary so 「没跑」(no-run/no-claim) and 「跑了
|
|
30
|
+
* 全败」(llm-error/timeout) are finally distinguishable in the log.
|
|
31
|
+
*/
|
|
32
|
+
export type SlowOutcome = 'no-run' | 'no-claim' | 'no-candidates' | 'llm-error' | 'timeout' | 'delivered' | 'expired';
|
|
26
33
|
export interface InjectionRecord {
|
|
27
34
|
at: string;
|
|
28
35
|
sessionId?: string;
|
|
@@ -69,6 +76,15 @@ export interface InjectionRecord {
|
|
|
69
76
|
slowMs?: number;
|
|
70
77
|
/** Slow-lane pointers this round, in delivery order. */
|
|
71
78
|
slow?: SlowItem[];
|
|
79
|
+
/**
|
|
80
|
+
* Slow-lane disposition for this round (A4 决议 3, spec §8.2): `delivered`
|
|
81
|
+
* when a pending was consumed and packed, `expired` at its hard bounds, and
|
|
82
|
+
* the producing run's failure point (no-run/no-claim/no-candidates/
|
|
83
|
+
* llm-error/timeout) when a spliced arrived with no pending. Optional —
|
|
84
|
+
* older lines predate the field and pure fast-lane rounds may omit it.
|
|
85
|
+
* Written once with the record, never backfilled (ilog append-only).
|
|
86
|
+
*/
|
|
87
|
+
slowOutcome?: SlowOutcome;
|
|
72
88
|
}
|
|
73
89
|
/**
|
|
74
90
|
* One topic_open event (pointer-open log, v4 §4.3). Everything beyond
|
package/lib/index.d.ts
CHANGED
|
@@ -121,6 +121,9 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
121
121
|
jevSecretFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
122
122
|
jevLayaFallback: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
123
123
|
jevLayaUrl: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
124
|
+
embeddingProvider: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
125
|
+
embeddingModel: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
126
|
+
packageMode: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
124
127
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
125
128
|
repo: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
126
129
|
autoInject: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
@@ -156,6 +159,9 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
156
159
|
jevSecretFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
157
160
|
jevLayaFallback: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
158
161
|
jevLayaUrl: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
162
|
+
embeddingProvider: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
163
|
+
embeddingModel: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
164
|
+
packageMode: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
159
165
|
}>>, "plain">;
|
|
160
166
|
/** Live config reference as a 0.1.7 host passes it (cron / mcp-adapter
|
|
161
167
|
* spelling): `.get()` snapshots the current value — the host swaps the
|
package/lib/index.js
CHANGED
|
@@ -359,8 +359,9 @@ export async function apply(ctx, config = {}) {
|
|
|
359
359
|
if (cfg.autoInject) {
|
|
360
360
|
const consumed = slowLane.consume(sessionId, observer.turnCountOf(sessionId));
|
|
361
361
|
if (consumed !== undefined) {
|
|
362
|
-
if ('pending' in consumed)
|
|
363
|
-
slow = consumed.pending;
|
|
362
|
+
if ('pending' in consumed) {
|
|
363
|
+
slow = { ...consumed.pending, outcome: consumed.pending.outcome };
|
|
364
|
+
}
|
|
364
365
|
else
|
|
365
366
|
slowExpired = consumed.expired;
|
|
366
367
|
}
|
|
@@ -578,7 +579,10 @@ export async function apply(ctx, config = {}) {
|
|
|
578
579
|
// and its own settle hook does the release.
|
|
579
580
|
const run = distiller.request(sessionId, reason);
|
|
580
581
|
if (run !== undefined) {
|
|
581
|
-
void run
|
|
582
|
+
void run
|
|
583
|
+
.finally(() => releaseSessionLlm(sessionId))
|
|
584
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
585
|
+
.catch(() => undefined);
|
|
582
586
|
}
|
|
583
587
|
});
|
|
584
588
|
// ---- Session events: injection + observation ----
|
|
@@ -786,8 +790,11 @@ export async function apply(ctx, config = {}) {
|
|
|
786
790
|
// no-adapter detail instead
|
|
787
791
|
}
|
|
788
792
|
const run = distiller.request(sessionId, 'boot-replay');
|
|
789
|
-
if (run !== undefined)
|
|
790
|
-
void run
|
|
793
|
+
if (run !== undefined) {
|
|
794
|
+
void run
|
|
795
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
796
|
+
.catch(() => undefined);
|
|
797
|
+
}
|
|
791
798
|
})
|
|
792
799
|
.catch(() => undefined);
|
|
793
800
|
// Consolidation cadence check — after the pull (freshest pool),
|
|
@@ -823,6 +830,9 @@ export async function apply(ctx, config = {}) {
|
|
|
823
830
|
})
|
|
824
831
|
.catch(() => undefined);
|
|
825
832
|
}
|
|
833
|
+
// Embedding index maintenance (A4/A6, spec §3): boot-time freshness
|
|
834
|
+
// check + incremental rebuild — provider '' / packageMode off = no-op.
|
|
835
|
+
void service.refreshEmbeddingIndex().catch(() => undefined);
|
|
826
836
|
})
|
|
827
837
|
.catch((error) => {
|
|
828
838
|
warn(`dsh-topics-memory 启动链失败(不影响会话启动):${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -948,7 +958,11 @@ export async function apply(ctx, config = {}) {
|
|
|
948
958
|
? { ok: false, reason: 'in-flight', created: [], updated: [], marked: 0 }
|
|
949
959
|
: { ok: false, reason: 'no-model', created: [], updated: [], marked: 0 };
|
|
950
960
|
}
|
|
951
|
-
|
|
961
|
+
void run
|
|
962
|
+
.finally(() => releaseSessionLlm(sessionId))
|
|
963
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
964
|
+
.catch(() => undefined);
|
|
965
|
+
return run;
|
|
952
966
|
};
|
|
953
967
|
// Manual /topics consolidate trigger: same lane as the cadence path, same
|
|
954
968
|
// trigger-time llm capture and in-flight guard (the command runs inside a
|
|
@@ -1018,6 +1032,9 @@ const DEFAULTS = {
|
|
|
1018
1032
|
distillMaxModelCalls: 8,
|
|
1019
1033
|
consolidateCadence: 'daily',
|
|
1020
1034
|
deprecatedTtlDays: 15,
|
|
1035
|
+
embeddingProvider: '',
|
|
1036
|
+
embeddingModel: '',
|
|
1037
|
+
packageMode: 'off',
|
|
1021
1038
|
usageBoost: 0.15,
|
|
1022
1039
|
pushDebounceSeconds: 45,
|
|
1023
1040
|
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* meta/pipeline-events.jsonl — the pipeline-level anomaly ledger (spec §8.1,
|
|
3
|
+
* A2/A6 BL-1 revised: the "记 wtf 账" mechanism never existed, this IS the
|
|
4
|
+
* explicit ledger). One line JSON per event; always on, no config key
|
|
5
|
+
* (恒开无键 — observability is not a gray-release feature).
|
|
6
|
+
*
|
|
7
|
+
* Discipline = decisions.jsonl (src/jev/log.ts) verbatim: field-whitelist
|
|
8
|
+
* copy on write (extra caller fields cannot leak in, detail values must be
|
|
9
|
+
* scalar), 512KB auto-compaction keeping the most recent quarter, and write
|
|
10
|
+
* failures swallowed silently — a broken sink must never break the pipeline.
|
|
11
|
+
* Unlike jev/log this writer takes the bundle root explicitly (callers like
|
|
12
|
+
* the slow lane already hold it; tests point it at a tmp dir).
|
|
13
|
+
*
|
|
14
|
+
* @module pipeline-events
|
|
15
|
+
*/
|
|
16
|
+
/** Closed vocabulary, spec §8.1 + W-C wave start set. `slow-lane-outcome` is
|
|
17
|
+
* the slow lane's per-run disposition row (quality.ts); the distill/link
|
|
18
|
+
* kinds are the A2/A6 producers' contract — they import this module. */
|
|
19
|
+
export type PipelineEventKind = 'slow-lane-outcome' | 'distill-op-invalid' | 'distill-op-retry-exhausted' | 'link-ref-rejected';
|
|
20
|
+
/** File NAME inside the bundle's meta dir (path via pipelineEventsFile). */
|
|
21
|
+
export declare const PIPELINE_EVENTS_FILE = "pipeline-events.jsonl";
|
|
22
|
+
export interface PipelineEvent {
|
|
23
|
+
at: string;
|
|
24
|
+
kind: PipelineEventKind;
|
|
25
|
+
/** Bounded scalar metadata only — never conversation or conclusion text. */
|
|
26
|
+
detail?: Record<string, string | number | boolean>;
|
|
27
|
+
}
|
|
28
|
+
export declare function pipelineEventsFile(bundleDir: string): string;
|
|
29
|
+
/**
|
|
30
|
+
* Copy one event through the field whitelist onto the ledger. Only `at`,
|
|
31
|
+
* `kind` and a scalar-valued `detail` map can pass; non-scalar detail values
|
|
32
|
+
* (objects/arrays/null/NaN), surplus entries and over-long keys/values are
|
|
33
|
+
* dropped here so no unbounded content can ride in. Write failures are
|
|
34
|
+
* swallowed downstream — this never rejects and never throws.
|
|
35
|
+
*/
|
|
36
|
+
export declare function recordPipelineEvent(bundleDir: string, event: PipelineEvent): Promise<void>;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* meta/pipeline-events.jsonl — the pipeline-level anomaly ledger (spec §8.1,
|
|
3
|
+
* A2/A6 BL-1 revised: the "记 wtf 账" mechanism never existed, this IS the
|
|
4
|
+
* explicit ledger). One line JSON per event; always on, no config key
|
|
5
|
+
* (恒开无键 — observability is not a gray-release feature).
|
|
6
|
+
*
|
|
7
|
+
* Discipline = decisions.jsonl (src/jev/log.ts) verbatim: field-whitelist
|
|
8
|
+
* copy on write (extra caller fields cannot leak in, detail values must be
|
|
9
|
+
* scalar), 512KB auto-compaction keeping the most recent quarter, and write
|
|
10
|
+
* failures swallowed silently — a broken sink must never break the pipeline.
|
|
11
|
+
* Unlike jev/log this writer takes the bundle root explicitly (callers like
|
|
12
|
+
* the slow lane already hold it; tests point it at a tmp dir).
|
|
13
|
+
*
|
|
14
|
+
* @module pipeline-events
|
|
15
|
+
*/
|
|
16
|
+
import { appendFile, mkdir, readFile, rename, writeFile } from 'node:fs/promises';
|
|
17
|
+
import { randomUUID } from 'node:crypto';
|
|
18
|
+
import { dirname, join } from 'node:path';
|
|
19
|
+
import { metaDir } from "./paths.js";
|
|
20
|
+
/** File NAME inside the bundle's meta dir (path via pipelineEventsFile). */
|
|
21
|
+
export const PIPELINE_EVENTS_FILE = 'pipeline-events.jsonl';
|
|
22
|
+
const COMPACT_LIMIT = 512 * 1024; // ~512KB cap, same as the decisions/ilog sidecars
|
|
23
|
+
/** Whitelist copy bounds: detail entries and their values stay metadata-sized. */
|
|
24
|
+
const DETAIL_MAX_ENTRIES = 16;
|
|
25
|
+
const DETAIL_KEY_MAX_CHARS = 64;
|
|
26
|
+
const DETAIL_VALUE_MAX_CHARS = 256;
|
|
27
|
+
/** Runtime mirror of PipelineEventKind — the closed vocabulary is enforced
|
|
28
|
+
* at the write gate too (a JS caller cannot smuggle an unknown kind in). */
|
|
29
|
+
const KNOWN_KINDS = new Set([
|
|
30
|
+
'slow-lane-outcome',
|
|
31
|
+
'distill-op-invalid',
|
|
32
|
+
'distill-op-retry-exhausted',
|
|
33
|
+
'link-ref-rejected',
|
|
34
|
+
]);
|
|
35
|
+
export function pipelineEventsFile(bundleDir) {
|
|
36
|
+
return join(metaDir(bundleDir), PIPELINE_EVENTS_FILE);
|
|
37
|
+
}
|
|
38
|
+
// Serialized like jev/log: single-record appends are line-atomic anyway, but
|
|
39
|
+
// the compaction rewrite must not interleave with an append.
|
|
40
|
+
let queue = Promise.resolve();
|
|
41
|
+
function enqueue(op) {
|
|
42
|
+
const run = queue.then(op, op);
|
|
43
|
+
queue = run.catch(() => undefined);
|
|
44
|
+
return run;
|
|
45
|
+
}
|
|
46
|
+
/** All write failures die HERE — never upstream. */
|
|
47
|
+
async function appendJsonl(file, lines) {
|
|
48
|
+
await enqueue(async () => {
|
|
49
|
+
try {
|
|
50
|
+
await mkdir(dirname(file), { recursive: true });
|
|
51
|
+
await appendFile(file, lines.map((l) => `${l}\n`).join(''), 'utf8');
|
|
52
|
+
await compactIfNeeded(file);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
// fail-open: a broken ledger must never break the pipeline (§8.1)
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
// jev/log compactIfNeeded pattern: ~512KB cap; keep the most recent quarter
|
|
60
|
+
// when exceeded (headroom for the next window).
|
|
61
|
+
async function compactIfNeeded(file) {
|
|
62
|
+
let raw;
|
|
63
|
+
try {
|
|
64
|
+
raw = await readFile(file, 'utf8');
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
if (raw.length <= COMPACT_LIMIT)
|
|
70
|
+
return;
|
|
71
|
+
const lines = raw.split('\n').filter((l) => l.trim() !== '');
|
|
72
|
+
const keep = lines.slice(-Math.max(1, Math.floor(lines.length / 4)));
|
|
73
|
+
const tmp = `${file}.tmp-${randomUUID()}`;
|
|
74
|
+
try {
|
|
75
|
+
await writeFile(tmp, keep.map((l) => `${l}\n`).join(''), 'utf8');
|
|
76
|
+
await rename(tmp, file);
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
// swallow — the next append retries the compaction
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Copy one event through the field whitelist onto the ledger. Only `at`,
|
|
84
|
+
* `kind` and a scalar-valued `detail` map can pass; non-scalar detail values
|
|
85
|
+
* (objects/arrays/null/NaN), surplus entries and over-long keys/values are
|
|
86
|
+
* dropped here so no unbounded content can ride in. Write failures are
|
|
87
|
+
* swallowed downstream — this never rejects and never throws.
|
|
88
|
+
*/
|
|
89
|
+
export async function recordPipelineEvent(bundleDir, event) {
|
|
90
|
+
if (bundleDir === '' || typeof bundleDir !== 'string')
|
|
91
|
+
return;
|
|
92
|
+
if (event === null || typeof event !== 'object')
|
|
93
|
+
return;
|
|
94
|
+
if (typeof event.kind !== 'string' || !KNOWN_KINDS.has(event.kind))
|
|
95
|
+
return;
|
|
96
|
+
const detail = {};
|
|
97
|
+
if (event.detail !== undefined && typeof event.detail === 'object') {
|
|
98
|
+
for (const [key, value] of Object.entries(event.detail)) {
|
|
99
|
+
if (Object.keys(detail).length >= DETAIL_MAX_ENTRIES)
|
|
100
|
+
break;
|
|
101
|
+
const t = typeof value;
|
|
102
|
+
if (t === 'string') {
|
|
103
|
+
if (value !== '')
|
|
104
|
+
detail[key.slice(0, DETAIL_KEY_MAX_CHARS)] = value.slice(0, DETAIL_VALUE_MAX_CHARS);
|
|
105
|
+
}
|
|
106
|
+
else if (t === 'number') {
|
|
107
|
+
if (Number.isFinite(value))
|
|
108
|
+
detail[key.slice(0, DETAIL_KEY_MAX_CHARS)] = value;
|
|
109
|
+
}
|
|
110
|
+
else if (t === 'boolean') {
|
|
111
|
+
detail[key.slice(0, DETAIL_KEY_MAX_CHARS)] = value;
|
|
112
|
+
}
|
|
113
|
+
// objects/arrays/null/undefined never pass the whitelist
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const at = typeof event.at === 'string' && event.at !== '' ? event.at : new Date().toISOString();
|
|
117
|
+
const row = Object.keys(detail).length === 0
|
|
118
|
+
? { at, kind: event.kind }
|
|
119
|
+
: { at, kind: event.kind, detail };
|
|
120
|
+
await appendJsonl(pipelineEventsFile(bundleDir), [JSON.stringify(row)]);
|
|
121
|
+
}
|
package/lib/quality.d.ts
CHANGED
|
@@ -15,8 +15,11 @@
|
|
|
15
15
|
* Picks rest in a per-session pending slot and inject at the NEXT spliced
|
|
16
16
|
* (steer message), then the slot is gone (消费即清). Hard bounds: per-call
|
|
17
17
|
* 20s, pipeline 45s, pending TTL 10min, turn-lag ≤2. Every failure is
|
|
18
|
-
* contained
|
|
19
|
-
*
|
|
18
|
+
* contained — async products may be missing, never blocking (I3 可丢弃性) —
|
|
19
|
+
* but no longer SILENT (A4 决议 3): each run lands a `slow-lane-outcome`
|
|
20
|
+
* disposition (SlowOutcome) on the session and in the pipeline-events
|
|
21
|
+
* ledger, so 「没跑」and 「跑了全败」are distinguishable in the log. The
|
|
22
|
+
* lexical re-gate at consumption is SHADOW-ONLY (v4 B3): it
|
|
20
23
|
* records verdicts into ilog and never blocks a pick.
|
|
21
24
|
*
|
|
22
25
|
* @module quality
|
|
@@ -24,7 +27,10 @@
|
|
|
24
27
|
import type { TopicsService } from './service.ts';
|
|
25
28
|
import type { ModelCaller } from './distill.ts';
|
|
26
29
|
import type { RingEntry } from './observer.ts';
|
|
27
|
-
import type { QueryBuildShape } from './ilog.ts';
|
|
30
|
+
import type { QueryBuildShape, SlowOutcome } from './ilog.ts';
|
|
31
|
+
/** Re-exported for the ilog writers (spec §8.2): the service layer takes a
|
|
32
|
+
* pending's `outcome` (or lastOutcomeOf) as the record's `slowOutcome`. */
|
|
33
|
+
export type { SlowOutcome };
|
|
28
34
|
export declare const PENDING_TTL_MS: number;
|
|
29
35
|
/** Hard drift bound: a pending older than this many turn-ends expires. */
|
|
30
36
|
export declare const TURN_LAG_LIMIT = 2;
|
|
@@ -69,6 +75,13 @@ export interface PendingInjection {
|
|
|
69
75
|
queryBuild: QueryBuildShape;
|
|
70
76
|
model: string;
|
|
71
77
|
ms: number;
|
|
78
|
+
/**
|
|
79
|
+
* The producing run's disposition (A4 决议 3, spec §8.2) — always
|
|
80
|
+
* 'delivered' here (a pending only exists after a settled run). The
|
|
81
|
+
* service layer copies it into the ilog record's `slowOutcome` at
|
|
82
|
+
* consumption; expiry surfaces as `expired` via ConsumeResult instead.
|
|
83
|
+
*/
|
|
84
|
+
outcome: SlowOutcome;
|
|
72
85
|
}
|
|
73
86
|
export type ConsumeResult = {
|
|
74
87
|
pending: PendingInjection;
|
|
@@ -94,6 +107,13 @@ export declare class SlowLane {
|
|
|
94
107
|
private inFlight;
|
|
95
108
|
/** Same-session shadow serialization — a second trigger never overlaps. */
|
|
96
109
|
private shadowInFlight;
|
|
110
|
+
/**
|
|
111
|
+
* The session's most recent slow-lane disposition (A4 决议 3): what a
|
|
112
|
+
* spliced with NO pending reports as its `slowOutcome` — without it a
|
|
113
|
+
* consumed-undefined round cannot say whether the lane never ran or ran
|
|
114
|
+
* and failed. One slot per session, overwritten by every trigger.
|
|
115
|
+
*/
|
|
116
|
+
private outcomeBySession;
|
|
97
117
|
private readonly service;
|
|
98
118
|
private readonly caller;
|
|
99
119
|
/**
|
|
@@ -109,6 +129,20 @@ export declare class SlowLane {
|
|
|
109
129
|
hasPending(sessionId: string): boolean;
|
|
110
130
|
/** True while the session's produce pipeline is still running (llm-capture guard). */
|
|
111
131
|
hasInFlight(sessionId: string): boolean;
|
|
132
|
+
/**
|
|
133
|
+
* The session's most recent slow-lane disposition (A4 §3.2): a round whose
|
|
134
|
+
* consume() came up empty writes THIS as its ilog `slowOutcome` (spec §8.2
|
|
135
|
+
* — 「spliced 收到但未 produce 也留痕」). Undefined when the lane never
|
|
136
|
+
* triggered for the session.
|
|
137
|
+
*/
|
|
138
|
+
lastOutcomeOf(sessionId: string): SlowOutcome | undefined;
|
|
139
|
+
/**
|
|
140
|
+
* One disposition, two sinks (A4 决议 3): the per-session slot above and a
|
|
141
|
+
* `slow-lane-outcome` row in the pipeline-events ledger. Metadata only
|
|
142
|
+
* (outcome/reason/session/turn/timing — never ring or query text). Never
|
|
143
|
+
* throws, never rejects: traceability must not become a failure source.
|
|
144
|
+
*/
|
|
145
|
+
private traceOutcome;
|
|
112
146
|
/** Drop the pending slot (session teardown, restore boundary). */
|
|
113
147
|
clear(sessionId: string): void;
|
|
114
148
|
/**
|
|
@@ -126,13 +160,19 @@ export declare class SlowLane {
|
|
|
126
160
|
private fastGateShadow;
|
|
127
161
|
/**
|
|
128
162
|
* Produce a pending slot for the session's next steer message. Fire-and-
|
|
129
|
-
* forget; every guard failure is
|
|
130
|
-
*
|
|
131
|
-
* the
|
|
163
|
+
* forget; every guard failure is contained (never breaks the event
|
|
164
|
+
* handler) and QUALIFIED (A4 决议 3): a skipped trigger lands a `no-run`
|
|
165
|
+
* outcome naming the guard, so a later spliced can report why there was
|
|
166
|
+
* nothing to consume. Policy gates live here (config, sampling, in-flight);
|
|
167
|
+
* host gates (delegation, distill yield) live with the caller, which owns
|
|
168
|
+
* those services.
|
|
132
169
|
*/
|
|
133
170
|
dispatch(sessionId: string, input: DispatchInput): void;
|
|
134
171
|
private produce;
|
|
135
|
-
/** One pending slot per settled pipeline (the 0-2 picks + lane metadata).
|
|
172
|
+
/** One pending slot per settled pipeline (the 0-2 picks + lane metadata).
|
|
173
|
+
* The pending carries its producing run's disposition (A4 §3.2) so the
|
|
174
|
+
* ilog writer can copy it verbatim — a pending only exists when the run
|
|
175
|
+
* delivered, hence the constant. */
|
|
136
176
|
private settlePending;
|
|
137
177
|
/**
|
|
138
178
|
* The jev rerank (design §2 row 1): one batched noul request over the whole
|