@aiwayds/dsh-topics-memory 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +55 -1
- package/lib/config.js +42 -0
- package/lib/digest.d.ts +111 -5
- package/lib/digest.js +437 -11
- package/lib/distill.d.ts +39 -0
- package/lib/distill.js +250 -15
- package/lib/embedding.d.ts +113 -0
- package/lib/embedding.js +208 -0
- package/lib/ilog.d.ts +30 -0
- package/lib/index.d.ts +15 -1
- package/lib/index.js +24 -6
- package/lib/okf.d.ts +34 -0
- package/lib/okf.js +41 -0
- package/lib/paths.d.ts +9 -0
- package/lib/paths.js +15 -0
- package/lib/pipeline-events.d.ts +38 -0
- package/lib/pipeline-events.js +123 -0
- package/lib/quality.d.ts +47 -7
- package/lib/quality.js +124 -20
- package/lib/retrieval.d.ts +106 -0
- package/lib/retrieval.js +280 -0
- package/lib/service.d.ts +103 -2
- package/lib/service.js +623 -16
- package/lib/store.d.ts +39 -3
- package/lib/store.js +60 -2
- package/package.json +2 -2
- package/src/config.ts +49 -0
- package/src/digest.ts +497 -8
- package/src/distill.ts +275 -14
- package/src/embedding.ts +294 -0
- package/src/ilog.ts +26 -0
- package/src/index.ts +32 -5
- package/src/okf.ts +68 -0
- package/src/paths.ts +15 -0
- package/src/pipeline-events.ts +136 -0
- package/src/quality.ts +145 -21
- package/src/retrieval.ts +335 -0
- package/src/service.ts +661 -19
- package/src/store.ts +77 -5
package/lib/embedding.js
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local embedding client (ollama) — the semantic layer behind the nearFloor
|
|
3
|
+
* reranker (A4 决议 2, spec §3.1).
|
|
4
|
+
*
|
|
5
|
+
* Semantics (mirror of jev/client's outcome mode):
|
|
6
|
+
* - every failure returns as a structured {ok:false, outcome} result, NEVER
|
|
7
|
+
* a throw — the caller degrades explicitly to the pure-lexical path;
|
|
8
|
+
* - one AbortSignal.timeout per request, NO retry (query 2s / batch 30s);
|
|
9
|
+
* - localhost only in v1 (ADR 0017 local exception). Remote providers do
|
|
10
|
+
* not exist here by design: a remote embedding may only ever enter a
|
|
11
|
+
* slow-lane/offline path, never the fast lane (red line ①).
|
|
12
|
+
*
|
|
13
|
+
* @module embedding
|
|
14
|
+
*/
|
|
15
|
+
import { execFileSync } from 'node:child_process';
|
|
16
|
+
/** Default ollama bind (the plugin's own default; OLLAMA_HOST overrides). */
|
|
17
|
+
export const DEFAULT_EMBED_HOST = 'http://127.0.0.1:11434';
|
|
18
|
+
/** Provider default model when embeddingModel is the '' sentinel
|
|
19
|
+
* (spec §7: ollama → nomic-embed-text, 768 dims, A1 §5 measured). */
|
|
20
|
+
export const DEFAULT_EMBED_MODEL = 'nomic-embed-text';
|
|
21
|
+
/** Query-side timeout — the reranker sits in the spliced hot path; a slow
|
|
22
|
+
* ollama must lose the race, not the injection. */
|
|
23
|
+
export const EMBED_QUERY_TIMEOUT_MS = 2000;
|
|
24
|
+
/** Batch-side timeout (index rebuild: A1 measured ~9.5s for 137 topics). */
|
|
25
|
+
export const EMBED_BATCH_TIMEOUT_MS = 30_000;
|
|
26
|
+
/** Per-text byte cap before anything leaves the process (query端 p50=153 tok
|
|
27
|
+
* ~ well under this; the cap only bites on paste pollution). */
|
|
28
|
+
export const MAX_TEXT_BYTES = 2048;
|
|
29
|
+
/** The only provider v1 speaks. Anything else is an explicit
|
|
30
|
+
* unsupported_provider outcome, not a silent attempt. */
|
|
31
|
+
export const SUPPORTED_PROVIDER = 'ollama';
|
|
32
|
+
/** Resolve the ollama base URL: explicit arg wins, then OLLAMA_HOST, then
|
|
33
|
+
* the localhost default. Bare `host:port` values get the http:// prefix
|
|
34
|
+
* (ollama's own OLLAMA_HOST convention); trailing slashes are trimmed. */
|
|
35
|
+
export function resolveEmbedHost(explicit) {
|
|
36
|
+
const raw = (explicit ?? process.env.OLLAMA_HOST ?? '').trim();
|
|
37
|
+
const withScheme = raw === '' ? DEFAULT_EMBED_HOST : /^https?:\/\//.test(raw) ? raw : `http://${raw.replace(/^tcp:\/\//, '')}`;
|
|
38
|
+
return withScheme.replace(/\/+$/, '');
|
|
39
|
+
}
|
|
40
|
+
/** '' sentinel → provider default model. */
|
|
41
|
+
export function resolveEmbedModel(model) {
|
|
42
|
+
const m = (model ?? '').trim();
|
|
43
|
+
return m === '' ? DEFAULT_EMBED_MODEL : m;
|
|
44
|
+
}
|
|
45
|
+
/** Truncate a string to ≤ maxBytes UTF-8 bytes WITHOUT splitting a multibyte
|
|
46
|
+
* character: cut at the byte cap, then back off over any continuation bytes
|
|
47
|
+
* (0b10xxxxxx) so the decoded tail is never a replacement char. */
|
|
48
|
+
export function truncateUtf8(text, maxBytes = MAX_TEXT_BYTES) {
|
|
49
|
+
if (maxBytes <= 0)
|
|
50
|
+
return '';
|
|
51
|
+
const buf = Buffer.from(text, 'utf8');
|
|
52
|
+
if (buf.length <= maxBytes)
|
|
53
|
+
return text;
|
|
54
|
+
let end = maxBytes;
|
|
55
|
+
while (end > 0 && (buf[end] & 0xc0) === 0x80)
|
|
56
|
+
end -= 1;
|
|
57
|
+
return buf.subarray(0, end).toString('utf8');
|
|
58
|
+
}
|
|
59
|
+
function isPlainObject(v) {
|
|
60
|
+
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
61
|
+
}
|
|
62
|
+
/** Shared request path: POST {host}/api/embed, outcome-mode failures only. */
|
|
63
|
+
async function embedRequest(texts, config, timeoutMs) {
|
|
64
|
+
const provider = (config.provider ?? '').trim();
|
|
65
|
+
if (provider === '') {
|
|
66
|
+
return { ok: false, outcome: 'provider_off', message: "embeddingProvider is '' — the semantic layer is off (pure lexical).", host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
67
|
+
}
|
|
68
|
+
if (provider !== SUPPORTED_PROVIDER) {
|
|
69
|
+
return { ok: false, outcome: 'unsupported_provider', message: `embeddingProvider '${provider}' is not supported in v1 (only '${SUPPORTED_PROVIDER}'); remote providers may not enter the fast lane.`, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
70
|
+
}
|
|
71
|
+
if (texts.length === 0) {
|
|
72
|
+
return { ok: false, outcome: 'empty_input', message: 'no texts to embed.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
73
|
+
}
|
|
74
|
+
const host = resolveEmbedHost(config.host);
|
|
75
|
+
const model = resolveEmbedModel(config.model);
|
|
76
|
+
const started = Date.now();
|
|
77
|
+
let res;
|
|
78
|
+
try {
|
|
79
|
+
res = await fetch(`${host}/api/embed`, {
|
|
80
|
+
method: 'POST',
|
|
81
|
+
headers: { 'content-type': 'application/json' },
|
|
82
|
+
body: JSON.stringify({ model, input: texts.map((t) => truncateUtf8(t)) }),
|
|
83
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
const outcome = err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError') ? 'timeout' : 'network';
|
|
88
|
+
return { ok: false, outcome, message: `${outcome}: ${err instanceof Error ? err.message : String(err)}`, host, model, latencyMs: Date.now() - started };
|
|
89
|
+
}
|
|
90
|
+
if (!res.ok) {
|
|
91
|
+
const text = (await res.text().catch(() => '')).slice(0, 300);
|
|
92
|
+
// ollama answers 404 {"error":"model \"…\" not found, try pulling it first"}
|
|
93
|
+
const outcome = res.status === 404 ? 'model_missing' : res.status >= 500 ? 'http_5xx' : 'http_4xx';
|
|
94
|
+
return { ok: false, outcome, message: `ollama responded ${res.status}: ${text}`, host, model, latencyMs: Date.now() - started };
|
|
95
|
+
}
|
|
96
|
+
let json;
|
|
97
|
+
try {
|
|
98
|
+
json = await res.json();
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
return { ok: false, outcome: 'bad_json', message: `non-JSON body: ${err instanceof Error ? err.message : String(err)}`, host, model, latencyMs: Date.now() - started };
|
|
102
|
+
}
|
|
103
|
+
const vectors = isPlainObject(json) ? json.embeddings : undefined;
|
|
104
|
+
if (!Array.isArray(vectors) ||
|
|
105
|
+
vectors.length !== texts.length ||
|
|
106
|
+
!vectors.every((v) => Array.isArray(v) && v.length > 0 && v.every((n) => typeof n === 'number' && Number.isFinite(n))) ||
|
|
107
|
+
new Set(vectors.map((v) => v.length)).size !== 1) {
|
|
108
|
+
return { ok: false, outcome: 'bad_shape', message: 'embeddings must be a rectangular [texts.length][dim] array of finite numbers.', host, model, latencyMs: Date.now() - started };
|
|
109
|
+
}
|
|
110
|
+
return { ok: true, vectors: vectors, model, dim: vectors[0].length, latencyMs: Date.now() - started };
|
|
111
|
+
}
|
|
112
|
+
/** Batch-embed texts (index build/rebuild path). Vectors align 1:1 with the
|
|
113
|
+
* input order; each text is truncated to MAX_TEXT_BYTES first. */
|
|
114
|
+
export async function embedTexts(texts, config = {}, opts = {}) {
|
|
115
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_BATCH_TIMEOUT_MS;
|
|
116
|
+
return embedRequest(texts, config, timeoutMs);
|
|
117
|
+
}
|
|
118
|
+
/** Embed one query (reranker hot path, 2s budget). Empty/whitespace queries
|
|
119
|
+
* fail locally with 'empty_input' — nothing leaves the process. */
|
|
120
|
+
export async function embedQuery(query, config = {}, opts = {}) {
|
|
121
|
+
if ((query ?? '').trim() === '') {
|
|
122
|
+
return { ok: false, outcome: 'empty_input', message: 'empty query.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
123
|
+
}
|
|
124
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_QUERY_TIMEOUT_MS;
|
|
125
|
+
const r = await embedRequest([query], config, timeoutMs);
|
|
126
|
+
if (!r.ok)
|
|
127
|
+
return r;
|
|
128
|
+
return { ok: true, vector: r.vectors[0], model: r.model, dim: r.dim, latencyMs: r.latencyMs };
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Probe: is the server there, is the model pulled? One micro embed call —
|
|
132
|
+
* unlike a /api/tags listing it validates the FULL path (server + model +
|
|
133
|
+
* response shape) and yields the vector dim for index validation, at the
|
|
134
|
+
* same 15-30ms cost (A1 §5). Structured result; never throws.
|
|
135
|
+
*/
|
|
136
|
+
export async function probeEmbedding(config = {}, opts = {}) {
|
|
137
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_QUERY_TIMEOUT_MS;
|
|
138
|
+
const r = await embedRequest(['dsh-topics-memory semantic layer probe'], config, timeoutMs);
|
|
139
|
+
if (!r.ok)
|
|
140
|
+
return r;
|
|
141
|
+
return { ok: true, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), dim: r.dim, latencyMs: r.latencyMs };
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Build an embedder handle for retrieval's `ensureIndexFresh` — a callable
|
|
145
|
+
* embedTexts bound to this config, carrying the RESOLVED model identity the
|
|
146
|
+
* index must match. ensureIndexFresh reads `.model` only; the actual
|
|
147
|
+
* rebuild stays with the caller.
|
|
148
|
+
*/
|
|
149
|
+
export function embedderWith(config) {
|
|
150
|
+
const fn = (texts) => embedTexts(texts, config);
|
|
151
|
+
return Object.assign(fn, { model: resolveEmbedModel(config.model) });
|
|
152
|
+
}
|
|
153
|
+
// --- Sync query embed (Phase-2 wiring) --------------------------------------
|
|
154
|
+
// The spliced hot path is synchronous (the chancelu lesson): retrieveSync
|
|
155
|
+
// cannot await. A local ollama query is 15-30ms steady state (A1 §5), so a
|
|
156
|
+
// bounded SYNC subprocess call keeps the reranker on the hot lane inside the
|
|
157
|
+
// ADR 0017 local exception — hard-capped so a wedged ollama loses the race,
|
|
158
|
+
// never the injection. curl ships with the host OS; node has no sync HTTP.
|
|
159
|
+
/** Hard sync budget for the hot-lane query embed. Generous over the measured
|
|
160
|
+
* 15-30ms, merciless for a wedged server. */
|
|
161
|
+
export const EMBED_QUERY_SYNC_TIMEOUT_MS = 250;
|
|
162
|
+
export function embedQuerySync(query, config = {}) {
|
|
163
|
+
const provider = (config.provider ?? '').trim();
|
|
164
|
+
if (provider === '') {
|
|
165
|
+
return { ok: false, outcome: 'provider_off', message: "embeddingProvider is '' — the semantic layer is off.", host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
166
|
+
}
|
|
167
|
+
if (provider !== SUPPORTED_PROVIDER) {
|
|
168
|
+
return { ok: false, outcome: 'unsupported_provider', message: `embeddingProvider '${provider}' is not supported in v1.`, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
169
|
+
}
|
|
170
|
+
if ((query ?? '').trim() === '') {
|
|
171
|
+
return { ok: false, outcome: 'empty_input', message: 'empty query.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 };
|
|
172
|
+
}
|
|
173
|
+
const host = resolveEmbedHost(config.host);
|
|
174
|
+
const model = resolveEmbedModel(config.model);
|
|
175
|
+
const started = Date.now();
|
|
176
|
+
let raw;
|
|
177
|
+
try {
|
|
178
|
+
raw = execFileSync('curl', [
|
|
179
|
+
'-sS',
|
|
180
|
+
'--max-time', String(Math.ceil(EMBED_QUERY_SYNC_TIMEOUT_MS / 1000)),
|
|
181
|
+
'-X', 'POST', `${host}/api/embed`,
|
|
182
|
+
'-H', 'content-type: application/json',
|
|
183
|
+
'-d', JSON.stringify({ model, input: [truncateUtf8(query)] }),
|
|
184
|
+
], { timeout: EMBED_QUERY_SYNC_TIMEOUT_MS, encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] });
|
|
185
|
+
}
|
|
186
|
+
catch (err) {
|
|
187
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
188
|
+
const outcome = /timed out|ETIMEDOUT|maxTime/i.test(message) ? 'timeout' : 'network';
|
|
189
|
+
return { ok: false, outcome, message: `${outcome}: ${message}`, host, model, latencyMs: Date.now() - started };
|
|
190
|
+
}
|
|
191
|
+
let json;
|
|
192
|
+
try {
|
|
193
|
+
json = JSON.parse(raw);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return { ok: false, outcome: 'bad_json', message: 'ollama returned non-JSON (sync path).', host, model, latencyMs: Date.now() - started };
|
|
197
|
+
}
|
|
198
|
+
const embeddings = isPlainObject(json) ? json.embeddings : undefined;
|
|
199
|
+
const first = Array.isArray(embeddings) && embeddings.length > 0 ? embeddings[0] : undefined;
|
|
200
|
+
if (!Array.isArray(first)) {
|
|
201
|
+
return { ok: false, outcome: 'bad_shape', message: 'ollama response has no embeddings[0] array (sync path).', host, model, latencyMs: Date.now() - started };
|
|
202
|
+
}
|
|
203
|
+
const vector = first.filter((n) => typeof n === 'number');
|
|
204
|
+
if (vector.length === 0 || vector.length !== first.length) {
|
|
205
|
+
return { ok: false, outcome: 'bad_shape', message: 'embeddings[0] malformed (sync path).', host, model, latencyMs: Date.now() - started };
|
|
206
|
+
}
|
|
207
|
+
return { ok: true, vector, model, dim: vector.length, latencyMs: Date.now() - started };
|
|
208
|
+
}
|
package/lib/ilog.d.ts
CHANGED
|
@@ -23,6 +23,13 @@ export interface SlowItem {
|
|
|
23
23
|
slug: string;
|
|
24
24
|
why: string;
|
|
25
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Slow-lane full-lifecycle outcome (A4 决议 3, spec §3.2): where a slow-lane
|
|
28
|
+
* run (or round with a slow-lane stake) ended. Precedent = decisions.jsonl's
|
|
29
|
+
* outcome enum — a closed vocabulary so 「没跑」(no-run/no-claim) and 「跑了
|
|
30
|
+
* 全败」(llm-error/timeout) are finally distinguishable in the log.
|
|
31
|
+
*/
|
|
32
|
+
export type SlowOutcome = 'no-run' | 'no-claim' | 'no-candidates' | 'llm-error' | 'timeout' | 'delivered' | 'expired';
|
|
26
33
|
export interface InjectionRecord {
|
|
27
34
|
at: string;
|
|
28
35
|
sessionId?: string;
|
|
@@ -69,6 +76,29 @@ export interface InjectionRecord {
|
|
|
69
76
|
slowMs?: number;
|
|
70
77
|
/** Slow-lane pointers this round, in delivery order. */
|
|
71
78
|
slow?: SlowItem[];
|
|
79
|
+
/**
|
|
80
|
+
* Slow-lane disposition for this round (A4 决议 3, spec §8.2): `delivered`
|
|
81
|
+
* when a pending was consumed and packed, `expired` at its hard bounds, and
|
|
82
|
+
* the producing run's failure point (no-run/no-claim/no-candidates/
|
|
83
|
+
* llm-error/timeout) when a spliced arrived with no pending. Optional —
|
|
84
|
+
* older lines predate the field and pure fast-lane rounds may omit it.
|
|
85
|
+
* Written once with the record, never backfilled (ilog append-only).
|
|
86
|
+
*/
|
|
87
|
+
slowOutcome?: SlowOutcome;
|
|
88
|
+
/**
|
|
89
|
+
* Lesson gate fields (B2, spec 卷一 §6.3/§7) — present only on rounds where
|
|
90
|
+
* lessonInject is shadow|on AND at least one established lesson matched the
|
|
91
|
+
* query. `lessons` is the trigger-channel match list with whether the entry
|
|
92
|
+
* entered the context; shadow rounds always carry included:false there (the
|
|
93
|
+
* double-compute lands in meta/lesson-shadow.jsonl instead).
|
|
94
|
+
*/
|
|
95
|
+
lessonInject?: 'shadow' | 'on';
|
|
96
|
+
lessons?: {
|
|
97
|
+
slug: string;
|
|
98
|
+
score: number;
|
|
99
|
+
source: string;
|
|
100
|
+
included: boolean;
|
|
101
|
+
}[];
|
|
72
102
|
}
|
|
73
103
|
/**
|
|
74
104
|
* One topic_open event (pointer-open log, v4 §4.3). Everything beyond
|
package/lib/index.d.ts
CHANGED
|
@@ -121,6 +121,10 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
121
121
|
jevSecretFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
122
122
|
jevLayaFallback: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
123
123
|
jevLayaUrl: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
124
|
+
embeddingProvider: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
125
|
+
embeddingModel: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
126
|
+
packageMode: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
127
|
+
lessonInject: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
124
128
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
125
129
|
repo: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
126
130
|
autoInject: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
@@ -156,6 +160,10 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
156
160
|
jevSecretFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
157
161
|
jevLayaFallback: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
158
162
|
jevLayaUrl: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
163
|
+
embeddingProvider: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
164
|
+
embeddingModel: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
165
|
+
packageMode: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
166
|
+
lessonInject: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
159
167
|
}>>, "plain">;
|
|
160
168
|
/** Live config reference as a 0.1.7 host passes it (cron / mcp-adapter
|
|
161
169
|
* spelling): `.get()` snapshots the current value — the host swaps the
|
|
@@ -168,6 +176,13 @@ interface VolatileRef<T> {
|
|
|
168
176
|
export type TopicsRuntimeConfig = Partial<{
|
|
169
177
|
[K in keyof TopicsConfigValue]: VolatileRef<TopicsConfigValue[K]> | TopicsConfigValue[K];
|
|
170
178
|
}>;
|
|
179
|
+
/**
|
|
180
|
+
* B-line dual-write seam contract (卷二 §6.2) — shared types are exported by
|
|
181
|
+
* THIS repo (C1 决议 3: no shared lib until a third consumer appears). The
|
|
182
|
+
* new repo calls TopicsService.feedObservations / setLessonStatus through the
|
|
183
|
+
* host process; direct bundle writes are forbidden on its side.
|
|
184
|
+
*/
|
|
185
|
+
export type { StructuredObservation, FeedResult, StoreResult } from './service.ts';
|
|
171
186
|
/**
|
|
172
187
|
* Strip a leading YAML frontmatter block (`---` / body / `---`) from a skill
|
|
173
188
|
* markdown file. `SkillDefinition.content` must be the instruction body after
|
|
@@ -180,4 +195,3 @@ export type TopicsRuntimeConfig = Partial<{
|
|
|
180
195
|
*/
|
|
181
196
|
export declare function stripFrontmatter(raw: string): string;
|
|
182
197
|
export declare function apply(ctx: Context, config?: TopicsRuntimeConfig): Promise<void>;
|
|
183
|
-
export {};
|
package/lib/index.js
CHANGED
|
@@ -359,8 +359,9 @@ export async function apply(ctx, config = {}) {
|
|
|
359
359
|
if (cfg.autoInject) {
|
|
360
360
|
const consumed = slowLane.consume(sessionId, observer.turnCountOf(sessionId));
|
|
361
361
|
if (consumed !== undefined) {
|
|
362
|
-
if ('pending' in consumed)
|
|
363
|
-
slow = consumed.pending;
|
|
362
|
+
if ('pending' in consumed) {
|
|
363
|
+
slow = { ...consumed.pending, outcome: consumed.pending.outcome };
|
|
364
|
+
}
|
|
364
365
|
else
|
|
365
366
|
slowExpired = consumed.expired;
|
|
366
367
|
}
|
|
@@ -578,7 +579,10 @@ export async function apply(ctx, config = {}) {
|
|
|
578
579
|
// and its own settle hook does the release.
|
|
579
580
|
const run = distiller.request(sessionId, reason);
|
|
580
581
|
if (run !== undefined) {
|
|
581
|
-
void run
|
|
582
|
+
void run
|
|
583
|
+
.finally(() => releaseSessionLlm(sessionId))
|
|
584
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
585
|
+
.catch(() => undefined);
|
|
582
586
|
}
|
|
583
587
|
});
|
|
584
588
|
// ---- Session events: injection + observation ----
|
|
@@ -786,8 +790,11 @@ export async function apply(ctx, config = {}) {
|
|
|
786
790
|
// no-adapter detail instead
|
|
787
791
|
}
|
|
788
792
|
const run = distiller.request(sessionId, 'boot-replay');
|
|
789
|
-
if (run !== undefined)
|
|
790
|
-
void run
|
|
793
|
+
if (run !== undefined) {
|
|
794
|
+
void run
|
|
795
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
796
|
+
.catch(() => undefined);
|
|
797
|
+
}
|
|
791
798
|
})
|
|
792
799
|
.catch(() => undefined);
|
|
793
800
|
// Consolidation cadence check — after the pull (freshest pool),
|
|
@@ -823,6 +830,9 @@ export async function apply(ctx, config = {}) {
|
|
|
823
830
|
})
|
|
824
831
|
.catch(() => undefined);
|
|
825
832
|
}
|
|
833
|
+
// Embedding index maintenance (A4/A6, spec §3): boot-time freshness
|
|
834
|
+
// check + incremental rebuild — provider '' / packageMode off = no-op.
|
|
835
|
+
void service.refreshEmbeddingIndex().catch(() => undefined);
|
|
826
836
|
})
|
|
827
837
|
.catch((error) => {
|
|
828
838
|
warn(`dsh-topics-memory 启动链失败(不影响会话启动):${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -948,7 +958,11 @@ export async function apply(ctx, config = {}) {
|
|
|
948
958
|
? { ok: false, reason: 'in-flight', created: [], updated: [], marked: 0 }
|
|
949
959
|
: { ok: false, reason: 'no-model', created: [], updated: [], marked: 0 };
|
|
950
960
|
}
|
|
951
|
-
|
|
961
|
+
void run
|
|
962
|
+
.finally(() => releaseSessionLlm(sessionId))
|
|
963
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
964
|
+
.catch(() => undefined);
|
|
965
|
+
return run;
|
|
952
966
|
};
|
|
953
967
|
// Manual /topics consolidate trigger: same lane as the cadence path, same
|
|
954
968
|
// trigger-time llm capture and in-flight guard (the command runs inside a
|
|
@@ -1018,6 +1032,10 @@ const DEFAULTS = {
|
|
|
1018
1032
|
distillMaxModelCalls: 8,
|
|
1019
1033
|
consolidateCadence: 'daily',
|
|
1020
1034
|
deprecatedTtlDays: 15,
|
|
1035
|
+
embeddingProvider: '',
|
|
1036
|
+
embeddingModel: '',
|
|
1037
|
+
packageMode: 'off',
|
|
1038
|
+
lessonInject: 'off',
|
|
1021
1039
|
usageBoost: 0.15,
|
|
1022
1040
|
pushDebounceSeconds: 45,
|
|
1023
1041
|
};
|
package/lib/okf.d.ts
CHANGED
|
@@ -13,6 +13,32 @@
|
|
|
13
13
|
* @module okf
|
|
14
14
|
*/
|
|
15
15
|
export type TopicStatus = 'draft' | 'stable' | 'deprecated';
|
|
16
|
+
/**
|
|
17
|
+
* Bundle entry kinds (B2, spec 2026-10-self-improve-topics §6.1): `topic` =
|
|
18
|
+
* factual knowledge (「是什么」), `lesson` = behavioral lesson (「怎么做事」).
|
|
19
|
+
* The kind rides the OKF `type` frontmatter field — the canonical serialized
|
|
20
|
+
* value for a lesson is `Lesson` (matching the `Topic` capitalization
|
|
21
|
+
* convention); recognition is case-insensitive so bundles written by the
|
|
22
|
+
* new repo's loop parse regardless of casing drift.
|
|
23
|
+
*/
|
|
24
|
+
export type TopicKind = 'topic' | 'lesson';
|
|
25
|
+
/** lesson scope (B2 MJ-2, spec §6.2): workspace lessons inject only in the
|
|
26
|
+
* workspace that produced them (cwd-hash match); global need no filter. */
|
|
27
|
+
export type LessonScope = 'workspace' | 'global';
|
|
28
|
+
/** lesson confidence (B2, spec §6.2): the FACT source — `status` derives from
|
|
29
|
+
* it (hint↔draft, established↔stable); hint never injects (MJ-3). */
|
|
30
|
+
export type LessonConfidence = 'hint' | 'established';
|
|
31
|
+
/** lesson provenance (B2 BL-2, spec §6.2): machine-recognizable source tag;
|
|
32
|
+
* uninstall reclamation keys on source≠manual (卷二 §9). */
|
|
33
|
+
export type LessonSource = 'distilled' | 'correction' | 'manual';
|
|
34
|
+
/** kind from the raw `type` frontmatter value (case-insensitive `lesson`). */
|
|
35
|
+
export declare function topicKindOf(type: string): TopicKind;
|
|
36
|
+
/** The canonical serialized `type` value per kind. */
|
|
37
|
+
export declare function kindTypeValue(kind: TopicKind): string;
|
|
38
|
+
/** confidence 为事实源(B2 Minor):hint↔draft、established↔stable.
|
|
39
|
+
* `deprecated` is a lifecycle endpoint OUTSIDE the confidence axis — callers
|
|
40
|
+
* set it directly (setLessonStatus / consolidation / TTL sweep). */
|
|
41
|
+
export declare function deriveLessonStatus(confidence: LessonConfidence): TopicStatus;
|
|
16
42
|
export interface SourceEntry {
|
|
17
43
|
id?: string;
|
|
18
44
|
resource: string;
|
|
@@ -46,6 +72,14 @@ export interface TopicFrontmatter {
|
|
|
46
72
|
verified?: TrustStamp;
|
|
47
73
|
stale_after?: string;
|
|
48
74
|
sources?: SourceEntry[];
|
|
75
|
+
/** lesson 必带 scope(默认 workspace;global 需显式)。 */
|
|
76
|
+
scope?: LessonScope;
|
|
77
|
+
/** confidence 事实源;status 派生(deriveLessonStatus)。默认 hint。 */
|
|
78
|
+
confidence?: LessonConfidence;
|
|
79
|
+
/** provenance 标记(卸载回收按 source≠manual)。默认 manual。 */
|
|
80
|
+
source?: LessonSource;
|
|
81
|
+
/** 写入时的 workspace 标识(cwd hash,MJ-2);scope=global 时可空。 */
|
|
82
|
+
workspaceId?: string;
|
|
49
83
|
/** Producer-defined extras — preserved on round-trip per OKF §4.1. */
|
|
50
84
|
[extra: string]: unknown;
|
|
51
85
|
}
|
package/lib/okf.js
CHANGED
|
@@ -13,6 +13,20 @@
|
|
|
13
13
|
* @module okf
|
|
14
14
|
*/
|
|
15
15
|
import { parseYaml, stringifyYaml, YamlError } from "./yaml.js";
|
|
16
|
+
/** kind from the raw `type` frontmatter value (case-insensitive `lesson`). */
|
|
17
|
+
export function topicKindOf(type) {
|
|
18
|
+
return typeof type === 'string' && type.trim().toLowerCase() === 'lesson' ? 'lesson' : 'topic';
|
|
19
|
+
}
|
|
20
|
+
/** The canonical serialized `type` value per kind. */
|
|
21
|
+
export function kindTypeValue(kind) {
|
|
22
|
+
return kind === 'lesson' ? 'Lesson' : 'Topic';
|
|
23
|
+
}
|
|
24
|
+
/** confidence 为事实源(B2 Minor):hint↔draft、established↔stable.
|
|
25
|
+
* `deprecated` is a lifecycle endpoint OUTSIDE the confidence axis — callers
|
|
26
|
+
* set it directly (setLessonStatus / consolidation / TTL sweep). */
|
|
27
|
+
export function deriveLessonStatus(confidence) {
|
|
28
|
+
return confidence === 'established' ? 'stable' : 'draft';
|
|
29
|
+
}
|
|
16
30
|
export class OkfError extends Error {
|
|
17
31
|
name = 'OkfError';
|
|
18
32
|
}
|
|
@@ -106,8 +120,26 @@ export function parseTopicDoc(raw) {
|
|
|
106
120
|
return s;
|
|
107
121
|
});
|
|
108
122
|
}
|
|
123
|
+
// ---- lesson-kind fields (B2, spec §6.2) — parsed on lesson docs only ----
|
|
124
|
+
// Invalid enum values DEGRADE TO THE DEFAULT (triggers precedent: a producer
|
|
125
|
+
// slip must never invalidate an otherwise sound doc); the seam writers own
|
|
126
|
+
// the closed vocabulary. On lesson docs these four keys join the known set
|
|
127
|
+
// so the extras loop cannot double-store a raw unvalidated value.
|
|
128
|
+
if (topicKindOf(fmRaw.type) === 'lesson') {
|
|
129
|
+
fm.scope = fmRaw.scope === 'global' ? 'global' : 'workspace';
|
|
130
|
+
fm.confidence = fmRaw.confidence === 'established' ? 'established' : 'hint';
|
|
131
|
+
fm.source = fmRaw.source === 'distilled' || fmRaw.source === 'correction' ? fmRaw.source : 'manual';
|
|
132
|
+
if (typeof fmRaw.workspaceId === 'string' && fmRaw.workspaceId !== '')
|
|
133
|
+
fm.workspaceId = fmRaw.workspaceId;
|
|
134
|
+
}
|
|
109
135
|
// Preserve unknown producer-defined keys verbatim (OKF §4.1 extensions).
|
|
110
136
|
const known = new Set(['type', 'title', 'description', 'tags', 'triggers', 'depends', 'open_questions', 'impact', 'status', 'generated', 'verified', 'stale_after', 'sources']);
|
|
137
|
+
if (fm.scope !== undefined || fm.confidence !== undefined || fm.source !== undefined) {
|
|
138
|
+
known.add('scope');
|
|
139
|
+
known.add('confidence');
|
|
140
|
+
known.add('source');
|
|
141
|
+
known.add('workspaceId');
|
|
142
|
+
}
|
|
111
143
|
for (const [k, v] of Object.entries(fmRaw)) {
|
|
112
144
|
if (!known.has(k))
|
|
113
145
|
fm[k] = v;
|
|
@@ -137,6 +169,15 @@ export function serializeTopicDoc(doc) {
|
|
|
137
169
|
out.stale_after = fm.stale_after;
|
|
138
170
|
if (fm.sources !== undefined && fm.sources.length > 0)
|
|
139
171
|
out.sources = fm.sources;
|
|
172
|
+
// lesson fields serialize in a fixed slot (deterministic order, spec §6.2).
|
|
173
|
+
if (fm.scope !== undefined)
|
|
174
|
+
out.scope = fm.scope;
|
|
175
|
+
if (fm.confidence !== undefined)
|
|
176
|
+
out.confidence = fm.confidence;
|
|
177
|
+
if (fm.source !== undefined)
|
|
178
|
+
out.source = fm.source;
|
|
179
|
+
if (fm.workspaceId !== undefined)
|
|
180
|
+
out.workspaceId = fm.workspaceId;
|
|
140
181
|
for (const [k, v] of Object.entries(fm)) {
|
|
141
182
|
if (!(k in out))
|
|
142
183
|
out[k] = v;
|
package/lib/paths.d.ts
CHANGED
|
@@ -45,4 +45,13 @@ export declare function conflictsFile(root: string): string;
|
|
|
45
45
|
export declare function hostId(): string;
|
|
46
46
|
/** The actor string stamped into `generated.by` for machine-written topics. */
|
|
47
47
|
export declare function actorFor(): string;
|
|
48
|
+
/**
|
|
49
|
+
* Current workspace identifier for lesson scope filtering (B2 MJ-2, spec
|
|
50
|
+
* 卷一 §6.2): the sha256 prefix of the process cwd, overridable via
|
|
51
|
+
* $DSH_TOPICS_WORKSPACE_ID (tests / multi-root hosts; the override also lets
|
|
52
|
+
* a host pin one identity across chdirs). 拍板(spec 未钉死来源): cwd hash at
|
|
53
|
+
* read time is the minimal viable source — the same derivation the lesson
|
|
54
|
+
* write path stamps into frontmatter.workspaceId, so no config plumbing.
|
|
55
|
+
*/
|
|
56
|
+
export declare function currentWorkspaceId(): string;
|
|
48
57
|
export {};
|
package/lib/paths.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* @module paths
|
|
13
13
|
*/
|
|
14
14
|
import { existsSync, renameSync } from 'node:fs';
|
|
15
|
+
import { createHash } from 'node:crypto';
|
|
15
16
|
import { homedir } from 'node:os';
|
|
16
17
|
import { hostname as osHostname } from 'node:os';
|
|
17
18
|
import { join } from 'node:path';
|
|
@@ -82,3 +83,17 @@ export function hostId() {
|
|
|
82
83
|
export function actorFor() {
|
|
83
84
|
return `agent:dsh-topics-memory@${hostId()}`;
|
|
84
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* Current workspace identifier for lesson scope filtering (B2 MJ-2, spec
|
|
88
|
+
* 卷一 §6.2): the sha256 prefix of the process cwd, overridable via
|
|
89
|
+
* $DSH_TOPICS_WORKSPACE_ID (tests / multi-root hosts; the override also lets
|
|
90
|
+
* a host pin one identity across chdirs). 拍板(spec 未钉死来源): cwd hash at
|
|
91
|
+
* read time is the minimal viable source — the same derivation the lesson
|
|
92
|
+
* write path stamps into frontmatter.workspaceId, so no config plumbing.
|
|
93
|
+
*/
|
|
94
|
+
export function currentWorkspaceId() {
|
|
95
|
+
const env = process.env.DSH_TOPICS_WORKSPACE_ID?.trim();
|
|
96
|
+
if (env !== undefined && env !== '')
|
|
97
|
+
return env;
|
|
98
|
+
return createHash('sha256').update(process.cwd()).digest('hex').slice(0, 16);
|
|
99
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* meta/pipeline-events.jsonl — the pipeline-level anomaly ledger (spec §8.1,
|
|
3
|
+
* A2/A6 BL-1 revised: the "记 wtf 账" mechanism never existed, this IS the
|
|
4
|
+
* explicit ledger). One line JSON per event; always on, no config key
|
|
5
|
+
* (恒开无键 — observability is not a gray-release feature).
|
|
6
|
+
*
|
|
7
|
+
* Discipline = decisions.jsonl (src/jev/log.ts) verbatim: field-whitelist
|
|
8
|
+
* copy on write (extra caller fields cannot leak in, detail values must be
|
|
9
|
+
* scalar), 512KB auto-compaction keeping the most recent quarter, and write
|
|
10
|
+
* failures swallowed silently — a broken sink must never break the pipeline.
|
|
11
|
+
* Unlike jev/log this writer takes the bundle root explicitly (callers like
|
|
12
|
+
* the slow lane already hold it; tests point it at a tmp dir).
|
|
13
|
+
*
|
|
14
|
+
* @module pipeline-events
|
|
15
|
+
*/
|
|
16
|
+
/** Closed vocabulary, spec §8.1 + W-C wave start set. `slow-lane-outcome` is
|
|
17
|
+
* the slow lane's per-run disposition row (quality.ts); the distill/link
|
|
18
|
+
* kinds are the A2/A6 producers' contract — they import this module. The
|
|
19
|
+
* two `seam-*` kinds are the B-line dual-write audit trail (卷二 §6.2:
|
|
20
|
+
* caller-identity audit — opts.actor — rides every seam call). */
|
|
21
|
+
export type PipelineEventKind = 'slow-lane-outcome' | 'distill-op-invalid' | 'distill-op-retry-exhausted' | 'link-ref-rejected' | 'seam-observations-fed' | 'seam-lesson-status';
|
|
22
|
+
/** File NAME inside the bundle's meta dir (path via pipelineEventsFile). */
|
|
23
|
+
export declare const PIPELINE_EVENTS_FILE = "pipeline-events.jsonl";
|
|
24
|
+
export interface PipelineEvent {
|
|
25
|
+
at: string;
|
|
26
|
+
kind: PipelineEventKind;
|
|
27
|
+
/** Bounded scalar metadata only — never conversation or conclusion text. */
|
|
28
|
+
detail?: Record<string, string | number | boolean>;
|
|
29
|
+
}
|
|
30
|
+
export declare function pipelineEventsFile(bundleDir: string): string;
|
|
31
|
+
/**
|
|
32
|
+
* Copy one event through the field whitelist onto the ledger. Only `at`,
|
|
33
|
+
* `kind` and a scalar-valued `detail` map can pass; non-scalar detail values
|
|
34
|
+
* (objects/arrays/null/NaN), surplus entries and over-long keys/values are
|
|
35
|
+
* dropped here so no unbounded content can ride in. Write failures are
|
|
36
|
+
* swallowed downstream — this never rejects and never throws.
|
|
37
|
+
*/
|
|
38
|
+
export declare function recordPipelineEvent(bundleDir: string, event: PipelineEvent): Promise<void>;
|