@aiwayds/dsh-topics-memory 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +55 -1
- package/lib/config.js +42 -0
- package/lib/digest.d.ts +111 -5
- package/lib/digest.js +437 -11
- package/lib/distill.d.ts +39 -0
- package/lib/distill.js +250 -15
- package/lib/embedding.d.ts +113 -0
- package/lib/embedding.js +208 -0
- package/lib/ilog.d.ts +30 -0
- package/lib/index.d.ts +15 -1
- package/lib/index.js +24 -6
- package/lib/okf.d.ts +34 -0
- package/lib/okf.js +41 -0
- package/lib/paths.d.ts +9 -0
- package/lib/paths.js +15 -0
- package/lib/pipeline-events.d.ts +38 -0
- package/lib/pipeline-events.js +123 -0
- package/lib/quality.d.ts +47 -7
- package/lib/quality.js +124 -20
- package/lib/retrieval.d.ts +106 -0
- package/lib/retrieval.js +280 -0
- package/lib/service.d.ts +103 -2
- package/lib/service.js +623 -16
- package/lib/store.d.ts +39 -3
- package/lib/store.js +60 -2
- package/package.json +2 -2
- package/src/config.ts +49 -0
- package/src/digest.ts +497 -8
- package/src/distill.ts +275 -14
- package/src/embedding.ts +294 -0
- package/src/ilog.ts +26 -0
- package/src/index.ts +32 -5
- package/src/okf.ts +68 -0
- package/src/paths.ts +15 -0
- package/src/pipeline-events.ts +136 -0
- package/src/quality.ts +145 -21
- package/src/retrieval.ts +335 -0
- package/src/service.ts +661 -19
- package/src/store.ts +77 -5
package/src/embedding.ts
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local embedding client (ollama) — the semantic layer behind the nearFloor
|
|
3
|
+
* reranker (A4 决议 2, spec §3.1).
|
|
4
|
+
*
|
|
5
|
+
* Semantics (mirror of jev/client's outcome mode):
|
|
6
|
+
* - every failure returns as a structured {ok:false, outcome} result, NEVER
|
|
7
|
+
* a throw — the caller degrades explicitly to the pure-lexical path;
|
|
8
|
+
* - one AbortSignal.timeout per request, NO retry (query 2s / batch 30s);
|
|
9
|
+
* - localhost only in v1 (ADR 0017 local exception). Remote providers do
|
|
10
|
+
* not exist here by design: a remote embedding may only ever enter a
|
|
11
|
+
* slow-lane/offline path, never the fast lane (red line ①).
|
|
12
|
+
*
|
|
13
|
+
* @module embedding
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { execFileSync } from 'node:child_process'
|
|
17
|
+
|
|
18
|
+
/** Default ollama bind (the plugin's own default; OLLAMA_HOST overrides). */
|
|
19
|
+
export const DEFAULT_EMBED_HOST = 'http://127.0.0.1:11434'
|
|
20
|
+
|
|
21
|
+
/** Provider default model when embeddingModel is the '' sentinel
|
|
22
|
+
* (spec §7: ollama → nomic-embed-text, 768 dims, A1 §5 measured). */
|
|
23
|
+
export const DEFAULT_EMBED_MODEL = 'nomic-embed-text'
|
|
24
|
+
|
|
25
|
+
/** Query-side timeout — the reranker sits in the spliced hot path; a slow
|
|
26
|
+
* ollama must lose the race, not the injection. */
|
|
27
|
+
export const EMBED_QUERY_TIMEOUT_MS = 2000
|
|
28
|
+
/** Batch-side timeout (index rebuild: A1 measured ~9.5s for 137 topics). */
|
|
29
|
+
export const EMBED_BATCH_TIMEOUT_MS = 30_000
|
|
30
|
+
|
|
31
|
+
/** Per-text byte cap before anything leaves the process (query端 p50=153 tok
|
|
32
|
+
* ~ well under this; the cap only bites on paste pollution). */
|
|
33
|
+
export const MAX_TEXT_BYTES = 2048
|
|
34
|
+
|
|
35
|
+
/** The only provider v1 speaks. Anything else is an explicit
|
|
36
|
+
* unsupported_provider outcome, not a silent attempt. */
|
|
37
|
+
export const SUPPORTED_PROVIDER = 'ollama'
|
|
38
|
+
|
|
39
|
+
/** The config slice the embed functions need (pass straight from
|
|
40
|
+
* TopicsConfigValue). '' sentinels: provider off / provider default model. */
|
|
41
|
+
export interface EmbeddingConfig {
|
|
42
|
+
provider?: string
|
|
43
|
+
model?: string
|
|
44
|
+
/** Explicit host; absent → OLLAMA_HOST env → DEFAULT_EMBED_HOST. */
|
|
45
|
+
host?: string
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Resolve the ollama base URL: explicit arg wins, then OLLAMA_HOST, then
|
|
49
|
+
* the localhost default. Bare `host:port` values get the http:// prefix
|
|
50
|
+
* (ollama's own OLLAMA_HOST convention); trailing slashes are trimmed. */
|
|
51
|
+
export function resolveEmbedHost(explicit?: string): string {
|
|
52
|
+
const raw = (explicit ?? process.env.OLLAMA_HOST ?? '').trim()
|
|
53
|
+
const withScheme = raw === '' ? DEFAULT_EMBED_HOST : /^https?:\/\//.test(raw) ? raw : `http://${raw.replace(/^tcp:\/\//, '')}`
|
|
54
|
+
return withScheme.replace(/\/+$/, '')
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** '' sentinel → provider default model. */
|
|
58
|
+
export function resolveEmbedModel(model?: string): string {
|
|
59
|
+
const m = (model ?? '').trim()
|
|
60
|
+
return m === '' ? DEFAULT_EMBED_MODEL : m
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Truncate a string to ≤ maxBytes UTF-8 bytes WITHOUT splitting a multibyte
|
|
64
|
+
* character: cut at the byte cap, then back off over any continuation bytes
|
|
65
|
+
* (0b10xxxxxx) so the decoded tail is never a replacement char. */
|
|
66
|
+
export function truncateUtf8(text: string, maxBytes: number = MAX_TEXT_BYTES): string {
|
|
67
|
+
if (maxBytes <= 0) return ''
|
|
68
|
+
const buf = Buffer.from(text, 'utf8')
|
|
69
|
+
if (buf.length <= maxBytes) return text
|
|
70
|
+
let end = maxBytes
|
|
71
|
+
while (end > 0 && (buf[end]! & 0xc0) === 0x80) end -= 1
|
|
72
|
+
return buf.subarray(0, end).toString('utf8')
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
// Outcomes — the explicit-degradation vocabulary. Callers branch on `ok`;
|
|
77
|
+
// `outcome` is the human/queryable why (降级状态可查, spec §3.1).
|
|
78
|
+
// ---------------------------------------------------------------------------
|
|
79
|
+
|
|
80
|
+
export type EmbeddingFailureOutcome =
|
|
81
|
+
| 'provider_off' // config provider is the '' sentinel — semantic layer off
|
|
82
|
+
| 'unsupported_provider' // configured, but not a v1 provider (no remote!)
|
|
83
|
+
| 'empty_input' // nothing to embed — refused before any network activity
|
|
84
|
+
| 'network' // connect refused / DNS / socket — ollama not serving
|
|
85
|
+
| 'timeout' // the one AbortSignal fired
|
|
86
|
+
| 'model_missing' // ollama answered 404: model not pulled
|
|
87
|
+
| 'http_4xx'
|
|
88
|
+
| 'http_5xx'
|
|
89
|
+
| 'bad_json'
|
|
90
|
+
| 'bad_shape' // 200 but the embeddings array is malformed / dims disagree
|
|
91
|
+
|
|
92
|
+
export interface EmbedBatchOk {
|
|
93
|
+
ok: true
|
|
94
|
+
/** Vectors aligned 1:1 with the (truncated) input texts. */
|
|
95
|
+
vectors: number[][]
|
|
96
|
+
model: string
|
|
97
|
+
dim: number
|
|
98
|
+
latencyMs: number
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface EmbedQueryOk {
|
|
102
|
+
ok: true
|
|
103
|
+
vector: number[]
|
|
104
|
+
model: string
|
|
105
|
+
dim: number
|
|
106
|
+
latencyMs: number
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export interface EmbedFailure {
|
|
110
|
+
ok: false
|
|
111
|
+
outcome: EmbeddingFailureOutcome
|
|
112
|
+
message: string
|
|
113
|
+
host: string
|
|
114
|
+
model: string
|
|
115
|
+
latencyMs: number
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export type EmbedResult = EmbedBatchOk | EmbedFailure
|
|
119
|
+
export type EmbedQueryResult = EmbedQueryOk | EmbedFailure
|
|
120
|
+
export type ProbeResult = { ok: true; host: string; model: string; dim: number; latencyMs: number } | EmbedFailure
|
|
121
|
+
|
|
122
|
+
function isPlainObject(v: unknown): v is Record<string, unknown> {
|
|
123
|
+
return typeof v === 'object' && v !== null && !Array.isArray(v)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Shared request path: POST {host}/api/embed, outcome-mode failures only. */
|
|
127
|
+
async function embedRequest(
|
|
128
|
+
texts: readonly string[],
|
|
129
|
+
config: EmbeddingConfig,
|
|
130
|
+
timeoutMs: number,
|
|
131
|
+
): Promise<EmbedResult> {
|
|
132
|
+
const provider = (config.provider ?? '').trim()
|
|
133
|
+
if (provider === '') {
|
|
134
|
+
return { ok: false, outcome: 'provider_off', message: "embeddingProvider is '' — the semantic layer is off (pure lexical).", host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
|
|
135
|
+
}
|
|
136
|
+
if (provider !== SUPPORTED_PROVIDER) {
|
|
137
|
+
return { ok: false, outcome: 'unsupported_provider', message: `embeddingProvider '${provider}' is not supported in v1 (only '${SUPPORTED_PROVIDER}'); remote providers may not enter the fast lane.`, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
|
|
138
|
+
}
|
|
139
|
+
if (texts.length === 0) {
|
|
140
|
+
return { ok: false, outcome: 'empty_input', message: 'no texts to embed.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
|
|
141
|
+
}
|
|
142
|
+
const host = resolveEmbedHost(config.host)
|
|
143
|
+
const model = resolveEmbedModel(config.model)
|
|
144
|
+
const started = Date.now()
|
|
145
|
+
let res: Response
|
|
146
|
+
try {
|
|
147
|
+
res = await fetch(`${host}/api/embed`, {
|
|
148
|
+
method: 'POST',
|
|
149
|
+
headers: { 'content-type': 'application/json' },
|
|
150
|
+
body: JSON.stringify({ model, input: texts.map((t) => truncateUtf8(t)) }),
|
|
151
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
152
|
+
})
|
|
153
|
+
} catch (err) {
|
|
154
|
+
const outcome: EmbeddingFailureOutcome =
|
|
155
|
+
err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError') ? 'timeout' : 'network'
|
|
156
|
+
return { ok: false, outcome, message: `${outcome}: ${err instanceof Error ? err.message : String(err)}`, host, model, latencyMs: Date.now() - started }
|
|
157
|
+
}
|
|
158
|
+
if (!res.ok) {
|
|
159
|
+
const text = (await res.text().catch(() => '')).slice(0, 300)
|
|
160
|
+
// ollama answers 404 {"error":"model \"…\" not found, try pulling it first"}
|
|
161
|
+
const outcome: EmbeddingFailureOutcome = res.status === 404 ? 'model_missing' : res.status >= 500 ? 'http_5xx' : 'http_4xx'
|
|
162
|
+
return { ok: false, outcome, message: `ollama responded ${res.status}: ${text}`, host, model, latencyMs: Date.now() - started }
|
|
163
|
+
}
|
|
164
|
+
let json: unknown
|
|
165
|
+
try {
|
|
166
|
+
json = await res.json()
|
|
167
|
+
} catch (err) {
|
|
168
|
+
return { ok: false, outcome: 'bad_json', message: `non-JSON body: ${err instanceof Error ? err.message : String(err)}`, host, model, latencyMs: Date.now() - started }
|
|
169
|
+
}
|
|
170
|
+
const vectors = isPlainObject(json) ? json.embeddings : undefined
|
|
171
|
+
if (
|
|
172
|
+
!Array.isArray(vectors) ||
|
|
173
|
+
vectors.length !== texts.length ||
|
|
174
|
+
!vectors.every((v) => Array.isArray(v) && v.length > 0 && v.every((n) => typeof n === 'number' && Number.isFinite(n))) ||
|
|
175
|
+
new Set(vectors.map((v) => (v as number[]).length)).size !== 1
|
|
176
|
+
) {
|
|
177
|
+
return { ok: false, outcome: 'bad_shape', message: 'embeddings must be a rectangular [texts.length][dim] array of finite numbers.', host, model, latencyMs: Date.now() - started }
|
|
178
|
+
}
|
|
179
|
+
return { ok: true, vectors: vectors as number[][], model, dim: (vectors[0] as number[]).length, latencyMs: Date.now() - started }
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Batch-embed texts (index build/rebuild path). Vectors align 1:1 with the
|
|
183
|
+
* input order; each text is truncated to MAX_TEXT_BYTES first. */
|
|
184
|
+
export async function embedTexts(
|
|
185
|
+
texts: readonly string[],
|
|
186
|
+
config: EmbeddingConfig = {},
|
|
187
|
+
opts: { timeoutMs?: number } = {},
|
|
188
|
+
): Promise<EmbedResult> {
|
|
189
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_BATCH_TIMEOUT_MS
|
|
190
|
+
return embedRequest(texts, config, timeoutMs)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Embed one query (reranker hot path, 2s budget). Empty/whitespace queries
|
|
194
|
+
* fail locally with 'empty_input' — nothing leaves the process. */
|
|
195
|
+
export async function embedQuery(
|
|
196
|
+
query: string,
|
|
197
|
+
config: EmbeddingConfig = {},
|
|
198
|
+
opts: { timeoutMs?: number } = {},
|
|
199
|
+
): Promise<EmbedQueryResult> {
|
|
200
|
+
if ((query ?? '').trim() === '') {
|
|
201
|
+
return { ok: false, outcome: 'empty_input', message: 'empty query.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
|
|
202
|
+
}
|
|
203
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_QUERY_TIMEOUT_MS
|
|
204
|
+
const r = await embedRequest([query], config, timeoutMs)
|
|
205
|
+
if (!r.ok) return r
|
|
206
|
+
return { ok: true, vector: r.vectors[0]!, model: r.model, dim: r.dim, latencyMs: r.latencyMs }
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Probe: is the server there, is the model pulled? One micro embed call —
|
|
211
|
+
* unlike a /api/tags listing it validates the FULL path (server + model +
|
|
212
|
+
* response shape) and yields the vector dim for index validation, at the
|
|
213
|
+
* same 15-30ms cost (A1 §5). Structured result; never throws.
|
|
214
|
+
*/
|
|
215
|
+
export async function probeEmbedding(config: EmbeddingConfig = {}, opts: { timeoutMs?: number } = {}): Promise<ProbeResult> {
|
|
216
|
+
const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_QUERY_TIMEOUT_MS
|
|
217
|
+
const r = await embedRequest(['dsh-topics-memory semantic layer probe'], config, timeoutMs)
|
|
218
|
+
if (!r.ok) return r
|
|
219
|
+
return { ok: true, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), dim: r.dim, latencyMs: r.latencyMs }
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Build an embedder handle for retrieval's `ensureIndexFresh` — a callable
|
|
224
|
+
* embedTexts bound to this config, carrying the RESOLVED model identity the
|
|
225
|
+
* index must match. ensureIndexFresh reads `.model` only; the actual
|
|
226
|
+
* rebuild stays with the caller.
|
|
227
|
+
*/
|
|
228
|
+
export function embedderWith(
|
|
229
|
+
config: EmbeddingConfig,
|
|
230
|
+
): ((texts: readonly string[]) => Promise<EmbedResult>) & { model: string } {
|
|
231
|
+
const fn = (texts: readonly string[]) => embedTexts(texts, config)
|
|
232
|
+
return Object.assign(fn, { model: resolveEmbedModel(config.model) })
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// --- Sync query embed (Phase-2 wiring) --------------------------------------
|
|
236
|
+
// The spliced hot path is synchronous (the chancelu lesson): retrieveSync
|
|
237
|
+
// cannot await. A local ollama query is 15-30ms steady state (A1 §5), so a
|
|
238
|
+
// bounded SYNC subprocess call keeps the reranker on the hot lane inside the
|
|
239
|
+
// ADR 0017 local exception — hard-capped so a wedged ollama loses the race,
|
|
240
|
+
// never the injection. curl ships with the host OS; node has no sync HTTP.
|
|
241
|
+
|
|
242
|
+
/** Hard sync budget for the hot-lane query embed. Generous over the measured
|
|
243
|
+
* 15-30ms, merciless for a wedged server. */
|
|
244
|
+
export const EMBED_QUERY_SYNC_TIMEOUT_MS = 250
|
|
245
|
+
|
|
246
|
+
export function embedQuerySync(query: string, config: EmbeddingConfig = {}): EmbedQueryResult {
|
|
247
|
+
const provider = (config.provider ?? '').trim()
|
|
248
|
+
if (provider === '') {
|
|
249
|
+
return { ok: false, outcome: 'provider_off', message: "embeddingProvider is '' — the semantic layer is off.", host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
|
|
250
|
+
}
|
|
251
|
+
if (provider !== SUPPORTED_PROVIDER) {
|
|
252
|
+
return { ok: false, outcome: 'unsupported_provider', message: `embeddingProvider '${provider}' is not supported in v1.`, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
|
|
253
|
+
}
|
|
254
|
+
if ((query ?? '').trim() === '') {
|
|
255
|
+
return { ok: false, outcome: 'empty_input', message: 'empty query.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
|
|
256
|
+
}
|
|
257
|
+
const host = resolveEmbedHost(config.host)
|
|
258
|
+
const model = resolveEmbedModel(config.model)
|
|
259
|
+
const started = Date.now()
|
|
260
|
+
let raw: string
|
|
261
|
+
try {
|
|
262
|
+
raw = execFileSync(
|
|
263
|
+
'curl',
|
|
264
|
+
[
|
|
265
|
+
'-sS',
|
|
266
|
+
'--max-time', String(Math.ceil(EMBED_QUERY_SYNC_TIMEOUT_MS / 1000)),
|
|
267
|
+
'-X', 'POST', `${host}/api/embed`,
|
|
268
|
+
'-H', 'content-type: application/json',
|
|
269
|
+
'-d', JSON.stringify({ model, input: [truncateUtf8(query)] }),
|
|
270
|
+
],
|
|
271
|
+
{ timeout: EMBED_QUERY_SYNC_TIMEOUT_MS, encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] },
|
|
272
|
+
)
|
|
273
|
+
} catch (err) {
|
|
274
|
+
const message = err instanceof Error ? err.message : String(err)
|
|
275
|
+
const outcome: EmbeddingFailureOutcome = /timed out|ETIMEDOUT|maxTime/i.test(message) ? 'timeout' : 'network'
|
|
276
|
+
return { ok: false, outcome, message: `${outcome}: ${message}`, host, model, latencyMs: Date.now() - started }
|
|
277
|
+
}
|
|
278
|
+
let json: unknown
|
|
279
|
+
try {
|
|
280
|
+
json = JSON.parse(raw)
|
|
281
|
+
} catch {
|
|
282
|
+
return { ok: false, outcome: 'bad_json', message: 'ollama returned non-JSON (sync path).', host, model, latencyMs: Date.now() - started }
|
|
283
|
+
}
|
|
284
|
+
const embeddings = isPlainObject(json) ? (json as Record<string, unknown>).embeddings : undefined
|
|
285
|
+
const first = Array.isArray(embeddings) && embeddings.length > 0 ? embeddings[0] : undefined
|
|
286
|
+
if (!Array.isArray(first)) {
|
|
287
|
+
return { ok: false, outcome: 'bad_shape', message: 'ollama response has no embeddings[0] array (sync path).', host, model, latencyMs: Date.now() - started }
|
|
288
|
+
}
|
|
289
|
+
const vector = (first as unknown[]).filter((n): n is number => typeof n === 'number')
|
|
290
|
+
if (vector.length === 0 || vector.length !== (first as unknown[]).length) {
|
|
291
|
+
return { ok: false, outcome: 'bad_shape', message: 'embeddings[0] malformed (sync path).', host, model, latencyMs: Date.now() - started }
|
|
292
|
+
}
|
|
293
|
+
return { ok: true, vector, model, dim: vector.length, latencyMs: Date.now() - started }
|
|
294
|
+
}
|
package/src/ilog.ts
CHANGED
|
@@ -27,6 +27,14 @@ export interface SlowItem {
|
|
|
27
27
|
why: string
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Slow-lane full-lifecycle outcome (A4 决议 3, spec §3.2): where a slow-lane
|
|
32
|
+
* run (or round with a slow-lane stake) ended. Precedent = decisions.jsonl's
|
|
33
|
+
* outcome enum — a closed vocabulary so 「没跑」(no-run/no-claim) and 「跑了
|
|
34
|
+
* 全败」(llm-error/timeout) are finally distinguishable in the log.
|
|
35
|
+
*/
|
|
36
|
+
export type SlowOutcome = 'no-run' | 'no-claim' | 'no-candidates' | 'llm-error' | 'timeout' | 'delivered' | 'expired'
|
|
37
|
+
|
|
30
38
|
export interface InjectionRecord {
|
|
31
39
|
at: string
|
|
32
40
|
sessionId?: string
|
|
@@ -60,6 +68,24 @@ export interface InjectionRecord {
|
|
|
60
68
|
slowMs?: number
|
|
61
69
|
/** Slow-lane pointers this round, in delivery order. */
|
|
62
70
|
slow?: SlowItem[]
|
|
71
|
+
/**
|
|
72
|
+
* Slow-lane disposition for this round (A4 决议 3, spec §8.2): `delivered`
|
|
73
|
+
* when a pending was consumed and packed, `expired` at its hard bounds, and
|
|
74
|
+
* the producing run's failure point (no-run/no-claim/no-candidates/
|
|
75
|
+
* llm-error/timeout) when a spliced arrived with no pending. Optional —
|
|
76
|
+
* older lines predate the field and pure fast-lane rounds may omit it.
|
|
77
|
+
* Written once with the record, never backfilled (ilog append-only).
|
|
78
|
+
*/
|
|
79
|
+
slowOutcome?: SlowOutcome
|
|
80
|
+
/**
|
|
81
|
+
* Lesson gate fields (B2, spec 卷一 §6.3/§7) — present only on rounds where
|
|
82
|
+
* lessonInject is shadow|on AND at least one established lesson matched the
|
|
83
|
+
* query. `lessons` is the trigger-channel match list with whether the entry
|
|
84
|
+
* entered the context; shadow rounds always carry included:false there (the
|
|
85
|
+
* double-compute lands in meta/lesson-shadow.jsonl instead).
|
|
86
|
+
*/
|
|
87
|
+
lessonInject?: 'shadow' | 'on'
|
|
88
|
+
lessons?: { slug: string; score: number; source: string; included: boolean }[]
|
|
63
89
|
}
|
|
64
90
|
|
|
65
91
|
/**
|
package/src/index.ts
CHANGED
|
@@ -259,6 +259,14 @@ interface VolatileRef<T> {
|
|
|
259
259
|
* on bare test harnesses — {@link readConfigValue} accepts both. */
|
|
260
260
|
export type TopicsRuntimeConfig = Partial<{ [K in keyof TopicsConfigValue]: VolatileRef<TopicsConfigValue[K]> | TopicsConfigValue[K] }>
|
|
261
261
|
|
|
262
|
+
/**
|
|
263
|
+
* B-line dual-write seam contract (卷二 §6.2) — shared types are exported by
|
|
264
|
+
* THIS repo (C1 决议 3: no shared lib until a third consumer appears). The
|
|
265
|
+
* new repo calls TopicsService.feedObservations / setLessonStatus through the
|
|
266
|
+
* host process; direct bundle writes are forbidden on its side.
|
|
267
|
+
*/
|
|
268
|
+
export type { StructuredObservation, FeedResult, StoreResult } from './service.ts'
|
|
269
|
+
|
|
262
270
|
/** Snapshot one config slot: live ref → current value, plain value → itself. */
|
|
263
271
|
function readConfigValue(slot: unknown): unknown {
|
|
264
272
|
const ref = slot as { get?: unknown } | undefined
|
|
@@ -501,8 +509,9 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
|
|
|
501
509
|
if (cfg.autoInject) {
|
|
502
510
|
const consumed = slowLane.consume(sessionId, observer.turnCountOf(sessionId))
|
|
503
511
|
if (consumed !== undefined) {
|
|
504
|
-
if ('pending' in consumed)
|
|
505
|
-
|
|
512
|
+
if ('pending' in consumed) {
|
|
513
|
+
slow = { ...consumed.pending, outcome: consumed.pending.outcome }
|
|
514
|
+
} else slowExpired = consumed.expired
|
|
506
515
|
}
|
|
507
516
|
}
|
|
508
517
|
const r = service.retrieveSync(query, sessionId, seen === undefined && echo === undefined ? undefined : { exclude: seen, echo }, slow, slowExpired)
|
|
@@ -708,7 +717,10 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
|
|
|
708
717
|
// and its own settle hook does the release.
|
|
709
718
|
const run = distiller.request(sessionId, reason)
|
|
710
719
|
if (run !== undefined) {
|
|
711
|
-
void run
|
|
720
|
+
void run
|
|
721
|
+
.finally(() => releaseSessionLlm(sessionId))
|
|
722
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
723
|
+
.catch(() => undefined)
|
|
712
724
|
}
|
|
713
725
|
})
|
|
714
726
|
|
|
@@ -916,7 +928,11 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
|
|
|
916
928
|
// no-adapter detail instead
|
|
917
929
|
}
|
|
918
930
|
const run = distiller.request(sessionId, 'boot-replay')
|
|
919
|
-
if (run !== undefined)
|
|
931
|
+
if (run !== undefined) {
|
|
932
|
+
void run
|
|
933
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
934
|
+
.catch(() => undefined)
|
|
935
|
+
}
|
|
920
936
|
})
|
|
921
937
|
.catch(() => undefined)
|
|
922
938
|
// Consolidation cadence check — after the pull (freshest pool),
|
|
@@ -951,6 +967,9 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
|
|
|
951
967
|
})
|
|
952
968
|
.catch(() => undefined)
|
|
953
969
|
}
|
|
970
|
+
// Embedding index maintenance (A4/A6, spec §3): boot-time freshness
|
|
971
|
+
// check + incremental rebuild — provider '' / packageMode off = no-op.
|
|
972
|
+
void service.refreshEmbeddingIndex().catch(() => undefined)
|
|
954
973
|
})
|
|
955
974
|
.catch((error: unknown) => {
|
|
956
975
|
warn(`dsh-topics-memory 启动链失败(不影响会话启动):${error instanceof Error ? error.message : String(error)}`)
|
|
@@ -1075,7 +1094,11 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
|
|
|
1075
1094
|
? { ok: false, reason: 'in-flight', created: [], updated: [], marked: 0 }
|
|
1076
1095
|
: { ok: false, reason: 'no-model', created: [], updated: [], marked: 0 }
|
|
1077
1096
|
}
|
|
1078
|
-
|
|
1097
|
+
void run
|
|
1098
|
+
.finally(() => releaseSessionLlm(sessionId))
|
|
1099
|
+
.then(() => service.refreshEmbeddingIndex())
|
|
1100
|
+
.catch(() => undefined)
|
|
1101
|
+
return run
|
|
1079
1102
|
}
|
|
1080
1103
|
// Manual /topics consolidate trigger: same lane as the cadence path, same
|
|
1081
1104
|
// trigger-time llm capture and in-flight guard (the command runs inside a
|
|
@@ -1146,6 +1169,10 @@ const DEFAULTS: TopicsConfigValue = {
|
|
|
1146
1169
|
distillMaxModelCalls: 8,
|
|
1147
1170
|
consolidateCadence: 'daily',
|
|
1148
1171
|
deprecatedTtlDays: 15,
|
|
1172
|
+
embeddingProvider: '',
|
|
1173
|
+
embeddingModel: '',
|
|
1174
|
+
packageMode: 'off',
|
|
1175
|
+
lessonInject: 'off',
|
|
1149
1176
|
usageBoost: 0.15,
|
|
1150
1177
|
pushDebounceSeconds: 45,
|
|
1151
1178
|
}
|
package/src/okf.ts
CHANGED
|
@@ -17,6 +17,43 @@ import { parseYaml, stringifyYaml, YamlError } from './yaml.ts'
|
|
|
17
17
|
|
|
18
18
|
export type TopicStatus = 'draft' | 'stable' | 'deprecated'
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* Bundle entry kinds (B2, spec 2026-10-self-improve-topics §6.1): `topic` =
|
|
22
|
+
* factual knowledge (「是什么」), `lesson` = behavioral lesson (「怎么做事」).
|
|
23
|
+
* The kind rides the OKF `type` frontmatter field — the canonical serialized
|
|
24
|
+
* value for a lesson is `Lesson` (matching the `Topic` capitalization
|
|
25
|
+
* convention); recognition is case-insensitive so bundles written by the
|
|
26
|
+
* new repo's loop parse regardless of casing drift.
|
|
27
|
+
*/
|
|
28
|
+
export type TopicKind = 'topic' | 'lesson'
|
|
29
|
+
|
|
30
|
+
/** lesson scope (B2 MJ-2, spec §6.2): workspace lessons inject only in the
|
|
31
|
+
* workspace that produced them (cwd-hash match); global need no filter. */
|
|
32
|
+
export type LessonScope = 'workspace' | 'global'
|
|
33
|
+
/** lesson confidence (B2, spec §6.2): the FACT source — `status` derives from
|
|
34
|
+
* it (hint↔draft, established↔stable); hint never injects (MJ-3). */
|
|
35
|
+
export type LessonConfidence = 'hint' | 'established'
|
|
36
|
+
/** lesson provenance (B2 BL-2, spec §6.2): machine-recognizable source tag;
|
|
37
|
+
* uninstall reclamation keys on source≠manual (卷二 §9). */
|
|
38
|
+
export type LessonSource = 'distilled' | 'correction' | 'manual'
|
|
39
|
+
|
|
40
|
+
/** kind from the raw `type` frontmatter value (case-insensitive `lesson`). */
|
|
41
|
+
export function topicKindOf(type: string): TopicKind {
|
|
42
|
+
return typeof type === 'string' && type.trim().toLowerCase() === 'lesson' ? 'lesson' : 'topic'
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The canonical serialized `type` value per kind. */
|
|
46
|
+
export function kindTypeValue(kind: TopicKind): string {
|
|
47
|
+
return kind === 'lesson' ? 'Lesson' : 'Topic'
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** confidence 为事实源(B2 Minor):hint↔draft、established↔stable.
|
|
51
|
+
* `deprecated` is a lifecycle endpoint OUTSIDE the confidence axis — callers
|
|
52
|
+
* set it directly (setLessonStatus / consolidation / TTL sweep). */
|
|
53
|
+
export function deriveLessonStatus(confidence: LessonConfidence): TopicStatus {
|
|
54
|
+
return confidence === 'established' ? 'stable' : 'draft'
|
|
55
|
+
}
|
|
56
|
+
|
|
20
57
|
export interface SourceEntry {
|
|
21
58
|
id?: string
|
|
22
59
|
resource: string
|
|
@@ -52,6 +89,15 @@ export interface TopicFrontmatter {
|
|
|
52
89
|
verified?: TrustStamp
|
|
53
90
|
stale_after?: string
|
|
54
91
|
sources?: SourceEntry[]
|
|
92
|
+
// ---- lesson-kind fields (B2, spec §6.2) — present on kind=lesson docs ----
|
|
93
|
+
/** lesson 必带 scope(默认 workspace;global 需显式)。 */
|
|
94
|
+
scope?: LessonScope
|
|
95
|
+
/** confidence 事实源;status 派生(deriveLessonStatus)。默认 hint。 */
|
|
96
|
+
confidence?: LessonConfidence
|
|
97
|
+
/** provenance 标记(卸载回收按 source≠manual)。默认 manual。 */
|
|
98
|
+
source?: LessonSource
|
|
99
|
+
/** 写入时的 workspace 标识(cwd hash,MJ-2);scope=global 时可空。 */
|
|
100
|
+
workspaceId?: string
|
|
55
101
|
/** Producer-defined extras — preserved on round-trip per OKF §4.1. */
|
|
56
102
|
[extra: string]: unknown
|
|
57
103
|
}
|
|
@@ -148,8 +194,25 @@ export function parseTopicDoc(raw: string): TopicDoc {
|
|
|
148
194
|
return s as SourceEntry
|
|
149
195
|
})
|
|
150
196
|
}
|
|
197
|
+
// ---- lesson-kind fields (B2, spec §6.2) — parsed on lesson docs only ----
|
|
198
|
+
// Invalid enum values DEGRADE TO THE DEFAULT (triggers precedent: a producer
|
|
199
|
+
// slip must never invalidate an otherwise sound doc); the seam writers own
|
|
200
|
+
// the closed vocabulary. On lesson docs these four keys join the known set
|
|
201
|
+
// so the extras loop cannot double-store a raw unvalidated value.
|
|
202
|
+
if (topicKindOf(fmRaw.type) === 'lesson') {
|
|
203
|
+
fm.scope = fmRaw.scope === 'global' ? 'global' : 'workspace'
|
|
204
|
+
fm.confidence = fmRaw.confidence === 'established' ? 'established' : 'hint'
|
|
205
|
+
fm.source = fmRaw.source === 'distilled' || fmRaw.source === 'correction' ? fmRaw.source : 'manual'
|
|
206
|
+
if (typeof fmRaw.workspaceId === 'string' && fmRaw.workspaceId !== '') fm.workspaceId = fmRaw.workspaceId
|
|
207
|
+
}
|
|
151
208
|
// Preserve unknown producer-defined keys verbatim (OKF §4.1 extensions).
|
|
152
209
|
const known = new Set(['type', 'title', 'description', 'tags', 'triggers', 'depends', 'open_questions', 'impact', 'status', 'generated', 'verified', 'stale_after', 'sources'])
|
|
210
|
+
if (fm.scope !== undefined || fm.confidence !== undefined || fm.source !== undefined) {
|
|
211
|
+
known.add('scope')
|
|
212
|
+
known.add('confidence')
|
|
213
|
+
known.add('source')
|
|
214
|
+
known.add('workspaceId')
|
|
215
|
+
}
|
|
153
216
|
for (const [k, v] of Object.entries(fmRaw)) {
|
|
154
217
|
if (!known.has(k)) fm[k] = v
|
|
155
218
|
}
|
|
@@ -174,6 +237,11 @@ export function serializeTopicDoc(doc: TopicDoc): string {
|
|
|
174
237
|
if (fm.verified !== undefined) out.verified = { by: fm.verified.by, at: fm.verified.at }
|
|
175
238
|
if (fm.stale_after !== undefined) out.stale_after = fm.stale_after
|
|
176
239
|
if (fm.sources !== undefined && fm.sources.length > 0) out.sources = fm.sources
|
|
240
|
+
// lesson fields serialize in a fixed slot (deterministic order, spec §6.2).
|
|
241
|
+
if (fm.scope !== undefined) out.scope = fm.scope
|
|
242
|
+
if (fm.confidence !== undefined) out.confidence = fm.confidence
|
|
243
|
+
if (fm.source !== undefined) out.source = fm.source
|
|
244
|
+
if (fm.workspaceId !== undefined) out.workspaceId = fm.workspaceId
|
|
177
245
|
for (const [k, v] of Object.entries(fm)) {
|
|
178
246
|
if (!(k in out)) out[k] = v
|
|
179
247
|
}
|
package/src/paths.ts
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import { existsSync, renameSync } from 'node:fs'
|
|
16
|
+
import { createHash } from 'node:crypto'
|
|
16
17
|
import { homedir } from 'node:os'
|
|
17
18
|
import { hostname as osHostname } from 'node:os'
|
|
18
19
|
import { join } from 'node:path'
|
|
@@ -102,3 +103,17 @@ export function hostId(): string {
|
|
|
102
103
|
export function actorFor(): string {
|
|
103
104
|
return `agent:dsh-topics-memory@${hostId()}`
|
|
104
105
|
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Current workspace identifier for lesson scope filtering (B2 MJ-2, spec
|
|
109
|
+
* 卷一 §6.2): the sha256 prefix of the process cwd, overridable via
|
|
110
|
+
* $DSH_TOPICS_WORKSPACE_ID (tests / multi-root hosts; the override also lets
|
|
111
|
+
* a host pin one identity across chdirs). 拍板(spec 未钉死来源): cwd hash at
|
|
112
|
+
* read time is the minimal viable source — the same derivation the lesson
|
|
113
|
+
* write path stamps into frontmatter.workspaceId, so no config plumbing.
|
|
114
|
+
*/
|
|
115
|
+
export function currentWorkspaceId(): string {
|
|
116
|
+
const env = process.env.DSH_TOPICS_WORKSPACE_ID?.trim()
|
|
117
|
+
if (env !== undefined && env !== '') return env
|
|
118
|
+
return createHash('sha256').update(process.cwd()).digest('hex').slice(0, 16)
|
|
119
|
+
}
|