@aiwayds/dsh-topics-memory 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +55 -1
- package/lib/config.js +42 -0
- package/lib/digest.d.ts +111 -5
- package/lib/digest.js +437 -11
- package/lib/distill.d.ts +39 -0
- package/lib/distill.js +250 -15
- package/lib/embedding.d.ts +113 -0
- package/lib/embedding.js +208 -0
- package/lib/ilog.d.ts +30 -0
- package/lib/index.d.ts +15 -1
- package/lib/index.js +24 -6
- package/lib/okf.d.ts +34 -0
- package/lib/okf.js +41 -0
- package/lib/paths.d.ts +9 -0
- package/lib/paths.js +15 -0
- package/lib/pipeline-events.d.ts +38 -0
- package/lib/pipeline-events.js +123 -0
- package/lib/quality.d.ts +47 -7
- package/lib/quality.js +124 -20
- package/lib/retrieval.d.ts +106 -0
- package/lib/retrieval.js +280 -0
- package/lib/service.d.ts +103 -2
- package/lib/service.js +623 -16
- package/lib/store.d.ts +39 -3
- package/lib/store.js +60 -2
- package/package.json +2 -2
- package/src/config.ts +49 -0
- package/src/digest.ts +497 -8
- package/src/distill.ts +275 -14
- package/src/embedding.ts +294 -0
- package/src/ilog.ts +26 -0
- package/src/index.ts +32 -5
- package/src/okf.ts +68 -0
- package/src/paths.ts +15 -0
- package/src/pipeline-events.ts +136 -0
- package/src/quality.ts +145 -21
- package/src/retrieval.ts +335 -0
- package/src/service.ts +661 -19
- package/src/store.ts +77 -5
package/lib/distill.js
CHANGED
|
@@ -10,17 +10,19 @@
|
|
|
10
10
|
* @module distill
|
|
11
11
|
*/
|
|
12
12
|
import * as okf from "./okf.js";
|
|
13
|
+
import { recordPipelineEvent } from "./pipeline-events.js";
|
|
13
14
|
export const SYSTEM_PROMPT = [
|
|
14
15
|
'你是 topic 记忆库的蒸馏引擎。输入是若干条未蒸馏的会话观察(JSON)和现有 topic 索引。',
|
|
15
16
|
'任务:把观察沉淀成 Topic 操作。只输出一个 JSON 对象,不要任何其他文字、Markdown 代码块或解释。',
|
|
16
17
|
'输出格式:{"ops": [...]},每个元素:',
|
|
17
18
|
' {"op":"create","title":"主题名","description":"一句话","tags":["小写标签"],"triggers":["应想起本条的触发词"],"depends":["前置topic的slug"],',
|
|
18
|
-
' "open_questions":["未决问题"],"impact":["影响面"],"conclusion":"当前结论(自含散文)","recommendations":"可执行建议","status":"draft",',
|
|
19
|
+
' "open_questions":["未决问题"],"impact":["影响面"],"links":["关联topic的slug"],"conclusion":"当前结论(自含散文)","recommendations":"可执行建议","status":"draft",',
|
|
19
20
|
' "observed_ids":["obs-..."]}',
|
|
20
|
-
' {"op":"update","slug":"已有slug","conclusion":"修订后的完整结论","open_questions":[...],"observed_ids":["obs-..."]}',
|
|
21
|
+
' {"op":"update","slug":"已有slug","conclusion":"修订后的完整结论","open_questions":[...],"links":["关联topic的slug"],"observed_ids":["obs-..."]}',
|
|
21
22
|
'规则:',
|
|
22
23
|
'- 硬性要求:每个 op 必须带 observed_ids 字段,值只能从本次输入「未蒸馏观察」里列出的 id 中逐字复制(形如 "obs-...");create 填它所综合依据的观察 id,update 填促使本次修订的观察 id。含列表之外 id 的条目会被过滤;observed_ids 缺失、为空或过滤后不剩任何有效 id 的 op 会被整体丢弃。',
|
|
23
|
-
'- create
|
|
24
|
+
'- 硬性要求:每个 create 必须带 description(一句话概括)与 triggers(恰好 3-8 个「什么输入出现时应该想起这条记忆」的短语,写具体名词/术语,禁止 dsh、配置、错误 之类的泛词);缺失或数量不符的 create 会被补问一次,补全后仍不合规的 op 会被丢弃并落账。',
|
|
25
|
+
'- links(可选输出,不给不惩罚):只建真实语义关联(依赖/同项目/同主题链),宁缺勿滥;每项只能是已存在 topic 的 slug 或本批其他 op 的 slug,列表之外的引用会被剔除并落账。没有真实关联就不要给 links。',
|
|
24
26
|
'- conclusion 必须自含(不依赖观察原文也能读懂),写「目前有效的结论」,不是流水账。',
|
|
25
27
|
'- 同一主题只允许一个 create;已有相近 topic 时用 update 修订它的 conclusion。',
|
|
26
28
|
'- 观察里有价值就沉淀,没价值就跳过该观察(被跳过观察的 id 不要出现在任何 op 里);没有可沉淀内容时输出 {"ops":[]}。',
|
|
@@ -31,6 +33,36 @@ const MIN_BATCH_SIZE = 5;
|
|
|
31
33
|
/** Fallbacks when the config fields are absent (bare test harnesses). */
|
|
32
34
|
const DEFAULT_BATCH_SIZE = 40;
|
|
33
35
|
const DEFAULT_MAX_MODEL_CALLS = 8;
|
|
36
|
+
// ---------------------------------------------------------------------
|
|
37
|
+
// A6 §5.1 op schema gate + A2 links whitelist (always on, no config key:
|
|
38
|
+
// the distill quality contract is not a behavior switch — failures are
|
|
39
|
+
// ledgered via pipeline-events, never swallowed silently).
|
|
40
|
+
// ---------------------------------------------------------------------
|
|
41
|
+
/** create-op trigger bounds (the recall design's 3-8 strong-field contract). */
|
|
42
|
+
const TRIGGERS_MIN = 3;
|
|
43
|
+
const TRIGGERS_MAX = 8;
|
|
44
|
+
/** Stable identity for matching an op against its schema-retry re-emission. */
|
|
45
|
+
function opKey(op) {
|
|
46
|
+
if (op.op === 'update')
|
|
47
|
+
return typeof op.slug === 'string' ? okf.slugify(op.slug) : '';
|
|
48
|
+
return typeof op.title === 'string' ? okf.slugify(op.title) : '';
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* The A6 §5.1 schema check for create ops: description 必填 + triggers 3-8
|
|
52
|
+
* 条. Update ops keep their narrower contract (slug) — the prompt never
|
|
53
|
+
* required metadata fields on a conclusion revision, and gating those would
|
|
54
|
+
* drop every revision op.
|
|
55
|
+
*/
|
|
56
|
+
function checkCreateSchema(op) {
|
|
57
|
+
const problems = [];
|
|
58
|
+
if (typeof op.description !== 'string' || op.description.trim() === '')
|
|
59
|
+
problems.push('description');
|
|
60
|
+
const triggers = Array.isArray(op.triggers) ? op.triggers.filter((t) => typeof t === 'string' && t.trim() !== '') : [];
|
|
61
|
+
if (triggers.length < TRIGGERS_MIN || triggers.length > TRIGGERS_MAX) {
|
|
62
|
+
problems.push(`triggers(实际 ${triggers.length},需 ${TRIGGERS_MIN}-${TRIGGERS_MAX})`);
|
|
63
|
+
}
|
|
64
|
+
return { valid: problems.length === 0, missing: problems.join('、') };
|
|
65
|
+
}
|
|
34
66
|
export class Distiller {
|
|
35
67
|
inFlight = new Map();
|
|
36
68
|
service;
|
|
@@ -160,6 +192,8 @@ export class Distiller {
|
|
|
160
192
|
let failureDetail;
|
|
161
193
|
let firstFetch = true;
|
|
162
194
|
let filteredTotal = 0;
|
|
195
|
+
let invalidTotal = 0;
|
|
196
|
+
let rejectedTotal = 0;
|
|
163
197
|
let stallDetail;
|
|
164
198
|
// GC bookkeeping (store.recordUnconsumed): every id the model EVALUATED
|
|
165
199
|
// this run (parseable answer, however useless) minus the ids an op
|
|
@@ -191,6 +225,8 @@ export class Distiller {
|
|
|
191
225
|
for (const id of outcome.consumedIds ?? [])
|
|
192
226
|
consumedRunIds.add(id);
|
|
193
227
|
filteredTotal += outcome.filteredIds ?? 0;
|
|
228
|
+
invalidTotal += outcome.invalidOps ?? 0;
|
|
229
|
+
rejectedTotal += outcome.rejectedLinks ?? 0;
|
|
194
230
|
created.push(...outcome.created);
|
|
195
231
|
updated.push(...outcome.updated);
|
|
196
232
|
marked += outcome.marked;
|
|
@@ -272,6 +308,9 @@ export class Distiller {
|
|
|
272
308
|
? `后续批次未消费任何观察,本轮停止${stallDetail !== undefined ? `:${stallDetail}` : ''}`
|
|
273
309
|
: undefined;
|
|
274
310
|
const filteredNote = filteredTotal > 0 ? `;filtered ${filteredTotal} invalid observed_ids` : '';
|
|
311
|
+
const schemaNote = invalidTotal > 0 ? `;schema-dropped ${invalidTotal} create op(s)` : '';
|
|
312
|
+
const linkNote = rejectedTotal > 0 ? `;rejected ${rejectedTotal} invalid link ref(s)` : '';
|
|
313
|
+
const notes = `${filteredNote}${schemaNote}${linkNote}`;
|
|
275
314
|
const base = head === undefined ? undefined : `partial: 已蒸馏标记 ${marked} 条观察(${calls} 次模型调用);${head}`;
|
|
276
315
|
result = {
|
|
277
316
|
ok: true,
|
|
@@ -279,10 +318,10 @@ export class Distiller {
|
|
|
279
318
|
updated,
|
|
280
319
|
marked,
|
|
281
320
|
detail: base === undefined
|
|
282
|
-
?
|
|
321
|
+
? notes === ''
|
|
283
322
|
? undefined
|
|
284
|
-
:
|
|
285
|
-
: `${base}${
|
|
323
|
+
: notes.replace(/^;/, '')
|
|
324
|
+
: `${base}${notes}`,
|
|
286
325
|
};
|
|
287
326
|
}
|
|
288
327
|
else {
|
|
@@ -369,6 +408,29 @@ export class Distiller {
|
|
|
369
408
|
if (ops.length === 0) {
|
|
370
409
|
return { created: [], updated: [], marked: 0, noOps: true, callsUsed: 1 };
|
|
371
410
|
}
|
|
411
|
+
// A6 §5.1 schema gate BEFORE any write: deficient creates get exactly one
|
|
412
|
+
// 补问 retry (budget-guarded), still-deficient ops are dropped and
|
|
413
|
+
// ledgered — never a silent drop. Their observations simply stay
|
|
414
|
+
// unconsumed and walk the existing GC/corrective path (three strikes).
|
|
415
|
+
const schema = await this.enforceSchema(sessionId, user, ops, budgetLeft);
|
|
416
|
+
let callsUsed = 1 + schema.retrySpent;
|
|
417
|
+
for (const drop of schema.drops) {
|
|
418
|
+
await recordPipelineEvent(this.service.store.root, {
|
|
419
|
+
at: new Date().toISOString(),
|
|
420
|
+
kind: drop.kind,
|
|
421
|
+
detail: { op: drop.key, missing: drop.missing },
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
if (schema.finalOps.length === 0) {
|
|
425
|
+
return {
|
|
426
|
+
created: [],
|
|
427
|
+
updated: [],
|
|
428
|
+
marked: 0,
|
|
429
|
+
callsUsed,
|
|
430
|
+
invalidOps: schema.drops.length,
|
|
431
|
+
stallDetail: `全部 create op 未通过 schema 校验(description 必填 + triggers 3-8 条),已丢弃并落账(${schema.drops.length} 个)`,
|
|
432
|
+
};
|
|
433
|
+
}
|
|
372
434
|
// observed_ids are the ONLY bridge between ops and marks — without a valid
|
|
373
435
|
// id the store cannot mark anything and the batch head can never advance
|
|
374
436
|
// (the real-machine zero-consumption livelock: topics landed, backlog
|
|
@@ -378,10 +440,18 @@ export class Distiller {
|
|
|
378
440
|
// corrective retry re-emits it with fixed ids.
|
|
379
441
|
const validIds = new Set(observations.map((o) => o.id));
|
|
380
442
|
const idList = observations.map((o) => o.id);
|
|
381
|
-
|
|
382
|
-
|
|
443
|
+
// Link whitelist universe (A2): slugs existing before this batch plus
|
|
444
|
+
// every same-batch op slug (predicted; the actual create slug joins as
|
|
445
|
+
// each op lands).
|
|
446
|
+
const knownSlugs = new Set(metas.map((m) => m.slug));
|
|
447
|
+
for (const op of schema.finalOps) {
|
|
448
|
+
const key = opKey(op);
|
|
449
|
+
if (key !== '')
|
|
450
|
+
knownSlugs.add(key);
|
|
451
|
+
}
|
|
452
|
+
let applied = await this.applyOps(schema.finalOps, validIds, knownSlugs);
|
|
383
453
|
if (applied.consumedIds.length === 0 && applied.droppedForIds > 0) {
|
|
384
|
-
if (budgetLeft <
|
|
454
|
+
if (budgetLeft < callsUsed + 1) {
|
|
385
455
|
// No free lane: the retry must fit the same run budget.
|
|
386
456
|
return {
|
|
387
457
|
created: [],
|
|
@@ -389,12 +459,14 @@ export class Distiller {
|
|
|
389
459
|
marked: 0,
|
|
390
460
|
callsUsed,
|
|
391
461
|
filteredIds: applied.filteredIds,
|
|
392
|
-
|
|
462
|
+
rejectedLinks: applied.rejectedLinks,
|
|
463
|
+
invalidOps: schema.drops.length,
|
|
464
|
+
stallDetail: `ops 未包含有效 observed_ids(已过滤 ${applied.filteredIds} 个无效 id),且模型调用预算不足以纠错重试(剩余 ${budgetLeft - callsUsed} 次)`,
|
|
393
465
|
};
|
|
394
466
|
}
|
|
395
467
|
const pass1 = applied;
|
|
396
|
-
const { applied: pass2, why } = await this.correctiveRetry(sessionId, user, idList, validIds);
|
|
397
|
-
callsUsed
|
|
468
|
+
const { applied: pass2, why } = await this.correctiveRetry(sessionId, user, idList, validIds, knownSlugs);
|
|
469
|
+
callsUsed += 1;
|
|
398
470
|
if (pass2 === undefined || pass2.consumedIds.length === 0) {
|
|
399
471
|
const tail = pass2 === undefined
|
|
400
472
|
? `纠错重试未产出可消费的 ops(${why})`
|
|
@@ -405,6 +477,8 @@ export class Distiller {
|
|
|
405
477
|
marked: 0,
|
|
406
478
|
callsUsed,
|
|
407
479
|
filteredIds: pass1.filteredIds + (pass2?.filteredIds ?? 0),
|
|
480
|
+
rejectedLinks: pass1.rejectedLinks + (pass2?.rejectedLinks ?? 0),
|
|
481
|
+
invalidOps: schema.drops.length,
|
|
408
482
|
stallDetail: `${tail};本批 ${observations.length} 条观察零消费`,
|
|
409
483
|
};
|
|
410
484
|
}
|
|
@@ -415,6 +489,7 @@ export class Distiller {
|
|
|
415
489
|
consumedSlugs: pass2.consumedSlugs,
|
|
416
490
|
filteredIds: pass1.filteredIds + pass2.filteredIds,
|
|
417
491
|
droppedForIds: pass2.droppedForIds,
|
|
492
|
+
rejectedLinks: pass1.rejectedLinks + pass2.rejectedLinks,
|
|
418
493
|
};
|
|
419
494
|
}
|
|
420
495
|
let marked = 0;
|
|
@@ -428,6 +503,8 @@ export class Distiller {
|
|
|
428
503
|
marked,
|
|
429
504
|
callsUsed,
|
|
430
505
|
filteredIds: applied.filteredIds,
|
|
506
|
+
invalidOps: schema.drops.length,
|
|
507
|
+
rejectedLinks: applied.rejectedLinks,
|
|
431
508
|
consumedIds: applied.consumedIds,
|
|
432
509
|
};
|
|
433
510
|
}
|
|
@@ -439,7 +516,7 @@ export class Distiller {
|
|
|
439
516
|
* stop — a max-tokens correction does NOT trigger the batch halving (that
|
|
440
517
|
* escape hatch belongs to the main pass).
|
|
441
518
|
*/
|
|
442
|
-
async correctiveRetry(sessionId, originalUser, idList, validIds) {
|
|
519
|
+
async correctiveRetry(sessionId, originalUser, idList, validIds, knownSlugs) {
|
|
443
520
|
const caller = this.caller;
|
|
444
521
|
if (caller === undefined)
|
|
445
522
|
return { why: 'distill caller unavailable' };
|
|
@@ -468,7 +545,112 @@ export class Distiller {
|
|
|
468
545
|
}
|
|
469
546
|
if (retried.length === 0)
|
|
470
547
|
return { why: '模型返回空 ops' };
|
|
471
|
-
return { applied: await this.applyOps(retried, validIds), why: '' };
|
|
548
|
+
return { applied: await this.applyOps(retried, validIds, knownSlugs), why: '' };
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* The A6 §5.1 schema gate, applied to the parsed batch BEFORE any write:
|
|
552
|
+
* create ops must carry description + triggers 3-8 条 (the prompt's standing
|
|
553
|
+
* requirement, now enforced). Deficient creates get exactly ONE 补问 retry
|
|
554
|
+
* that re-asks for just those ops completed — budget-guarded exactly like
|
|
555
|
+
* the corrective ids retry (`retrySpent` reports the extra call). Every
|
|
556
|
+
* still-deficient op is dropped and comes back as a SchemaDrop carrying the
|
|
557
|
+
* ledger kind: `distill-op-invalid` after a used-but-unsuccessful retry,
|
|
558
|
+
* `distill-op-retry-exhausted` when no retry could run or the retry call
|
|
559
|
+
* itself failed. Dropped ops' observations stay unconsumed — the existing
|
|
560
|
+
* GC/corrective path accounts for them, never a silent loss. Update ops
|
|
561
|
+
* keep their narrower contract (slug only).
|
|
562
|
+
*/
|
|
563
|
+
async enforceSchema(sessionId, originalUser, ops, budgetLeft) {
|
|
564
|
+
const valid = [];
|
|
565
|
+
const invalid = [];
|
|
566
|
+
for (const op of ops) {
|
|
567
|
+
if (op.op !== 'create') {
|
|
568
|
+
valid.push(op);
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
const check = checkCreateSchema(op);
|
|
572
|
+
if (check.valid)
|
|
573
|
+
valid.push(op);
|
|
574
|
+
else
|
|
575
|
+
invalid.push({ op, key: opKey(op), missing: check.missing });
|
|
576
|
+
}
|
|
577
|
+
if (invalid.length === 0)
|
|
578
|
+
return { finalOps: valid, drops: [], retrySpent: 0 };
|
|
579
|
+
if (budgetLeft < 2) {
|
|
580
|
+
return {
|
|
581
|
+
finalOps: valid,
|
|
582
|
+
drops: invalid.map((c) => ({
|
|
583
|
+
key: c.key,
|
|
584
|
+
kind: 'distill-op-retry-exhausted',
|
|
585
|
+
missing: `${c.missing}(模型调用预算不足以补问)`,
|
|
586
|
+
})),
|
|
587
|
+
retrySpent: 0,
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
const rescued = await this.schemaRetryCall(sessionId, originalUser, invalid);
|
|
591
|
+
if (rescued === undefined) {
|
|
592
|
+
return {
|
|
593
|
+
finalOps: valid,
|
|
594
|
+
drops: invalid.map((c) => ({
|
|
595
|
+
key: c.key,
|
|
596
|
+
kind: 'distill-op-retry-exhausted',
|
|
597
|
+
missing: `${c.missing}(补问调用失败)`,
|
|
598
|
+
})),
|
|
599
|
+
retrySpent: 1,
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
const drops = [];
|
|
603
|
+
for (const c of invalid) {
|
|
604
|
+
const replacement = rescued.get(c.key);
|
|
605
|
+
if (replacement !== undefined && checkCreateSchema(replacement).valid) {
|
|
606
|
+
valid.push(replacement);
|
|
607
|
+
}
|
|
608
|
+
else {
|
|
609
|
+
drops.push({ key: c.key, kind: 'distill-op-invalid', missing: c.missing });
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
return { finalOps: valid, drops, retrySpent: 1 };
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* The one schema 补问: re-ask for exactly the deficient creates, completed.
|
|
616
|
+
* Returns key→op for what the model re-emitted (matched by slugified title),
|
|
617
|
+
* or undefined when the call itself failed / output was unparseable — the
|
|
618
|
+
* caller then ledgeres `distill-op-retry-exhausted` for every deficient op.
|
|
619
|
+
*/
|
|
620
|
+
async schemaRetryCall(sessionId, originalUser, invalid) {
|
|
621
|
+
const caller = this.caller;
|
|
622
|
+
if (caller === undefined)
|
|
623
|
+
return undefined;
|
|
624
|
+
const list = invalid.map((c) => `- create「${typeof c.op.title === 'string' ? c.op.title : ''}」(缺:${c.missing})`).join('\n');
|
|
625
|
+
const user = [
|
|
626
|
+
originalUser,
|
|
627
|
+
'',
|
|
628
|
+
'---',
|
|
629
|
+
'schema 补问:你上一次返回的以下 create op 缺少必填字段:',
|
|
630
|
+
list,
|
|
631
|
+
'请只重新输出这些 op 的补全版本(严格 JSON,{"ops":[...]}):每个 create 必须带 description(一句话概括)与 triggers(恰好 3-8 条具体触发短语,禁止 dsh、配置、错误 之类的泛词),并保留原有的 title/conclusion/observed_ids 等字段。无法补全的 op 不要输出。',
|
|
632
|
+
].join('\n');
|
|
633
|
+
let raw;
|
|
634
|
+
try {
|
|
635
|
+
raw = await caller({ system: SYSTEM_PROMPT, user, purpose: 'topics-distill', sessionId, maxTokens: 4000 });
|
|
636
|
+
}
|
|
637
|
+
catch {
|
|
638
|
+
return undefined;
|
|
639
|
+
}
|
|
640
|
+
let retried;
|
|
641
|
+
try {
|
|
642
|
+
retried = parseOps(raw);
|
|
643
|
+
}
|
|
644
|
+
catch {
|
|
645
|
+
return undefined;
|
|
646
|
+
}
|
|
647
|
+
const map = new Map();
|
|
648
|
+
for (const op of retried) {
|
|
649
|
+
const key = opKey(op);
|
|
650
|
+
if (key !== '')
|
|
651
|
+
map.set(key, op);
|
|
652
|
+
}
|
|
653
|
+
return map;
|
|
472
654
|
}
|
|
473
655
|
/**
|
|
474
656
|
* Execute a batch's ops with observed_ids sanitized upfront. A scalar
|
|
@@ -481,8 +663,21 @@ export class Distiller {
|
|
|
481
663
|
* failures (missing fields, unknown update slug) keep the per-op try/catch
|
|
482
664
|
* isolation — one bad op never sinks the batch, and a stall they cause is
|
|
483
665
|
* NOT attributed to observed_ids (no corrective retry for those).
|
|
666
|
+
*
|
|
667
|
+
* A2 links: every link target is whitelist-checked against `knownSlugs`
|
|
668
|
+
* (existing slugs ∪ same-batch op slugs); rejected refs are stripped and
|
|
669
|
+
* ledgered (`link-ref-rejected`), survivors ride the op's topic write into
|
|
670
|
+
* the store transaction (先落 topic 再建边 — one write, no separate edge
|
|
671
|
+
* pass). The set is locally seeded with this call's predicted op slugs and
|
|
672
|
+
* grows as creates land, so same-batch references resolve in op order.
|
|
484
673
|
*/
|
|
485
|
-
async applyOps(ops, validIds) {
|
|
674
|
+
async applyOps(ops, validIds, knownSlugs) {
|
|
675
|
+
const slugs = new Set(knownSlugs);
|
|
676
|
+
for (const op of ops) {
|
|
677
|
+
const key = opKey(op);
|
|
678
|
+
if (key !== '')
|
|
679
|
+
slugs.add(key);
|
|
680
|
+
}
|
|
486
681
|
const applied = {
|
|
487
682
|
created: [],
|
|
488
683
|
updated: [],
|
|
@@ -490,6 +685,7 @@ export class Distiller {
|
|
|
490
685
|
consumedSlugs: [],
|
|
491
686
|
filteredIds: 0,
|
|
492
687
|
droppedForIds: 0,
|
|
688
|
+
rejectedLinks: 0,
|
|
493
689
|
};
|
|
494
690
|
for (const op of ops) {
|
|
495
691
|
try {
|
|
@@ -510,6 +706,8 @@ export class Distiller {
|
|
|
510
706
|
continue;
|
|
511
707
|
}
|
|
512
708
|
if (op.op === 'create' && typeof op.title === 'string' && typeof op.conclusion === 'string') {
|
|
709
|
+
const links = await this.acceptLinks(op, slugs);
|
|
710
|
+
applied.rejectedLinks += links.rejected;
|
|
513
711
|
const res = await this.service.saveTopic({
|
|
514
712
|
title: op.title,
|
|
515
713
|
conclusion: op.conclusion,
|
|
@@ -522,14 +720,18 @@ export class Distiller {
|
|
|
522
720
|
recommendations: op.recommendations,
|
|
523
721
|
status: op.status === 'stable' ? 'stable' : 'draft',
|
|
524
722
|
source: 'distill',
|
|
723
|
+
links: links.targets,
|
|
525
724
|
});
|
|
526
725
|
applied.created.push(res.slug);
|
|
527
726
|
applied.consumedSlugs.push(res.slug);
|
|
727
|
+
slugs.add(res.slug);
|
|
528
728
|
}
|
|
529
729
|
else if (op.op === 'update' && typeof op.slug === 'string') {
|
|
530
730
|
const existing = await this.service.store.readTopic(op.slug);
|
|
531
731
|
if (existing === undefined)
|
|
532
732
|
continue;
|
|
733
|
+
const links = await this.acceptLinks(op, slugs);
|
|
734
|
+
applied.rejectedLinks += links.rejected;
|
|
533
735
|
const res = await this.service.saveTopic({
|
|
534
736
|
title: existing.fm.title,
|
|
535
737
|
conclusion: typeof op.conclusion === 'string' ? op.conclusion : undefined,
|
|
@@ -537,9 +739,11 @@ export class Distiller {
|
|
|
537
739
|
status: op.status === 'stable' || op.status === 'deprecated' ? op.status : undefined,
|
|
538
740
|
slug: op.slug,
|
|
539
741
|
source: 'distill',
|
|
742
|
+
links: links.targets,
|
|
540
743
|
});
|
|
541
744
|
applied.updated.push(res.slug);
|
|
542
745
|
applied.consumedSlugs.push(res.slug);
|
|
746
|
+
slugs.add(res.slug);
|
|
543
747
|
}
|
|
544
748
|
else {
|
|
545
749
|
continue;
|
|
@@ -552,6 +756,37 @@ export class Distiller {
|
|
|
552
756
|
}
|
|
553
757
|
return applied;
|
|
554
758
|
}
|
|
759
|
+
/**
|
|
760
|
+
* A2 links whitelist: a link survives only when its normalized target is a
|
|
761
|
+
* slug that already exists or belongs to this batch (`knownSlugs`), and not
|
|
762
|
+
* the op's own topic. Everything else is stripped and ledgered
|
|
763
|
+
* (`link-ref-rejected` — a dead edge must at least leave a trace). Returns
|
|
764
|
+
* the surviving targets (undefined when none — no Links section is written)
|
|
765
|
+
* plus the rejection count for the run detail.
|
|
766
|
+
*/
|
|
767
|
+
async acceptLinks(op, knownSlugs) {
|
|
768
|
+
if (!Array.isArray(op.links) || op.links.length === 0)
|
|
769
|
+
return { targets: undefined, rejected: 0 };
|
|
770
|
+
const self = opKey(op);
|
|
771
|
+
const accepted = new Set();
|
|
772
|
+
let rejected = 0;
|
|
773
|
+
for (const raw of op.links) {
|
|
774
|
+
if (typeof raw !== 'string')
|
|
775
|
+
continue;
|
|
776
|
+
const target = okf.normalizeLinkTarget(raw);
|
|
777
|
+
if (target === undefined || target === self || !knownSlugs.has(target)) {
|
|
778
|
+
await recordPipelineEvent(this.service.store.root, {
|
|
779
|
+
at: new Date().toISOString(),
|
|
780
|
+
kind: 'link-ref-rejected',
|
|
781
|
+
detail: { op: self, ref: raw },
|
|
782
|
+
});
|
|
783
|
+
rejected += 1;
|
|
784
|
+
continue;
|
|
785
|
+
}
|
|
786
|
+
accepted.add(target);
|
|
787
|
+
}
|
|
788
|
+
return { targets: accepted.size > 0 ? [...accepted] : undefined, rejected };
|
|
789
|
+
}
|
|
555
790
|
}
|
|
556
791
|
/** Extract the first balanced JSON object from model output (tolerates fences). */
|
|
557
792
|
export function parseOps(raw) {
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local embedding client (ollama) — the semantic layer behind the nearFloor
|
|
3
|
+
* reranker (A4 决议 2, spec §3.1).
|
|
4
|
+
*
|
|
5
|
+
* Semantics (mirror of jev/client's outcome mode):
|
|
6
|
+
* - every failure returns as a structured {ok:false, outcome} result, NEVER
|
|
7
|
+
* a throw — the caller degrades explicitly to the pure-lexical path;
|
|
8
|
+
* - one AbortSignal.timeout per request, NO retry (query 2s / batch 30s);
|
|
9
|
+
* - localhost only in v1 (ADR 0017 local exception). Remote providers do
|
|
10
|
+
* not exist here by design: a remote embedding may only ever enter a
|
|
11
|
+
* slow-lane/offline path, never the fast lane (red line ①).
|
|
12
|
+
*
|
|
13
|
+
* @module embedding
|
|
14
|
+
*/
|
|
15
|
+
/** Default ollama bind (the plugin's own default; OLLAMA_HOST overrides). */
|
|
16
|
+
export declare const DEFAULT_EMBED_HOST = "http://127.0.0.1:11434";
|
|
17
|
+
/** Provider default model when embeddingModel is the '' sentinel
|
|
18
|
+
* (spec §7: ollama → nomic-embed-text, 768 dims, A1 §5 measured). */
|
|
19
|
+
export declare const DEFAULT_EMBED_MODEL = "nomic-embed-text";
|
|
20
|
+
/** Query-side timeout — the reranker sits in the spliced hot path; a slow
|
|
21
|
+
* ollama must lose the race, not the injection. */
|
|
22
|
+
export declare const EMBED_QUERY_TIMEOUT_MS = 2000;
|
|
23
|
+
/** Batch-side timeout (index rebuild: A1 measured ~9.5s for 137 topics). */
|
|
24
|
+
export declare const EMBED_BATCH_TIMEOUT_MS = 30000;
|
|
25
|
+
/** Per-text byte cap before anything leaves the process (query端 p50=153 tok
|
|
26
|
+
* ~ well under this; the cap only bites on paste pollution). */
|
|
27
|
+
export declare const MAX_TEXT_BYTES = 2048;
|
|
28
|
+
/** The only provider v1 speaks. Anything else is an explicit
|
|
29
|
+
* unsupported_provider outcome, not a silent attempt. */
|
|
30
|
+
export declare const SUPPORTED_PROVIDER = "ollama";
|
|
31
|
+
/** The config slice the embed functions need (pass straight from
|
|
32
|
+
* TopicsConfigValue). '' sentinels: provider off / provider default model. */
|
|
33
|
+
export interface EmbeddingConfig {
|
|
34
|
+
provider?: string;
|
|
35
|
+
model?: string;
|
|
36
|
+
/** Explicit host; absent → OLLAMA_HOST env → DEFAULT_EMBED_HOST. */
|
|
37
|
+
host?: string;
|
|
38
|
+
}
|
|
39
|
+
/** Resolve the ollama base URL: explicit arg wins, then OLLAMA_HOST, then
|
|
40
|
+
* the localhost default. Bare `host:port` values get the http:// prefix
|
|
41
|
+
* (ollama's own OLLAMA_HOST convention); trailing slashes are trimmed. */
|
|
42
|
+
export declare function resolveEmbedHost(explicit?: string): string;
|
|
43
|
+
/** '' sentinel → provider default model. */
|
|
44
|
+
export declare function resolveEmbedModel(model?: string): string;
|
|
45
|
+
/** Truncate a string to ≤ maxBytes UTF-8 bytes WITHOUT splitting a multibyte
|
|
46
|
+
* character: cut at the byte cap, then back off over any continuation bytes
|
|
47
|
+
* (0b10xxxxxx) so the decoded tail is never a replacement char. */
|
|
48
|
+
export declare function truncateUtf8(text: string, maxBytes?: number): string;
|
|
49
|
+
export type EmbeddingFailureOutcome = 'provider_off' | 'unsupported_provider' | 'empty_input' | 'network' | 'timeout' | 'model_missing' | 'http_4xx' | 'http_5xx' | 'bad_json' | 'bad_shape';
|
|
50
|
+
export interface EmbedBatchOk {
|
|
51
|
+
ok: true;
|
|
52
|
+
/** Vectors aligned 1:1 with the (truncated) input texts. */
|
|
53
|
+
vectors: number[][];
|
|
54
|
+
model: string;
|
|
55
|
+
dim: number;
|
|
56
|
+
latencyMs: number;
|
|
57
|
+
}
|
|
58
|
+
export interface EmbedQueryOk {
|
|
59
|
+
ok: true;
|
|
60
|
+
vector: number[];
|
|
61
|
+
model: string;
|
|
62
|
+
dim: number;
|
|
63
|
+
latencyMs: number;
|
|
64
|
+
}
|
|
65
|
+
export interface EmbedFailure {
|
|
66
|
+
ok: false;
|
|
67
|
+
outcome: EmbeddingFailureOutcome;
|
|
68
|
+
message: string;
|
|
69
|
+
host: string;
|
|
70
|
+
model: string;
|
|
71
|
+
latencyMs: number;
|
|
72
|
+
}
|
|
73
|
+
export type EmbedResult = EmbedBatchOk | EmbedFailure;
|
|
74
|
+
export type EmbedQueryResult = EmbedQueryOk | EmbedFailure;
|
|
75
|
+
export type ProbeResult = {
|
|
76
|
+
ok: true;
|
|
77
|
+
host: string;
|
|
78
|
+
model: string;
|
|
79
|
+
dim: number;
|
|
80
|
+
latencyMs: number;
|
|
81
|
+
} | EmbedFailure;
|
|
82
|
+
/** Batch-embed texts (index build/rebuild path). Vectors align 1:1 with the
|
|
83
|
+
* input order; each text is truncated to MAX_TEXT_BYTES first. */
|
|
84
|
+
export declare function embedTexts(texts: readonly string[], config?: EmbeddingConfig, opts?: {
|
|
85
|
+
timeoutMs?: number;
|
|
86
|
+
}): Promise<EmbedResult>;
|
|
87
|
+
/** Embed one query (reranker hot path, 2s budget). Empty/whitespace queries
|
|
88
|
+
* fail locally with 'empty_input' — nothing leaves the process. */
|
|
89
|
+
export declare function embedQuery(query: string, config?: EmbeddingConfig, opts?: {
|
|
90
|
+
timeoutMs?: number;
|
|
91
|
+
}): Promise<EmbedQueryResult>;
|
|
92
|
+
/**
|
|
93
|
+
* Probe: is the server there, is the model pulled? One micro embed call —
|
|
94
|
+
* unlike a /api/tags listing it validates the FULL path (server + model +
|
|
95
|
+
* response shape) and yields the vector dim for index validation, at the
|
|
96
|
+
* same 15-30ms cost (A1 §5). Structured result; never throws.
|
|
97
|
+
*/
|
|
98
|
+
export declare function probeEmbedding(config?: EmbeddingConfig, opts?: {
|
|
99
|
+
timeoutMs?: number;
|
|
100
|
+
}): Promise<ProbeResult>;
|
|
101
|
+
/**
|
|
102
|
+
* Build an embedder handle for retrieval's `ensureIndexFresh` — a callable
|
|
103
|
+
* embedTexts bound to this config, carrying the RESOLVED model identity the
|
|
104
|
+
* index must match. ensureIndexFresh reads `.model` only; the actual
|
|
105
|
+
* rebuild stays with the caller.
|
|
106
|
+
*/
|
|
107
|
+
export declare function embedderWith(config: EmbeddingConfig): ((texts: readonly string[]) => Promise<EmbedResult>) & {
|
|
108
|
+
model: string;
|
|
109
|
+
};
|
|
110
|
+
/** Hard sync budget for the hot-lane query embed. Generous over the measured
|
|
111
|
+
* 15-30ms, merciless for a wedged server. */
|
|
112
|
+
export declare const EMBED_QUERY_SYNC_TIMEOUT_MS = 250;
|
|
113
|
+
export declare function embedQuerySync(query: string, config?: EmbeddingConfig): EmbedQueryResult;
|