@aiwayds/dsh-topics-memory 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +18 -1
- package/lib/config.js +13 -0
- package/lib/digest.d.ts +38 -5
- package/lib/digest.js +57 -15
- package/lib/ilog.d.ts +14 -0
- package/lib/index.d.ts +9 -1
- package/lib/index.js +1 -0
- package/lib/okf.d.ts +34 -0
- package/lib/okf.js +41 -0
- package/lib/paths.d.ts +9 -0
- package/lib/paths.js +15 -0
- package/lib/pipeline-events.d.ts +4 -2
- package/lib/pipeline-events.js +2 -0
- package/lib/service.d.ts +88 -3
- package/lib/service.js +435 -19
- package/lib/store.d.ts +30 -3
- package/lib/store.js +28 -1
- package/package.json +2 -2
- package/src/config.ts +15 -0
- package/src/digest.ts +85 -12
- package/src/ilog.ts +9 -0
- package/src/index.ts +9 -0
- package/src/okf.ts +68 -0
- package/src/paths.ts +15 -0
- package/src/pipeline-events.ts +6 -2
- package/src/service.ts +476 -21
- package/src/store.ts +45 -3
package/lib/service.js
CHANGED
|
@@ -10,10 +10,11 @@ import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
|
10
10
|
import { join } from 'node:path';
|
|
11
11
|
import * as okf from "./okf.js";
|
|
12
12
|
import { searchTopics, scoreTopic, passesGate, tokenize, sanitizeQuery, loadEmbeddingIndex, rerankNearFloor, ensureIndexFresh, saveEmbeddingIndex } from "./retrieval.js";
|
|
13
|
-
import { assembleInjection, assemblePointer, assemblePackage, POINTER_PER_TOPIC, POINTER_TOTAL } from "./digest.js";
|
|
13
|
+
import { assembleInjection, assemblePointer, assemblePackage, lessonEntryText, lessonSourceTag, POINTER_PER_TOPIC, POINTER_TOTAL } from "./digest.js";
|
|
14
14
|
import { embedTexts, embedQuerySync, embedderWith, resolveEmbedModel } from "./embedding.js";
|
|
15
|
-
import { resolveBundleRoot } from "./paths.js";
|
|
15
|
+
import { currentWorkspaceId, resolveBundleRoot } from "./paths.js";
|
|
16
16
|
import { aggregateStats, querySample, aggregateUsage, aggregateZeroOpen, ZERO_OPEN_TAIL, USAGE_WINDOW_DAYS, attributeOpen, OPEN_ATTRIBUTION_TAIL } from "./ilog.js";
|
|
17
|
+
import { recordPipelineEvent } from "./pipeline-events.js";
|
|
17
18
|
import { fileHistory, fileAtRev } from "./git.js";
|
|
18
19
|
/** Tolerant JSONL line read: undefined = unreadable file. Unlike an empty
|
|
19
20
|
* array, undefined tells the caller to fail open AND skip caching — a
|
|
@@ -113,6 +114,12 @@ export class TopicsService {
|
|
|
113
114
|
// with its survivor for the injection budget.
|
|
114
115
|
if (doc.fm.status === 'deprecated')
|
|
115
116
|
continue;
|
|
117
|
+
// Lesson-kind entries (B2) never ride the topic pipeline — they have
|
|
118
|
+
// their own gate (lessonInject), confidence rule and scope filter in
|
|
119
|
+
// lessonsSync(). Gate-independent separation: even with lessonInject
|
|
120
|
+
// off, a hint lesson must not leak through the topic lexical lane.
|
|
121
|
+
if (okf.topicKindOf(doc.fm.type) === 'lesson')
|
|
122
|
+
continue;
|
|
116
123
|
out.push({
|
|
117
124
|
slug: entry.slug,
|
|
118
125
|
title: doc.fm.title,
|
|
@@ -292,17 +299,18 @@ export class TopicsService {
|
|
|
292
299
|
}
|
|
293
300
|
/** Shadow bypass ledger (spec §8.3): one line per shadow round — feature
|
|
294
301
|
* fields only, never query text (jev-log discipline). Silent on failure;
|
|
295
|
-
* 512KB truncation keeps the newest three quarters.
|
|
296
|
-
|
|
302
|
+
* 512KB truncation keeps the newest three quarters. Shared by the package
|
|
303
|
+
* shadow (meta/package-shadow.jsonl) and the lesson shadow (B2,
|
|
304
|
+
* meta/lesson-shadow.jsonl). */
|
|
305
|
+
appendShadowLedger(file, entry) {
|
|
297
306
|
try {
|
|
298
|
-
const file = join(resolveBundleRoot(), 'meta', 'package-shadow.jsonl');
|
|
299
307
|
let lines = [];
|
|
300
308
|
try {
|
|
301
309
|
const st = statSync(file);
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
310
|
+
const all = readFileSync(file, 'utf8').split('\n').filter((l) => l !== '');
|
|
311
|
+
// Append, never clobber: existing lines are always carried forward;
|
|
312
|
+
// compaction (keep newest 3/4) only bites past the 512KB cap.
|
|
313
|
+
lines = st.size > 512 * 1024 ? all.slice(Math.floor(all.length / 4)) : all;
|
|
306
314
|
}
|
|
307
315
|
catch { /* first write */ }
|
|
308
316
|
lines.push(JSON.stringify(entry));
|
|
@@ -310,6 +318,12 @@ export class TopicsService {
|
|
|
310
318
|
}
|
|
311
319
|
catch { /* ledger must never break the lane */ }
|
|
312
320
|
}
|
|
321
|
+
packageShadowLedgerPath() {
|
|
322
|
+
return join(resolveBundleRoot(), 'meta', 'package-shadow.jsonl');
|
|
323
|
+
}
|
|
324
|
+
lessonShadowLedgerPath() {
|
|
325
|
+
return join(resolveBundleRoot(), 'meta', 'lesson-shadow.jsonl');
|
|
326
|
+
}
|
|
313
327
|
/** Boot-time embedding index maintenance (spec §3: boot 校验+增量重建).
|
|
314
328
|
* Provider '' or packageMode off → no-op (default-off byte-for-byte).
|
|
315
329
|
* Full-roster embed on rebuild (A1 §5: ~9.5s for 137 topics). */
|
|
@@ -427,7 +441,12 @@ export class TopicsService {
|
|
|
427
441
|
const cfg = this.cfg;
|
|
428
442
|
const roster = this.rosterSync();
|
|
429
443
|
const empty = { hits: [], nearMisses: [], rosterSize: roster.length };
|
|
430
|
-
|
|
444
|
+
// Lesson gate (B2): established lessons are INDEPENDENT of the topic
|
|
445
|
+
// roster — a lesson-only bundle still injects lessons. The roster read is
|
|
446
|
+
// shared with the matching block below; gate off → no read at all.
|
|
447
|
+
const lessonGate = cfg.lessonInject === 'shadow' || cfg.lessonInject === 'on' ? cfg.lessonInject : undefined;
|
|
448
|
+
const lessonRoster = lessonGate !== undefined ? this.lessonsSync() : [];
|
|
449
|
+
if (!cfg.autoInject || query.trim() === '' || (roster.length === 0 && lessonRoster.length === 0)) {
|
|
431
450
|
// A consumed/expired pending must always leave an ilog trace (ADR 0014:
|
|
432
451
|
// 过期/消费都写 ilog) — even when the fast path early-returns here
|
|
433
452
|
// because the roster vanished between production and consumption.
|
|
@@ -456,7 +475,7 @@ export class TopicsService {
|
|
|
456
475
|
}
|
|
457
476
|
void this.store.appendInjectionRecord(trace).catch(() => undefined);
|
|
458
477
|
}
|
|
459
|
-
return { text: '', outcome: empty, deduped: [], echoed: [], included: [], slowIncluded: [] };
|
|
478
|
+
return { text: '', outcome: empty, deduped: [], echoed: [], included: [], slowIncluded: [], lessonIncluded: [] };
|
|
460
479
|
}
|
|
461
480
|
// A-line gate (A3/A4): query cleaning + package + semantic rerank ride
|
|
462
481
|
// behind packageMode (default off = byte-for-byte legacy path).
|
|
@@ -527,6 +546,63 @@ export class TopicsService {
|
|
|
527
546
|
}
|
|
528
547
|
entries.push({ slug: hit.slug, doc, hit });
|
|
529
548
|
}
|
|
549
|
+
// ---- Lesson gate (B2, spec 卷一 §6.3): established lessons only (MJ-3:
|
|
550
|
+
// hint never injects), trigger-channel match = 条件命中 (triggers carry
|
|
551
|
+
// the condition), top-1 by score. No hit → no lesson (教训不撒背景网).
|
|
552
|
+
// dedup/echo follow the same rules as topics. ----
|
|
553
|
+
let lessonMatch;
|
|
554
|
+
if (lessonGate !== undefined && lessonRoster.length > 0) {
|
|
555
|
+
const qTokens = new Set(tokenize(effectiveQuery));
|
|
556
|
+
let best;
|
|
557
|
+
for (const l of lessonRoster) {
|
|
558
|
+
if (exclude?.has(l.slug)) {
|
|
559
|
+
if (!deduped.includes(l.slug))
|
|
560
|
+
deduped.push(l.slug);
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
if (echo?.has(l.slug)) {
|
|
564
|
+
if (!echoed.includes(l.slug))
|
|
565
|
+
echoed.push(l.slug);
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
const v = scoreTopic(qTokens, l, {
|
|
569
|
+
threshold: cfg.matchThreshold,
|
|
570
|
+
topK: cfg.topK,
|
|
571
|
+
tagBoost: cfg.tagBoost,
|
|
572
|
+
graphDepth: 0,
|
|
573
|
+
recencyWindowDays: cfg.recencyWindowDays,
|
|
574
|
+
conflicts,
|
|
575
|
+
structuralGate: true,
|
|
576
|
+
usageBoost: cfg.usageBoost,
|
|
577
|
+
usage: this.usageSignalsSync(),
|
|
578
|
+
...this.zeroOpenCfg(),
|
|
579
|
+
});
|
|
580
|
+
if (!v.reasons.some((r) => r.startsWith('triggers:')))
|
|
581
|
+
continue;
|
|
582
|
+
if (best === undefined || v.score > best.v.score)
|
|
583
|
+
best = { l, v };
|
|
584
|
+
}
|
|
585
|
+
if (best !== undefined) {
|
|
586
|
+
lessonMatch = {
|
|
587
|
+
pointer: {
|
|
588
|
+
slug: best.l.slug,
|
|
589
|
+
title: best.l.title,
|
|
590
|
+
status: best.l.status,
|
|
591
|
+
description: best.l.description,
|
|
592
|
+
score: best.v.score,
|
|
593
|
+
source: best.l.source,
|
|
594
|
+
},
|
|
595
|
+
reasons: best.v.reasons,
|
|
596
|
+
conclusion: best.l.conclusion,
|
|
597
|
+
};
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
// MJ-C combination matrix: lesson injection does not depend on the A3
|
|
601
|
+
// package gray-release. While the emitted shape is a flat list (digest
|
|
602
|
+
// mode, pointer mode without the package, AND the package-shadow legacy
|
|
603
|
+
// emit), lessons ride it as flat entries with the source marker; only a
|
|
604
|
+
// live package (packageMode on) gives them the direct-layer priority seat.
|
|
605
|
+
const lessonFlat = lessonGate === 'on' && lessonMatch !== undefined ? [lessonMatch.pointer] : undefined;
|
|
530
606
|
// ---- Slow-lane merge (log-only shadow gate; consumption bookkeeping) ----
|
|
531
607
|
const slowPointers = [];
|
|
532
608
|
const slowDelivered = [];
|
|
@@ -589,16 +665,16 @@ export class TopicsService {
|
|
|
589
665
|
let packageShadow;
|
|
590
666
|
let injection;
|
|
591
667
|
if (!pointerMode) {
|
|
592
|
-
injection = assembleInjection(entries, { perTopicBudget: cfg.perTopicBudget, totalBudget: cfg.totalBudget }, slowPointers);
|
|
668
|
+
injection = assembleInjection(entries, { perTopicBudget: cfg.perTopicBudget, totalBudget: cfg.totalBudget }, slowPointers, lessonFlat);
|
|
593
669
|
}
|
|
594
670
|
else if (pkgMode === undefined) {
|
|
595
|
-
injection = assemblePointer(entries, { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) }, slowPointers);
|
|
671
|
+
injection = assemblePointer(entries, { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) }, slowPointers, lessonFlat);
|
|
596
672
|
}
|
|
597
673
|
else {
|
|
598
674
|
// A-line relevance package (A3, spec §2): direct = lexical hits,
|
|
599
675
|
// neighborhood = graph hits + slow picks, weak = reranked band head.
|
|
600
676
|
const budget = { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) };
|
|
601
|
-
const legacy = assemblePointer(entries, budget, slowPointers);
|
|
677
|
+
const legacy = assemblePointer(entries, budget, slowPointers, lessonFlat);
|
|
602
678
|
const direct = [];
|
|
603
679
|
const neighborhood = [];
|
|
604
680
|
for (const e of entries) {
|
|
@@ -615,6 +691,23 @@ export class TopicsService {
|
|
|
615
691
|
};
|
|
616
692
|
(e.hit.viaGraph ? neighborhood : direct).push(cand);
|
|
617
693
|
}
|
|
694
|
+
// Lesson priority seat (B2, spec 卷一 §6.3): 主层 ≤2 席中命中 lesson
|
|
695
|
+
// 至少 1 席 — sorts first and is exempt from the direct-seat cap
|
|
696
|
+
// (budget-bound). Only on the LIVE package leg; the legacy emit above
|
|
697
|
+
// carries the flat lesson entry instead (MJ-C).
|
|
698
|
+
if (lessonFlat !== undefined && lessonMatch !== undefined) {
|
|
699
|
+
direct.unshift({
|
|
700
|
+
slug: lessonMatch.pointer.slug,
|
|
701
|
+
title: lessonMatch.pointer.title,
|
|
702
|
+
status: lessonMatch.pointer.status,
|
|
703
|
+
description: lessonMatch.pointer.description,
|
|
704
|
+
conclusion: firstContentLine(lessonMatch.conclusion),
|
|
705
|
+
score: lessonMatch.pointer.score,
|
|
706
|
+
reasons: lessonMatch.reasons,
|
|
707
|
+
prioritySeat: true,
|
|
708
|
+
sourceTag: lessonSourceTag(lessonMatch.pointer.source),
|
|
709
|
+
});
|
|
710
|
+
}
|
|
618
711
|
for (const sp of slowPointers) {
|
|
619
712
|
neighborhood.push({ slug: sp.slug, title: sp.title, status: sp.status, description: sp.description, score: 0, reasons: [], via: 'slow-pick' });
|
|
620
713
|
}
|
|
@@ -654,6 +747,13 @@ export class TopicsService {
|
|
|
654
747
|
if (slowIncluded.includes(pointer.slug))
|
|
655
748
|
slowDelivered.push({ slug: pointer.slug, why: pointer.why });
|
|
656
749
|
}
|
|
750
|
+
// ---- Lesson gate accounting (B2): what entered the context, the shadow
|
|
751
|
+
// bypass ledger (meta/lesson-shadow.jsonl — lesson seat vs no-lesson
|
|
752
|
+
// emitted disposition, same discipline as the package shadow), and the
|
|
753
|
+
// record fields (absent when the gate is off or nothing matched). ----
|
|
754
|
+
const lessonIncluded = lessonGate === 'on' && lessonMatch !== undefined
|
|
755
|
+
? injection.lessonIncluded ?? (injection.included.includes(lessonMatch.pointer.slug) ? [lessonMatch.pointer.slug] : [])
|
|
756
|
+
: [];
|
|
657
757
|
const record = {
|
|
658
758
|
at: new Date().toISOString(),
|
|
659
759
|
queryTokenCount: tokenize(effectiveQuery).length,
|
|
@@ -678,7 +778,7 @@ export class TopicsService {
|
|
|
678
778
|
record.dropped = [...(record.dropped ?? []), ...unreadable.map((slug) => ({ slug, reason: 'doc-unreadable' }))];
|
|
679
779
|
}
|
|
680
780
|
if (pkgMode === 'shadow' && pointerMode) {
|
|
681
|
-
this.
|
|
781
|
+
this.appendShadowLedger(this.packageShadowLedgerPath(), {
|
|
682
782
|
at: record.at,
|
|
683
783
|
...(sessionId !== undefined ? { sessionId } : {}),
|
|
684
784
|
queryTokenCount: record.queryTokenCount,
|
|
@@ -691,6 +791,37 @@ export class TopicsService {
|
|
|
691
791
|
nearFloorRerank: nearFloorRerank ?? null,
|
|
692
792
|
});
|
|
693
793
|
}
|
|
794
|
+
if (lessonGate === 'shadow' && lessonMatch !== undefined) {
|
|
795
|
+
const rendered = lessonEntryText(lessonMatch.pointer, POINTER_PER_TOPIC);
|
|
796
|
+
const budgetTotal = pointerMode ? Math.min(cfg.totalBudget, POINTER_TOTAL) : cfg.totalBudget;
|
|
797
|
+
this.appendShadowLedger(this.lessonShadowLedgerPath(), {
|
|
798
|
+
at: record.at,
|
|
799
|
+
...(sessionId !== undefined ? { sessionId } : {}),
|
|
800
|
+
queryTokenCount: record.queryTokenCount,
|
|
801
|
+
lessonInject: 'shadow',
|
|
802
|
+
lessons: [
|
|
803
|
+
{
|
|
804
|
+
slug: lessonMatch.pointer.slug,
|
|
805
|
+
score: lessonMatch.pointer.score,
|
|
806
|
+
source: lessonMatch.pointer.source,
|
|
807
|
+
entryTokens: rendered.usedTokens,
|
|
808
|
+
wouldFit: injection.usedTokens + rendered.usedTokens <= budgetTotal,
|
|
809
|
+
},
|
|
810
|
+
],
|
|
811
|
+
emitted: { tokens: injection.usedTokens, included: injection.included },
|
|
812
|
+
});
|
|
813
|
+
}
|
|
814
|
+
if (lessonGate !== undefined && lessonMatch !== undefined) {
|
|
815
|
+
record.lessonInject = lessonGate;
|
|
816
|
+
record.lessons = [
|
|
817
|
+
{
|
|
818
|
+
slug: lessonMatch.pointer.slug,
|
|
819
|
+
score: lessonMatch.pointer.score,
|
|
820
|
+
source: lessonMatch.pointer.source,
|
|
821
|
+
included: lessonIncluded.length > 0,
|
|
822
|
+
},
|
|
823
|
+
];
|
|
824
|
+
}
|
|
694
825
|
if (slow !== undefined) {
|
|
695
826
|
if (slow.outcome !== undefined)
|
|
696
827
|
record.slowOutcome = slow.outcome;
|
|
@@ -729,7 +860,7 @@ export class TopicsService {
|
|
|
729
860
|
if (injection.dropped.length > 0)
|
|
730
861
|
record.dropped = [...(record.dropped ?? []), ...injection.dropped];
|
|
731
862
|
void this.store.appendInjectionRecord(record).catch(() => undefined);
|
|
732
|
-
return { text: injection.text, outcome, deduped, echoed, included: injection.included, slowIncluded };
|
|
863
|
+
return { text: injection.text, outcome, deduped, echoed, included: injection.included, slowIncluded, lessonIncluded };
|
|
733
864
|
}
|
|
734
865
|
/**
|
|
735
866
|
* topic_open tool path (v4 §4.1): full conclusion / open questions /
|
|
@@ -829,6 +960,10 @@ export class TopicsService {
|
|
|
829
960
|
// Same retirement rule as roster(): deprecated knowledge never injects.
|
|
830
961
|
if (doc.fm.status === 'deprecated')
|
|
831
962
|
continue;
|
|
963
|
+
// Same lesson separation as roster() (B2): lessons inject only through
|
|
964
|
+
// the lessonInject gate, never the topic lane.
|
|
965
|
+
if (okf.topicKindOf(doc.fm.type) === 'lesson')
|
|
966
|
+
continue;
|
|
832
967
|
out.push({
|
|
833
968
|
slug: entry.slug,
|
|
834
969
|
title: doc.fm.title,
|
|
@@ -844,6 +979,76 @@ export class TopicsService {
|
|
|
844
979
|
}
|
|
845
980
|
return out;
|
|
846
981
|
}
|
|
982
|
+
/** Established, scope-passing lesson entries (B2, spec 卷一 §6.3) — sync,
|
|
983
|
+
* sharing the roster's mtime cache. Injection reads ONLY established
|
|
984
|
+
* lessons here (MJ-3: hint 仅供审批队列展示,永不注入); scope filter =
|
|
985
|
+
* workspace lessons must carry the current workspace id (MJ-2); scope=global
|
|
986
|
+
* and legacy entries without a workspaceId pass (fail-open — the filter's
|
|
987
|
+
* default is permissive, the confidence gate is the hard one). */
|
|
988
|
+
lessonsSync() {
|
|
989
|
+
const ws = currentWorkspaceId();
|
|
990
|
+
let files;
|
|
991
|
+
try {
|
|
992
|
+
files = readdirSync(this.store.topicsDir());
|
|
993
|
+
}
|
|
994
|
+
catch {
|
|
995
|
+
return [];
|
|
996
|
+
}
|
|
997
|
+
const out = [];
|
|
998
|
+
for (const file of files) {
|
|
999
|
+
if (!file.endsWith('.md') || file === 'index.md')
|
|
1000
|
+
continue;
|
|
1001
|
+
const path = join(this.store.topicsDir(), file);
|
|
1002
|
+
let st;
|
|
1003
|
+
try {
|
|
1004
|
+
st = statSync(path);
|
|
1005
|
+
}
|
|
1006
|
+
catch {
|
|
1007
|
+
continue;
|
|
1008
|
+
}
|
|
1009
|
+
const cached = this.cache.get(file);
|
|
1010
|
+
let entry = cached;
|
|
1011
|
+
if (cached === undefined || cached.mtimeMs !== st.mtimeMs || cached.size !== st.size) {
|
|
1012
|
+
try {
|
|
1013
|
+
const doc = okf.parseTopicDoc(readFileSync(path, 'utf8'));
|
|
1014
|
+
entry = { mtimeMs: st.mtimeMs, size: st.size, doc, slug: file.slice(0, -3) };
|
|
1015
|
+
this.cache.set(file, entry);
|
|
1016
|
+
}
|
|
1017
|
+
catch {
|
|
1018
|
+
this.cache.delete(file);
|
|
1019
|
+
continue;
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
if (entry === undefined)
|
|
1023
|
+
continue;
|
|
1024
|
+
const fm = entry.doc.fm;
|
|
1025
|
+
if (okf.topicKindOf(fm.type) !== 'lesson')
|
|
1026
|
+
continue;
|
|
1027
|
+
// Confidence 事实源:established ⇄ stable;deprecated 已被上面的通用
|
|
1028
|
+
// 退休规则挡住,这里再兜一层(hint↔draft 永不注入)。
|
|
1029
|
+
if (fm.confidence !== 'established' || fm.status === 'deprecated')
|
|
1030
|
+
continue;
|
|
1031
|
+
if (fm.scope === 'workspace' && fm.workspaceId !== undefined && fm.workspaceId !== ws)
|
|
1032
|
+
continue;
|
|
1033
|
+
out.push({
|
|
1034
|
+
slug: entry.slug,
|
|
1035
|
+
title: fm.title,
|
|
1036
|
+
description: fm.description,
|
|
1037
|
+
status: fm.status,
|
|
1038
|
+
tags: fm.tags,
|
|
1039
|
+
triggers: fm.triggers,
|
|
1040
|
+
depends: fm.depends,
|
|
1041
|
+
links: okf.bodyLinkSlugs(entry.doc.body),
|
|
1042
|
+
generatedAt: fm.generated.at,
|
|
1043
|
+
conclusion: okf.sectionOf(entry.doc.body, okf.CONCLUSION_HEADING) ?? '',
|
|
1044
|
+
confidence: fm.confidence,
|
|
1045
|
+
source: fm.source ?? 'manual',
|
|
1046
|
+
scope: fm.scope ?? 'workspace',
|
|
1047
|
+
...(fm.workspaceId !== undefined ? { workspaceId: fm.workspaceId } : {}),
|
|
1048
|
+
});
|
|
1049
|
+
}
|
|
1050
|
+
return out;
|
|
1051
|
+
}
|
|
847
1052
|
readDocSync(slug) {
|
|
848
1053
|
try {
|
|
849
1054
|
return okf.parseTopicDoc(readFileSync(join(this.store.topicsDir(), `${slug}.md`), 'utf8'));
|
|
@@ -860,7 +1065,11 @@ export class TopicsService {
|
|
|
860
1065
|
return undefined;
|
|
861
1066
|
}
|
|
862
1067
|
}
|
|
863
|
-
/** topic_save tool path: create or update with actor stamping.
|
|
1068
|
+
/** topic_save tool path: create or update with actor stamping. `kind:
|
|
1069
|
+
* 'lesson'` (B2 data plane) switches the OKF type to Lesson and stamps the
|
|
1070
|
+
* lesson frontmatter (scope/confidence/source/workspaceId) with the
|
|
1071
|
+
* confidence↔status derivation (hint↔draft, established↔stable —
|
|
1072
|
+
* confidence is the fact source). */
|
|
864
1073
|
async saveTopic(input) {
|
|
865
1074
|
const now = new Date().toISOString();
|
|
866
1075
|
const existing = input.slug !== undefined ? await this.store.readTopic(input.slug) : undefined;
|
|
@@ -876,9 +1085,10 @@ export class TopicsService {
|
|
|
876
1085
|
}
|
|
877
1086
|
const baseSlug = input.slug ?? okf.slugify(input.title);
|
|
878
1087
|
const slug = existing === undefined ? await this.store.uniqueSlug(baseSlug) : okf.slugify(input.slug ?? baseSlug);
|
|
1088
|
+
const isLesson = input.kind === 'lesson' || (existing !== undefined && okf.topicKindOf(existing.fm.type) === 'lesson');
|
|
879
1089
|
// store.saveTopic stamps generated {by, at} itself (write-through actor).
|
|
880
1090
|
const fm = {
|
|
881
|
-
type: '
|
|
1091
|
+
type: isLesson ? okf.kindTypeValue('lesson') : okf.kindTypeValue('topic'),
|
|
882
1092
|
title: input.title,
|
|
883
1093
|
tags: dedupeLower(input.tags ?? existing?.fm.tags ?? []),
|
|
884
1094
|
// normalizeDependsEntry, never bare slugToPath: the preserved-on-update
|
|
@@ -898,9 +1108,34 @@ export class TopicsService {
|
|
|
898
1108
|
const triggers = sanitizeTriggers(input.triggers ?? existing?.fm.triggers ?? []);
|
|
899
1109
|
if (triggers.length > 0)
|
|
900
1110
|
fm.triggers = triggers;
|
|
1111
|
+
if (isLesson) {
|
|
1112
|
+
// B2 frontmatter (spec 卷一 §6.2): preserve-on-update like tags; the
|
|
1113
|
+
// workspace id is stamped AT WRITE TIME (MJ-2, cwd hash) and survives
|
|
1114
|
+
// updates; confidence is the FACT source → status derives from it,
|
|
1115
|
+
// except a deprecated lesson stays deprecated unless confidence is
|
|
1116
|
+
// explicitly re-driven (no silent revival through topic_save).
|
|
1117
|
+
fm.scope = input.scope ?? existing?.fm.scope ?? 'workspace';
|
|
1118
|
+
fm.confidence = input.confidence ?? existing?.fm.confidence ?? 'hint';
|
|
1119
|
+
fm.source = input.lessonSource ?? existing?.fm.source ?? 'manual';
|
|
1120
|
+
if (fm.scope === 'workspace') {
|
|
1121
|
+
const wsid = existing?.fm.workspaceId ?? currentWorkspaceId();
|
|
1122
|
+
if (wsid !== undefined)
|
|
1123
|
+
fm.workspaceId = wsid;
|
|
1124
|
+
}
|
|
1125
|
+
else {
|
|
1126
|
+
delete fm.workspaceId;
|
|
1127
|
+
}
|
|
1128
|
+
fm.status =
|
|
1129
|
+
existing?.fm.status === 'deprecated' && input.confidence === undefined
|
|
1130
|
+
? 'deprecated'
|
|
1131
|
+
: okf.deriveLessonStatus(fm.confidence);
|
|
1132
|
+
}
|
|
901
1133
|
if (existing !== undefined && existing.fm.verified !== undefined)
|
|
902
1134
|
fm.verified = existing.fm.verified;
|
|
903
|
-
const result = await this.store.saveTopic({ slug, doc: { fm, body }, links: input.links }, {
|
|
1135
|
+
const result = await this.store.saveTopic({ slug, doc: { fm, body }, links: input.links }, {
|
|
1136
|
+
message: existing === undefined ? `topics(${isLesson ? 'lesson' : 'topic'}): create ${slug}` : `topics(${isLesson ? 'lesson' : 'topic'}): update ${slug}`,
|
|
1137
|
+
created: existing === undefined,
|
|
1138
|
+
});
|
|
904
1139
|
this.invalidate();
|
|
905
1140
|
void this.sync?.schedulePush();
|
|
906
1141
|
return { ...result, slug: result.slug };
|
|
@@ -908,6 +1143,127 @@ export class TopicsService {
|
|
|
908
1143
|
async observe(input) {
|
|
909
1144
|
return this.store.appendObservation({ kind: input.kind, source: input.source ?? 'model', text: input.text, sessionId: input.sessionId });
|
|
910
1145
|
}
|
|
1146
|
+
// -------------------------------------------------------------------------
|
|
1147
|
+
// Dual-write seams (卷二 §6.2 MJ-5) — the new repo's only write surface.
|
|
1148
|
+
// Both run behind the store lock / index recompute; both audit the caller
|
|
1149
|
+
// identity (opts.actor REQUIRED — a blank actor refuses the whole call and
|
|
1150
|
+
// lands a `seam-*` audit row) into meta/pipeline-events.jsonl.
|
|
1151
|
+
// -------------------------------------------------------------------------
|
|
1152
|
+
/**
|
|
1153
|
+
* ① 观察投喂: structured observations (B1 §2.1 whitelist, FEATURES ONLY —
|
|
1154
|
+
* 对话原文禁入, 红线⑤) appended to the observations stream with the `fed`
|
|
1155
|
+
* provenance marker and `kind: 'signal'`; each line carries `by: actor`.
|
|
1156
|
+
* The whitelist copy drops unknown caller fields, caps detail entries and
|
|
1157
|
+
* string sizes, and renders a sorted feature line as the observation text.
|
|
1158
|
+
* Duplicates (same type+session+detail in the recent tail) are counted, not
|
|
1159
|
+
* appended (重复观察由原仓去重). Fail-soft: one malformed entry is counted
|
|
1160
|
+
* as rejected, never sinks the batch.
|
|
1161
|
+
*/
|
|
1162
|
+
async feedObservations(obs, opts) {
|
|
1163
|
+
const actor = typeof opts?.actor === 'string' ? opts.actor.trim() : '';
|
|
1164
|
+
const total = Array.isArray(obs) ? obs.length : 0;
|
|
1165
|
+
if (actor === '') {
|
|
1166
|
+
await recordPipelineEvent(this.store.root, {
|
|
1167
|
+
at: new Date().toISOString(),
|
|
1168
|
+
kind: 'seam-observations-fed',
|
|
1169
|
+
detail: { actor: '<missing>', accepted: 0, duplicates: 0, rejected: total, refused: true },
|
|
1170
|
+
});
|
|
1171
|
+
return { accepted: 0, duplicates: 0, rejected: total, ids: [], error: 'unauthorized' };
|
|
1172
|
+
}
|
|
1173
|
+
const accepted = [];
|
|
1174
|
+
let duplicates = 0;
|
|
1175
|
+
let rejected = 0;
|
|
1176
|
+
// Dedup window: the recent observation tail — the signal layer may retry
|
|
1177
|
+
// on its own fail-open local queue (spec 卷二 §2.1) and identical feature
|
|
1178
|
+
// tuples must not double-distill.
|
|
1179
|
+
const recent = await this.store.allObservations(500);
|
|
1180
|
+
const seenKeys = new Set(recent.map((o) => observationKey(o.kind, o.source, o.sessionId, o.text)));
|
|
1181
|
+
for (const raw of obs) {
|
|
1182
|
+
const clean = sanitizeStructuredObservation(raw);
|
|
1183
|
+
if (clean === undefined) {
|
|
1184
|
+
rejected += 1;
|
|
1185
|
+
continue;
|
|
1186
|
+
}
|
|
1187
|
+
const text = renderFeatureLine(clean);
|
|
1188
|
+
const key = observationKey('signal', 'fed', clean.sessionId, text);
|
|
1189
|
+
if (seenKeys.has(key)) {
|
|
1190
|
+
duplicates += 1;
|
|
1191
|
+
continue;
|
|
1192
|
+
}
|
|
1193
|
+
seenKeys.add(key);
|
|
1194
|
+
const saved = await this.store.appendObservation({
|
|
1195
|
+
kind: 'signal',
|
|
1196
|
+
source: 'fed',
|
|
1197
|
+
text,
|
|
1198
|
+
sessionId: clean.sessionId,
|
|
1199
|
+
by: actor,
|
|
1200
|
+
});
|
|
1201
|
+
accepted.push(saved.id);
|
|
1202
|
+
}
|
|
1203
|
+
await recordPipelineEvent(this.store.root, {
|
|
1204
|
+
at: new Date().toISOString(),
|
|
1205
|
+
kind: 'seam-observations-fed',
|
|
1206
|
+
detail: { actor, accepted: accepted.length, duplicates, rejected },
|
|
1207
|
+
});
|
|
1208
|
+
return { accepted: accepted.length, duplicates, rejected, ids: accepted };
|
|
1209
|
+
}
|
|
1210
|
+
/**
|
|
1211
|
+
* ② lesson 状态变更 (B2/卷二 §6.2): hint→established / →deprecated /
|
|
1212
|
+
* →removed. established flips confidence (the FACT source) and derives
|
|
1213
|
+
* status (established↔stable); deprecated flips status only (the TTL sweep
|
|
1214
|
+
* owns the rest); removed = archive FIRST (byte-exact copy to
|
|
1215
|
+
* `archive/lessons/` inside the bundle, git-revertable — 红线④) then out of
|
|
1216
|
+
* the roster, one commit. Each call lands a `seam-lesson-status` audit row.
|
|
1217
|
+
*/
|
|
1218
|
+
async setLessonStatus(slug, to, opts) {
|
|
1219
|
+
const actor = typeof opts?.actor === 'string' ? opts.actor.trim() : '';
|
|
1220
|
+
const clean = okf.slugify(slug ?? '');
|
|
1221
|
+
const audit = async (ok, reason, extra) => {
|
|
1222
|
+
await recordPipelineEvent(this.store.root, {
|
|
1223
|
+
at: new Date().toISOString(),
|
|
1224
|
+
kind: 'seam-lesson-status',
|
|
1225
|
+
detail: { actor: actor === '' ? '<missing>' : actor, slug: clean, to, ok, ...(reason !== undefined ? { reason } : {}), ...(extra ?? {}) },
|
|
1226
|
+
});
|
|
1227
|
+
};
|
|
1228
|
+
if (actor === '') {
|
|
1229
|
+
await audit(false, 'unauthorized');
|
|
1230
|
+
return { ok: false, slug: clean, reason: 'unauthorized' };
|
|
1231
|
+
}
|
|
1232
|
+
if (to !== 'established' && to !== 'deprecated' && to !== 'removed') {
|
|
1233
|
+
await audit(false, 'invalid-state');
|
|
1234
|
+
return { ok: false, slug: clean, reason: 'invalid-state' };
|
|
1235
|
+
}
|
|
1236
|
+
const doc = await this.store.readTopic(clean).catch(() => undefined);
|
|
1237
|
+
if (doc === undefined) {
|
|
1238
|
+
await audit(false, 'not-found');
|
|
1239
|
+
return { ok: false, slug: clean, reason: 'not-found' };
|
|
1240
|
+
}
|
|
1241
|
+
if (okf.topicKindOf(doc.fm.type) !== 'lesson') {
|
|
1242
|
+
await audit(false, 'not-a-lesson');
|
|
1243
|
+
return { ok: false, slug: clean, reason: 'not-a-lesson' };
|
|
1244
|
+
}
|
|
1245
|
+
const from = doc.fm.confidence ?? 'hint';
|
|
1246
|
+
if (to === 'removed') {
|
|
1247
|
+
const r = await this.store.archiveLesson(clean, {
|
|
1248
|
+
message: `topics(lesson): remove ${clean} (archive-first, by ${actor})`,
|
|
1249
|
+
});
|
|
1250
|
+
if (r === undefined) {
|
|
1251
|
+
await audit(false, 'not-found');
|
|
1252
|
+
return { ok: false, slug: clean, reason: 'not-found' };
|
|
1253
|
+
}
|
|
1254
|
+
await audit(true, undefined, { archivedTo: r.archivedTo, committed: r.committed });
|
|
1255
|
+
this.invalidate();
|
|
1256
|
+
return { ok: true, slug: clean, from, archivedTo: r.archivedTo, committed: r.committed };
|
|
1257
|
+
}
|
|
1258
|
+
if (to === 'established')
|
|
1259
|
+
doc.fm.confidence = 'established';
|
|
1260
|
+
doc.fm.status = to === 'established' ? okf.deriveLessonStatus('established') : 'deprecated';
|
|
1261
|
+
const result = await this.store.saveTopic({ slug: clean, doc }, { message: `topics(lesson): ${to} ${clean} (by ${actor})`, actor });
|
|
1262
|
+
this.invalidate();
|
|
1263
|
+
void this.sync?.schedulePush();
|
|
1264
|
+
await audit(true, undefined, { committed: result.committed });
|
|
1265
|
+
return { ok: true, slug: clean, from, committed: result.committed };
|
|
1266
|
+
}
|
|
911
1267
|
async search(query, topK) {
|
|
912
1268
|
const cfg = this.cfg;
|
|
913
1269
|
return searchTopics(query, await this.roster(), {
|
|
@@ -953,6 +1309,66 @@ export class TopicsService {
|
|
|
953
1309
|
function inclusionDropped(injection) {
|
|
954
1310
|
return injection.dropped.length > 0;
|
|
955
1311
|
}
|
|
1312
|
+
// --- feedObservations whitelist machinery (B-line seam ①) -------------------
|
|
1313
|
+
// jev/log REDACTION + pipeline-events discipline: whitelist copy on write,
|
|
1314
|
+
// bounded scalar features only — 对话原文(tool arguments、message content)
|
|
1315
|
+
// 结构上进不来(红线⑤)。
|
|
1316
|
+
const SEAM_TYPE_MAX_CHARS = 64;
|
|
1317
|
+
const SEAM_SESSION_MAX_CHARS = 128;
|
|
1318
|
+
const SEAM_DETAIL_MAX_ENTRIES = 16;
|
|
1319
|
+
const SEAM_DETAIL_KEY_MAX_CHARS = 64;
|
|
1320
|
+
const SEAM_DETAIL_VALUE_MAX_CHARS = 256;
|
|
1321
|
+
/** Whitelist copy of one structured observation; undefined = reject. A raw
|
|
1322
|
+
* entry whose detail existed but sanitizes to EMPTY (object/array smuggle,
|
|
1323
|
+
* garbage values) is rejected outright — a type with no survivable features
|
|
1324
|
+
* is a malformed signal, not a featureless one. */
|
|
1325
|
+
function sanitizeStructuredObservation(raw) {
|
|
1326
|
+
if (raw === null || typeof raw !== 'object')
|
|
1327
|
+
return undefined;
|
|
1328
|
+
if (typeof raw.type !== 'string' || raw.type.trim() === '')
|
|
1329
|
+
return undefined;
|
|
1330
|
+
const out = { type: raw.type.trim().slice(0, SEAM_TYPE_MAX_CHARS) };
|
|
1331
|
+
if (typeof raw.at === 'string' && raw.at !== '')
|
|
1332
|
+
out.at = raw.at;
|
|
1333
|
+
if (typeof raw.sessionId === 'string' && raw.sessionId !== '')
|
|
1334
|
+
out.sessionId = raw.sessionId.slice(0, SEAM_SESSION_MAX_CHARS);
|
|
1335
|
+
const hadDetail = raw.detail !== undefined && typeof raw.detail === 'object' && !Array.isArray(raw.detail);
|
|
1336
|
+
if (hadDetail) {
|
|
1337
|
+
const detail = {};
|
|
1338
|
+
for (const [k, v] of Object.entries(raw.detail)) {
|
|
1339
|
+
if (Object.keys(detail).length >= SEAM_DETAIL_MAX_ENTRIES)
|
|
1340
|
+
break;
|
|
1341
|
+
if (typeof v === 'string') {
|
|
1342
|
+
if (v !== '')
|
|
1343
|
+
detail[k.slice(0, SEAM_DETAIL_KEY_MAX_CHARS)] = v.slice(0, SEAM_DETAIL_VALUE_MAX_CHARS);
|
|
1344
|
+
}
|
|
1345
|
+
else if (typeof v === 'number') {
|
|
1346
|
+
if (Number.isFinite(v))
|
|
1347
|
+
detail[k.slice(0, SEAM_DETAIL_KEY_MAX_CHARS)] = v;
|
|
1348
|
+
}
|
|
1349
|
+
else if (typeof v === 'boolean') {
|
|
1350
|
+
detail[k.slice(0, SEAM_DETAIL_KEY_MAX_CHARS)] = v;
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
if (Object.keys(detail).length === 0)
|
|
1354
|
+
return undefined;
|
|
1355
|
+
out.detail = detail;
|
|
1356
|
+
}
|
|
1357
|
+
return out;
|
|
1358
|
+
}
|
|
1359
|
+
/** Sorted feature line as the observation text: `[type] k=v k=v` — stable
|
|
1360
|
+
* (dedup-keyable), feature-only, bounded. */
|
|
1361
|
+
function renderFeatureLine(o) {
|
|
1362
|
+
const parts = Object.keys(o.detail ?? {})
|
|
1363
|
+
.sort()
|
|
1364
|
+
.map((k) => `${k}=${String(o.detail[k])}`);
|
|
1365
|
+
return `[${o.type}]${parts.length > 0 ? ` ${parts.join(' ')}` : ''}`.slice(0, 512);
|
|
1366
|
+
}
|
|
1367
|
+
/** Dedup key over the stored observation identity (kind/source/session/feature
|
|
1368
|
+
* line) — candidates key the same way they are stored. */
|
|
1369
|
+
function observationKey(kind, source, sessionId, text) {
|
|
1370
|
+
return `${kind}|${source}|${sessionId ?? ''}|${text}`;
|
|
1371
|
+
}
|
|
956
1372
|
function firstLine(text) {
|
|
957
1373
|
return firstParagraphOf(text);
|
|
958
1374
|
}
|
package/lib/store.d.ts
CHANGED
|
@@ -19,18 +19,31 @@ export interface TopicMeta {
|
|
|
19
19
|
depends: string[];
|
|
20
20
|
generatedAt: string;
|
|
21
21
|
}
|
|
22
|
-
|
|
22
|
+
/** Bundle entry kinds (B2, spec §6.5 落点表: store kind 枚举). Canonical
|
|
23
|
+
* model lives in okf.ts next to TopicStatus; re-exported here as the store
|
|
24
|
+
* surface the spec anchors on. */
|
|
25
|
+
export type TopicKind = okf.TopicKind;
|
|
26
|
+
export declare const topicKindOf: typeof okf.topicKindOf;
|
|
27
|
+
export type ObservationKind = 'decision' | 'finding' | 'constraint' | 'question' | 'turn' | 'signal';
|
|
28
|
+
/**
|
|
29
|
+
* Where an observation came from. `fed` marks entries that arrived through
|
|
30
|
+
* the feedObservations write seam (B-line dual-write, spec 卷二 §6.2) — the
|
|
31
|
+
* provenance marker the distill lane and audits key on.
|
|
32
|
+
*/
|
|
33
|
+
export type ObservationSource = 'model' | 'auto' | 'fed';
|
|
23
34
|
export interface Observation {
|
|
24
35
|
id: string;
|
|
25
36
|
at: string;
|
|
26
37
|
kind: ObservationKind;
|
|
27
|
-
source:
|
|
38
|
+
source: ObservationSource;
|
|
28
39
|
text: string;
|
|
29
40
|
sessionId?: string;
|
|
30
41
|
distilled: boolean;
|
|
31
42
|
distilledInto?: string[];
|
|
32
43
|
/** Failed distill attempts: fed to the model but consumed by no op (GC counter). */
|
|
33
44
|
attempts?: number;
|
|
45
|
+
/** Write-seam audit: the feedObservations caller identity (opts.actor). */
|
|
46
|
+
by?: string;
|
|
34
47
|
}
|
|
35
48
|
export interface SaveResult {
|
|
36
49
|
slug: string;
|
|
@@ -135,6 +148,19 @@ export declare class BundleStore {
|
|
|
135
148
|
* safe write path. Returns false when the slug doesn't exist (idempotent).
|
|
136
149
|
*/
|
|
137
150
|
deleteTopic(slug: string, message: string): Promise<boolean>;
|
|
151
|
+
/**
|
|
152
|
+
* Lesson removal seam (B2/卷二 §6.2 MJ-B, spec 红线④): archive BEFORE the
|
|
153
|
+
* roster removal — the file is copied byte-exact to `archive/lessons/`
|
|
154
|
+
* inside the bundle and only then deleted from `topics/`, both landing in
|
|
155
|
+
* ONE git commit so `git revert` restores the lesson exactly (自动写资产
|
|
156
|
+
* 必须可回滚). Returns undefined when the slug doesn't exist (idempotent).
|
|
157
|
+
*/
|
|
158
|
+
archiveLesson(slug: string, opts: {
|
|
159
|
+
message: string;
|
|
160
|
+
}): Promise<{
|
|
161
|
+
archivedTo: string;
|
|
162
|
+
committed: boolean;
|
|
163
|
+
} | undefined>;
|
|
138
164
|
private regenerateIndex;
|
|
139
165
|
/**
|
|
140
166
|
* Reverse-reference index: slug → who references it and how (`depends`
|
|
@@ -149,9 +175,10 @@ export declare class BundleStore {
|
|
|
149
175
|
}[]>>;
|
|
150
176
|
appendObservation(input: {
|
|
151
177
|
kind: ObservationKind;
|
|
152
|
-
source:
|
|
178
|
+
source: ObservationSource;
|
|
153
179
|
text: string;
|
|
154
180
|
sessionId?: string;
|
|
181
|
+
by?: string;
|
|
155
182
|
}): Promise<Observation>;
|
|
156
183
|
allObservations(limit?: number): Promise<Observation[]>;
|
|
157
184
|
undistilledObservations(limit?: number): Promise<Observation[]>;
|