@theseus-whale/dsh-continuity 0.0.0-stage → 4.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTALL.md +25 -0
- package/PANEL.md +92 -0
- package/README.md +286 -2
- package/RELEASE-NOTES.md +732 -0
- package/bin/bc-acceptance-full.mjs +91 -0
- package/bin/bc-acceptance-tight.mjs +66 -0
- package/bin/bc-acceptance.mjs +22 -0
- package/bin/bc-add-keywords.mjs +21 -0
- package/bin/bc-demo.mjs +47 -0
- package/bin/bc-expand-hint.mjs +179 -0
- package/bin/bc-ingest-index.mjs +35 -0
- package/bin/bc-ingest.mjs +31 -0
- package/bin/bc-label-demo.mjs +18 -0
- package/bin/bc-relabel.mjs +150 -0
- package/bin/bc-sessionize.mjs +59 -0
- package/bin/coref_pipeline.py +47 -0
- package/bin/coref_service.py +57 -0
- package/bin/debug_ckip.py +16 -0
- package/bin/debug_ner_zh.py +18 -0
- package/bin/doctor.mjs +275 -0
- package/bin/en_ner_error_audit.py +33 -0
- package/bin/eval-label-rules.mjs +342 -0
- package/bin/export-turns.py +68 -0
- package/bin/extract-user-paragraph.mjs +14 -0
- package/bin/extraction-composite-smoke.mjs +28 -0
- package/bin/extraction-latency.mjs +58 -0
- package/bin/extraction-user-smoke.mjs +28 -0
- package/bin/extraction_service.py +164 -0
- package/bin/facet-active-experiment.mjs +110 -0
- package/bin/facet-covid.mjs +8 -0
- package/bin/facet-curve.mjs +27 -0
- package/bin/facet-embedding-train.mjs +20 -0
- package/bin/facet-embedding-validate.mjs +83 -0
- package/bin/facet-eval-roleplay.mjs +42 -0
- package/bin/facet-eval.mjs +38 -0
- package/bin/facet-fold.mjs +31 -0
- package/bin/facet-load.mjs +16 -0
- package/bin/facet-mlp-scan.mjs +45 -0
- package/bin/facet-mvp.mjs +34 -0
- package/bin/facet-online-experiments.mjs +78 -0
- package/bin/facet-online-race.mjs +99 -0
- package/bin/facet-online-validate.mjs +51 -0
- package/bin/facet-panel.mjs +22 -0
- package/bin/facet-passage.mjs +41 -0
- package/bin/facet-random-validate.mjs +48 -0
- package/bin/facet-roleplay.mjs +33 -0
- package/bin/facet-scan.mjs +56 -0
- package/bin/facet-segment.mjs +34 -0
- package/bin/facet-train.mjs +19 -0
- package/bin/facet-validate.mjs +103 -0
- package/bin/facet-zeroshot.mjs +14 -0
- package/bin/install-templates.mjs +72 -0
- package/bin/memory-bench.mjs +53 -0
- package/bin/memory-build-summary.mjs +43 -0
- package/bin/memory-build-vector.mjs +31 -0
- package/bin/memory-build.mjs +41 -0
- package/bin/memory-calibration.mjs +70 -0
- package/bin/memory-closed-loop.mjs +119 -0
- package/bin/memory-dry-run.mjs +70 -0
- package/bin/memory-eval.mjs +104 -0
- package/bin/memory-event-alias-test.mjs +89 -0
- package/bin/memory-extraction-bench.mjs +58 -0
- package/bin/memory-load-gate-replay.mjs +137 -0
- package/bin/memory-loop-test.mjs +90 -0
- package/bin/memory-pre-turn-harness.mjs +63 -0
- package/bin/memory-query.mjs +39 -0
- package/bin/memory-replay-hints.mjs +84 -0
- package/bin/memory-scale.mjs +58 -0
- package/bin/memory-scenario-test.mjs +146 -0
- package/bin/memory-user-loop.mjs +95 -0
- package/bin/ner_bench_tiny_zh.py +59 -0
- package/bin/ner_bench_zh.py +60 -0
- package/bin/ner_conll_en.py +139 -0
- package/bin/ner_direct_en.py +123 -0
- package/bin/ner_direct_zh.py +100 -0
- package/bin/ner_pipeline.py +208 -0
- package/bin/noise-selfcheck.mjs +50 -0
- package/bin/pack-integrity.mjs +47 -0
- package/bin/panel-check.mjs +57 -0
- package/bin/plugin-integrity.mjs +74 -0
- package/bin/preview-live.mjs +13 -0
- package/bin/public-install-check.mjs +222 -0
- package/bin/relabel-apply.mjs +272 -0
- package/bin/relabel-dry-run.mjs +381 -0
- package/bin/relik_pipeline.py +39 -0
- package/bin/relik_service.py +52 -0
- package/bin/srl_pipeline.py +55 -0
- package/bin/srl_service.py +68 -0
- package/bin/summary-index.mjs +151 -0
- package/bin/summary-query.mjs +79 -0
- package/bin/vec-check.mjs +27 -0
- package/bin/zh_ner_error_audit.py +36 -0
- package/bin/zh_ner_slot_eval.py +38 -0
- package/client.js +760 -0
- package/cordis.patch.yml +378 -0
- package/docs/API-NOTES.md +1743 -0
- package/docs/ARCHITECTURE.md +355 -0
- package/docs/CACHE.md +253 -0
- package/docs/CONDUCT.md +30 -0
- package/docs/CREATOR-HANDOFF.md +102 -0
- package/docs/EXPERIMENT.md +136 -0
- package/docs/FACET-MVP-REPORT.md +473 -0
- package/docs/GUIDE-0.4.66.md +195 -0
- package/docs/GUIDE.md +127 -0
- package/docs/MECHANISM.md +243 -0
- package/docs/NEW-INSTANCE-HANDOFF.md +219 -0
- package/docs/README.md +37 -0
- package/docs/SESSION-LAYOUT.md +151 -0
- package/docs/history-map.md +249 -0
- package/docs/relabel-migration-wiring.md +366 -0
- package/guides/00-READ-FIRST.md +206 -0
- package/guides/01-INSTALL.md +207 -0
- package/guides/02-TABLES.md +225 -0
- package/guides/03-AUDIT.md +201 -0
- package/guides/04-REBUILD.md +306 -0
- package/guides/05-FIRST-RUN.md +191 -0
- package/guides/06-EMBEDDING.md +146 -0
- package/guides/07-PTC-AND-ASSETS.md +221 -0
- package/icon.svg +7 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/models/README.md +13 -0
- package/models/Xenova/bge-small-zh-v1.5/config.json +31 -0
- package/models/Xenova/bge-small-zh-v1.5/onnx/model_quantized.onnx +0 -0
- package/models/Xenova/bge-small-zh-v1.5/tokenizer.json +21278 -0
- package/models/Xenova/bge-small-zh-v1.5/tokenizer_config.json +15 -0
- package/package.json +139 -4
- package/src/asset-tool.mjs +237 -0
- package/src/assets.mjs +244 -0
- package/src/backfill.mjs +74 -0
- package/src/backup.mjs +80 -0
- package/src/deadend.mjs +95 -0
- package/src/deleted-store.mjs +35 -0
- package/src/engine.mjs +3831 -0
- package/src/facets/classifier.mjs +146 -0
- package/src/facets/embedding-service.mjs +53 -0
- package/src/facets/embedding-worker.mjs +27 -0
- package/src/facets/embedding.mjs +163 -0
- package/src/facets/extract.mjs +80 -0
- package/src/facets/fold-plan.mjs +84 -0
- package/src/facets/hybrid.mjs +29 -0
- package/src/facets/index.mjs +113 -0
- package/src/facets/linear-probe.mjs +43 -0
- package/src/facets/load.mjs +37 -0
- package/src/facets/mlp-probe.mjs +54 -0
- package/src/facets/mlp-service.mjs +53 -0
- package/src/facets/mlp-worker.mjs +26 -0
- package/src/facets/online-knn.mjs +58 -0
- package/src/facets/online-models.mjs +227 -0
- package/src/facets/panel-state.mjs +23 -0
- package/src/facets/panel.mjs +73 -0
- package/src/facets/random.mjs +26 -0
- package/src/facets/scanner.mjs +35 -0
- package/src/facets/schema.mjs +75 -0
- package/src/facets/seed-examples.mjs +58 -0
- package/src/facets/sensitivity.mjs +98 -0
- package/src/facets/service.mjs +53 -0
- package/src/facets/store.mjs +39 -0
- package/src/facets/worker.mjs +23 -0
- package/src/folds-store.mjs +109 -0
- package/src/groups.mjs +167 -0
- package/src/handoff-tool.mjs +51 -0
- package/src/handoff.mjs +127 -0
- package/src/index.mjs +649 -0
- package/src/inject-policy.mjs +17 -0
- package/src/learning.mjs +88 -0
- package/src/memory/alias-proposal.mjs +40 -0
- package/src/memory/bc-audit.mjs +245 -0
- package/src/memory/bc-index-updater.mjs +404 -0
- package/src/memory/bc-index.mjs +48 -0
- package/src/memory/bc-label.mjs +220 -0
- package/src/memory/bc-panel-state.mjs +107 -0
- package/src/memory/bc-pre-step.mjs +463 -0
- package/src/memory/bc-recall.mjs +470 -0
- package/src/memory/bc-shift.mjs +123 -0
- package/src/memory/bc-store.mjs +41 -0
- package/src/memory/bm25.mjs +112 -0
- package/src/memory/calibration.mjs +54 -0
- package/src/memory/chatter.mjs +29 -0
- package/src/memory/cmap-file.mjs +107 -0
- package/src/memory/concepts-file.mjs +41 -0
- package/src/memory/coref-slots.mjs +40 -0
- package/src/memory/event-alias-ledger.mjs +87 -0
- package/src/memory/event-log.mjs +95 -0
- package/src/memory/event-model.mjs +132 -0
- package/src/memory/event-segmenter.mjs +36 -0
- package/src/memory/event-summary-tree.mjs +122 -0
- package/src/memory/extraction-adapter.mjs +76 -0
- package/src/memory/extraction-composite.mjs +75 -0
- package/src/memory/extraction-core.mjs +76 -0
- package/src/memory/extraction-pre-step.mjs +161 -0
- package/src/memory/extraction-provider.mjs +300 -0
- package/src/memory/extraction-score.mjs +36 -0
- package/src/memory/facet-stabilizer.mjs +59 -0
- package/src/memory/history-map.mjs +139 -0
- package/src/memory/language-router.mjs +30 -0
- package/src/memory/load-gate.mjs +161 -0
- package/src/memory/mechanical-note.mjs +104 -0
- package/src/memory/onboarding.mjs +163 -0
- package/src/memory/persona.mjs +78 -0
- package/src/memory/planner.mjs +249 -0
- package/src/memory/pre-turn.mjs +80 -0
- package/src/memory/query-aliases.mjs +39 -0
- package/src/memory/recall-cue.mjs +43 -0
- package/src/memory/recall.mjs +65 -0
- package/src/memory/referent-candidates.mjs +55 -0
- package/src/memory/referent-ledger.mjs +86 -0
- package/src/memory/relabel-migrate.mjs +376 -0
- package/src/memory/relik-adapter.mjs +25 -0
- package/src/memory/special-keywords.mjs +60 -0
- package/src/memory/structured-adapters.mjs +75 -0
- package/src/memory/summary-index.mjs +335 -0
- package/src/memory/summary-tree.mjs +110 -0
- package/src/memory/term-extractor.mjs +44 -0
- package/src/memory/tokenize.mjs +9 -0
- package/src/memory/vector-index.mjs +54 -0
- package/src/memory/window-summary.mjs +133 -0
- package/src/mood.mjs +133 -0
- package/src/peer-tool.mjs +63 -0
- package/src/peer.mjs +144 -0
- package/src/policy.mjs +170 -0
- package/src/preflight.mjs +113 -0
- package/src/provenance.mjs +94 -0
- package/src/range.mjs +75 -0
- package/src/recall-exact-tool.mjs +267 -0
- package/src/recall-tool.mjs +521 -0
- package/src/recheck.mjs +99 -0
- package/src/save-start.mjs +29 -0
- package/src/session-guard.mjs +140 -0
- package/src/session-log-tool.mjs +322 -0
- package/src/session-log.mjs +606 -0
- package/src/skills.mjs +111 -0
- package/src/split-range.mjs +115 -0
- package/src/state.mjs +198 -0
- package/src/t0.mjs +149 -0
- package/src/tag-confirm.mjs +78 -0
- package/src/tag-detect.mjs +42 -0
- package/src/tag-maintenance.mjs +285 -0
- package/src/tag-reject.mjs +73 -0
- package/src/tags-store.mjs +94 -0
- package/src/tags.mjs +692 -0
- package/src/threads-backend.mjs +26 -0
- package/src/threads-core.mjs +1887 -0
- package/src/threads-store.mjs +207 -0
- package/src/threads-tool.mjs +522 -0
- package/src/threads.mjs +929 -0
- package/src/tool-preview.mjs +427 -0
- package/src/topic.mjs +63 -0
- package/src/training.mjs +75 -0
- package/src/turn-tag.mjs +62 -0
- package/src/user-turn.mjs +39 -0
- package/templates/cmap.md +11 -0
- package/templates/concepts.md +12 -0
- package/templates/habits.md +40 -0
- package/templates/keywords.md +6 -0
- package/templates/rebuild.md +134 -0
- package/templates/rules-safe.md +184 -0
- package/templates/rules.md +177 -0
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Incremental B-index updater.
|
|
3
|
+
*
|
|
4
|
+
* One pointer record per completed turn. The raw text stays in the session log;
|
|
5
|
+
* this module only writes generic concepts, scores, keywords, and seq pointers.
|
|
6
|
+
*/
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import { createHash } from 'node:crypto';
|
|
10
|
+
import { embedTexts } from '../facets/embedding.mjs';
|
|
11
|
+
import { createBLabeler, LABELER_VERSION } from './bc-label.mjs';
|
|
12
|
+
import { extractKeywords, makeRecord } from './bc-recall.mjs';
|
|
13
|
+
import { loadIndexJsonl, saveIndexJsonl } from './bc-index.mjs';
|
|
14
|
+
import { loadSpecialKeywords, matchSpecialKeywords } from './special-keywords.mjs';
|
|
15
|
+
import { loadCMapEntries } from './cmap-file.mjs';
|
|
16
|
+
import { buildDf, chatterScore } from './chatter.mjs';
|
|
17
|
+
import { eventText } from '../session-log.mjs';
|
|
18
|
+
|
|
19
|
+
const INJECTED = /^\[(?:抽取|工作线|标签|规则|习惯|技能|归位|候选|存\?|闸|回正|学习|追忆|记忆|continuity:[a-z-]+)\]/;
|
|
20
|
+
const MACHINE = /^(?:Current runtime context|Current working directory|Current DSH file policy|Approval prompts|Time sampled)/;
|
|
21
|
+
const BASELINE_WINDOW = 300;
|
|
22
|
+
// 打标阈值:既喂给 labeler,也参与 labelStamp 的计算(改这里等于改依据)
|
|
23
|
+
export const TOPIC_THRESHOLDS = Object.freeze({ topicMinScore: 0.33, centerMin: 0.02, lead: 0.02 });
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* labelStamp:这一行是"用哪套依据"标的。
|
|
27
|
+
* stamp = sha256(concepts.md 内容 | 阈值 | LABELER_VERSION)[:12]
|
|
28
|
+
* 表没变、阈值没变、判定逻辑没变 → stamp 不变 → 增量迁移时这行不需要重标。
|
|
29
|
+
*/
|
|
30
|
+
export function computeLabelStamp({ conceptsText = '', thresholds = TOPIC_THRESHOLDS, labelerVersion = LABELER_VERSION } = {}) {
|
|
31
|
+
return createHash('sha256')
|
|
32
|
+
.update(`${conceptsText}|${thresholds.topicMinScore}|${thresholds.centerMin}|${thresholds.lead}|${labelerVersion}`)
|
|
33
|
+
.digest('hex')
|
|
34
|
+
.slice(0, 12);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function emptyBaseline() {
|
|
38
|
+
return { version: 2, window: BASELINE_WINDOW, topics: {}, workflows: {} };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function loadBaseline(file) {
|
|
42
|
+
try {
|
|
43
|
+
const parsed = JSON.parse(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
|
|
44
|
+
if (parsed && parsed.version === 2 && parsed.topics && parsed.workflows) return parsed;
|
|
45
|
+
if (parsed && parsed.version === 1 && parsed.samples && typeof parsed.samples === 'object') {
|
|
46
|
+
return { version: 2, window: BASELINE_WINDOW, topics: parsed.samples, workflows: {} };
|
|
47
|
+
}
|
|
48
|
+
} catch { /* first run */ }
|
|
49
|
+
return emptyBaseline();
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function baselineMeans(state, group = 'topics') {
|
|
53
|
+
const means = {};
|
|
54
|
+
for (const [concept, samples] of Object.entries(state?.[group] ?? {})) {
|
|
55
|
+
if (!Array.isArray(samples) || samples.length === 0) continue;
|
|
56
|
+
means[concept] = samples.reduce((sum, value) => sum + Number(value || 0), 0) / samples.length;
|
|
57
|
+
}
|
|
58
|
+
return means;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function baselineGlobalMean(means) {
|
|
62
|
+
const values = Object.values(means ?? {}).filter((value) => Number.isFinite(value));
|
|
63
|
+
if (values.length === 0) return null;
|
|
64
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function updateBaseline(state, scores, group = 'topics') {
|
|
68
|
+
if (!scores || typeof scores !== 'object') return state;
|
|
69
|
+
if (!state[group] || typeof state[group] !== 'object') state[group] = {};
|
|
70
|
+
for (const [concept, score] of Object.entries(scores)) {
|
|
71
|
+
if (!Number.isFinite(Number(score))) continue;
|
|
72
|
+
const samples = Array.isArray(state[group][concept]) ? state[group][concept] : [];
|
|
73
|
+
samples.push(Number(score));
|
|
74
|
+
if (samples.length > BASELINE_WINDOW) samples.shift();
|
|
75
|
+
state[group][concept] = samples;
|
|
76
|
+
}
|
|
77
|
+
return state;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function saveBaseline(file, state) {
|
|
81
|
+
try {
|
|
82
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
83
|
+
const temp = `${file}.tmp-${process.pid}`;
|
|
84
|
+
fs.writeFileSync(temp, JSON.stringify(state), 'utf8');
|
|
85
|
+
fs.renameSync(temp, file);
|
|
86
|
+
} catch { /* baseline failure must not block indexing */ }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function isHuman(event) {
|
|
90
|
+
const source = event?.data?.source ?? event?.source;
|
|
91
|
+
if (source && typeof source.kind === 'string' && source.kind !== 'user') return false;
|
|
92
|
+
if (source && source.peer === true) return false;
|
|
93
|
+
const text = eventText(event, { includeReasoning: false }).trim();
|
|
94
|
+
return Boolean(text) && !INJECTED.test(text) && !MACHINE.test(text);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function turnsFromEvents(events, afterTurn, afterSeq = -1) {
|
|
98
|
+
const list = Array.isArray(events) ? events : [];
|
|
99
|
+
const markers = list
|
|
100
|
+
.map((event) => ({ seq: Number(event?.seq), turn: Number(event?.data?.turn) }))
|
|
101
|
+
.filter((item) => Number.isInteger(item.seq) && Number.isInteger(item.turn))
|
|
102
|
+
.sort((a, b) => a.seq - b.seq);
|
|
103
|
+
const turnAt = (seq) => {
|
|
104
|
+
let found = null;
|
|
105
|
+
for (const marker of markers) {
|
|
106
|
+
if (marker.seq > seq) break;
|
|
107
|
+
found = marker.turn;
|
|
108
|
+
}
|
|
109
|
+
if (found !== null) return found;
|
|
110
|
+
const next = markers.find((marker) => marker.seq >= seq);
|
|
111
|
+
return next?.turn ?? null;
|
|
112
|
+
};
|
|
113
|
+
const byTurn = new Map();
|
|
114
|
+
for (const event of list) {
|
|
115
|
+
const type = event?.type;
|
|
116
|
+
if (type !== 'user/message' && type !== 'assistant/message') continue;
|
|
117
|
+
if (type === 'user/message' && !isHuman(event)) continue;
|
|
118
|
+
const text = eventText(event, { includeReasoning: false }).trim();
|
|
119
|
+
if (!text || INJECTED.test(text) || MACHINE.test(text)) continue;
|
|
120
|
+
const seq = Number(event?.seq);
|
|
121
|
+
const turn = Number.isInteger(Number(event?.data?.turn))
|
|
122
|
+
? Number(event.data.turn)
|
|
123
|
+
: turnAt(seq);
|
|
124
|
+
if (!Number.isInteger(turn) || !Number.isInteger(seq)) continue;
|
|
125
|
+
if (turn <= afterTurn || seq <= afterSeq) continue;
|
|
126
|
+
const item = byTurn.get(turn) ?? {
|
|
127
|
+
turn,
|
|
128
|
+
seqStart: seq,
|
|
129
|
+
seqEnd: seq,
|
|
130
|
+
userTexts: [],
|
|
131
|
+
assistantTexts: [],
|
|
132
|
+
texts: [],
|
|
133
|
+
};
|
|
134
|
+
item.seqStart = Math.min(item.seqStart, seq);
|
|
135
|
+
item.seqEnd = Math.max(item.seqEnd, seq);
|
|
136
|
+
if (type === 'user/message') item.userTexts.push(text);
|
|
137
|
+
else item.assistantTexts.push(text);
|
|
138
|
+
byTurn.set(turn, item);
|
|
139
|
+
}
|
|
140
|
+
return [...byTurn.values()]
|
|
141
|
+
.map((item) => ({ ...item, texts: [...item.userTexts, ...item.assistantTexts] }))
|
|
142
|
+
.sort((a, b) => a.turn - b.turn);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function createIndexUpdater({
|
|
146
|
+
file,
|
|
147
|
+
stateFile = null,
|
|
148
|
+
baselineFile = null,
|
|
149
|
+
sessionQuery = null,
|
|
150
|
+
labeler = null,
|
|
151
|
+
labels = null,
|
|
152
|
+
kinds = null,
|
|
153
|
+
keywordsFile = null,
|
|
154
|
+
cmapFile = null,
|
|
155
|
+
conceptsFile = null,
|
|
156
|
+
gate = null,
|
|
157
|
+
maxPerRun = 8,
|
|
158
|
+
onError = null,
|
|
159
|
+
onCost = null,
|
|
160
|
+
} = {}) {
|
|
161
|
+
let activeLabeler = labeler;
|
|
162
|
+
let running = false;
|
|
163
|
+
let warnedMissingCmap = false;
|
|
164
|
+
// 保存的是 promise 而不是 bool:bool 在"刚结束但尚未置回"的窗口里不可靠
|
|
165
|
+
let runningPromise = null;
|
|
166
|
+
// gate 命中时被挡下的行留在这里,下一轮先合并再写(不丢数据)
|
|
167
|
+
const pendingBySession = new Map();
|
|
168
|
+
|
|
169
|
+
const readState = () => {
|
|
170
|
+
if (stateFile === null) return {};
|
|
171
|
+
try { return JSON.parse(fs.readFileSync(stateFile, 'utf8').replace(/^\uFEFF/, '')); } catch { return {}; }
|
|
172
|
+
};
|
|
173
|
+
const writeState = (value) => {
|
|
174
|
+
if (stateFile === null) return;
|
|
175
|
+
try {
|
|
176
|
+
fs.mkdirSync(path.dirname(stateFile), { recursive: true });
|
|
177
|
+
fs.writeFileSync(stateFile, JSON.stringify(value, null, 2), 'utf8');
|
|
178
|
+
} catch { /* state failure must not block indexing */ }
|
|
179
|
+
};
|
|
180
|
+
const takePending = (sessionId) => {
|
|
181
|
+
const rows = pendingBySession.get(sessionId) ?? [];
|
|
182
|
+
pendingBySession.delete(sessionId);
|
|
183
|
+
return rows;
|
|
184
|
+
};
|
|
185
|
+
const pushPending = (sessionId, rows) => {
|
|
186
|
+
if (!rows || rows.length === 0) return;
|
|
187
|
+
const existing = pendingBySession.get(sessionId) ?? [];
|
|
188
|
+
const byTurn = new Map(existing.map((row) => [row.turn, row]));
|
|
189
|
+
for (const row of rows) byTurn.set(row.turn, row);
|
|
190
|
+
pendingBySession.set(sessionId, [...byTurn.values()]);
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
const runOnce = async ({ agent = null, lastTurn = null } = {}) => {
|
|
194
|
+
const started = performance.now();
|
|
195
|
+
try {
|
|
196
|
+
const sessionId = agent?.session?.id;
|
|
197
|
+
if (sessionId === undefined || sessionId === null) return { added: 0, skipped: 'no-session-id' };
|
|
198
|
+
// labelStamp:本 session 的 concepts.md 内容 + 阈值 + LABELER_VERSION。
|
|
199
|
+
// 拿不到 concepts.md 就不写 stamp(宁缺勿假,后续迁移会补)。
|
|
200
|
+
const conceptsText = conceptsFile && fs.existsSync(conceptsFile)
|
|
201
|
+
? fs.readFileSync(conceptsFile, 'utf8')
|
|
202
|
+
: '';
|
|
203
|
+
const labelStamp = conceptsText ? computeLabelStamp({ conceptsText }) : null;
|
|
204
|
+
const records = loadIndexJsonl(file);
|
|
205
|
+
const maxTurn = records.reduce((max, record) => Number.isInteger(record.turn) ? Math.max(max, record.turn) : max, -1);
|
|
206
|
+
const maxSeq = records.reduce((max, record) => Number.isInteger(record.seqEnd) ? Math.max(max, record.seqEnd) : max, -1);
|
|
207
|
+
const state = readState();
|
|
208
|
+
const afterSeq = Number.isInteger(state.lastSeq) ? Math.max(maxSeq, state.lastSeq) : maxSeq;
|
|
209
|
+
|
|
210
|
+
let readMs = 0;
|
|
211
|
+
const readStarted = performance.now();
|
|
212
|
+
let events;
|
|
213
|
+
let mode = 'full';
|
|
214
|
+
if (typeof sessionQuery.readSessionTail === 'function') {
|
|
215
|
+
const snapshot = await sessionQuery.readSessionTail(sessionId, { afterSeq, limit: 512 });
|
|
216
|
+
events = snapshot?.events ?? [];
|
|
217
|
+
mode = 'tail';
|
|
218
|
+
} else {
|
|
219
|
+
const snapshot = await sessionQuery.readSession(sessionId, { afterSeq, limit: 512 });
|
|
220
|
+
events = snapshot?.events ?? [];
|
|
221
|
+
mode = events.length > 0 && events.every((event) => Number(event?.seq) > afterSeq) ? 'tail' : 'full';
|
|
222
|
+
}
|
|
223
|
+
readMs = Math.round(performance.now() - readStarted);
|
|
224
|
+
|
|
225
|
+
const missing = turnsFromEvents(events, maxTurn, afterSeq)
|
|
226
|
+
.filter((entry) => !Number.isInteger(lastTurn) || entry.turn < lastTurn);
|
|
227
|
+
const turns = missing.slice(0, maxPerRun);
|
|
228
|
+
const remaining = Math.max(0, missing.length - turns.length);
|
|
229
|
+
if (turns.length === 0) return { added: 0, maxTurn, remaining: 0, mode, readMs, totalMs: Math.round(performance.now() - started) };
|
|
230
|
+
|
|
231
|
+
const resolvedBaselineFile = baselineFile ?? path.join(path.dirname(file), 'bc-topic-baseline.json');
|
|
232
|
+
const baseline = loadBaseline(resolvedBaselineFile);
|
|
233
|
+
const topicMeans = baselineMeans(baseline, 'topics');
|
|
234
|
+
const topicFallback = baselineGlobalMean(topicMeans);
|
|
235
|
+
const workflowMeans = baselineMeans(baseline, 'workflows');
|
|
236
|
+
const workflowFallback = baselineGlobalMean(workflowMeans);
|
|
237
|
+
|
|
238
|
+
if (activeLabeler === null) activeLabeler = createBLabeler({
|
|
239
|
+
embed: embedTexts,
|
|
240
|
+
...(labels ? { labels, concepts: Object.keys(labels) } : {}),
|
|
241
|
+
...(kinds ? { kinds } : {}),
|
|
242
|
+
});
|
|
243
|
+
const specialKeywords = keywordsFile && !fs.existsSync(keywordsFile)
|
|
244
|
+
? []
|
|
245
|
+
: loadSpecialKeywords(keywordsFile ?? undefined);
|
|
246
|
+
const cmapTerms = cmapFile === null
|
|
247
|
+
? []
|
|
248
|
+
: loadCMapEntries(cmapFile)
|
|
249
|
+
.map((entry) => entry.term)
|
|
250
|
+
.filter((term) => typeof term === 'string' && term.length > 0);
|
|
251
|
+
if (cmapFile === null && !warnedMissingCmap) {
|
|
252
|
+
warnedMissingCmap = true;
|
|
253
|
+
const error = new Error('cmapFile not provided; C terms skipped');
|
|
254
|
+
error.code = 'cmap-file-missing';
|
|
255
|
+
onError?.(error);
|
|
256
|
+
}
|
|
257
|
+
const df = buildDf([
|
|
258
|
+
...records.map((record) => (record.keywords ?? []).join(' ')),
|
|
259
|
+
...turns.map((entry) => entry.texts.join('\n')),
|
|
260
|
+
]);
|
|
261
|
+
const added = [];
|
|
262
|
+
const labelStarted = performance.now();
|
|
263
|
+
for (const entry of turns) {
|
|
264
|
+
const text = entry.texts.join('\n');
|
|
265
|
+
const labelOptions = {
|
|
266
|
+
topK: 3,
|
|
267
|
+
minScore: 0.25,
|
|
268
|
+
topicMinScore: 0.33,
|
|
269
|
+
topicCenterMin: 0.02,
|
|
270
|
+
topicLead: 0.02,
|
|
271
|
+
topicTopK: 1,
|
|
272
|
+
workflowTopK: 2,
|
|
273
|
+
workflowCenterMin: 0.02,
|
|
274
|
+
workflowLead: 0.02,
|
|
275
|
+
topicBaseline: topicMeans,
|
|
276
|
+
topicBaselineFallback: topicFallback,
|
|
277
|
+
workflowBaseline: workflowMeans,
|
|
278
|
+
workflowBaselineFallback: workflowFallback,
|
|
279
|
+
chunkChars: 600,
|
|
280
|
+
maxChunks: 12,
|
|
281
|
+
};
|
|
282
|
+
const label = typeof activeLabeler.labelChunked === 'function'
|
|
283
|
+
? await activeLabeler.labelChunked(text, labelOptions)
|
|
284
|
+
: await activeLabeler.label(text, labelOptions);
|
|
285
|
+
updateBaseline(baseline, label.topicScores, 'topics');
|
|
286
|
+
updateBaseline(baseline, label.workflowScores, 'workflows');
|
|
287
|
+
const isV2 = label.version === 2;
|
|
288
|
+
const generic = label.candidates.map((candidate) => candidate.concept);
|
|
289
|
+
const conceptScores = Object.fromEntries(
|
|
290
|
+
(label.scores ? Object.entries(label.scores) : label.candidates.map((candidate) => [candidate.concept, candidate.score])),
|
|
291
|
+
);
|
|
292
|
+
const genericTopic = isV2 ? (label.topic ?? []).map((candidate) => candidate.concept) : undefined;
|
|
293
|
+
const genericWorkflow = isV2 ? (label.workflow ?? []).map((candidate) => candidate.concept) : undefined;
|
|
294
|
+
const keywords = [...new Set([
|
|
295
|
+
...extractKeywords(text, { limit: 12 }),
|
|
296
|
+
...matchSpecialKeywords(text, specialKeywords),
|
|
297
|
+
])];
|
|
298
|
+
const terms = [...new Set(cmapTerms.filter((term) => text.includes(term) || keywords.includes(term)))];
|
|
299
|
+
added.push(makeRecord({
|
|
300
|
+
turn: entry.turn,
|
|
301
|
+
seqStart: entry.seqStart,
|
|
302
|
+
seqEnd: entry.seqEnd,
|
|
303
|
+
text,
|
|
304
|
+
generic,
|
|
305
|
+
...(isV2 ? {
|
|
306
|
+
genericTopic,
|
|
307
|
+
genericWorkflow,
|
|
308
|
+
topicScore: label.topicScore,
|
|
309
|
+
topicConfidence: label.topicConfidence,
|
|
310
|
+
topicMinScore: label.topicMinScore,
|
|
311
|
+
} : {}),
|
|
312
|
+
labelVersion: isV2 ? 2 : 1,
|
|
313
|
+
...(labelStamp ? { labelStamp } : {}),
|
|
314
|
+
terms,
|
|
315
|
+
keywords,
|
|
316
|
+
conceptScores,
|
|
317
|
+
confidence: label.top?.score ?? 0,
|
|
318
|
+
status: 'weak',
|
|
319
|
+
...(Array.isArray(label.vector) ? { vec: Buffer.from(Float32Array.from(label.vector).buffer).toString('base64') } : {}),
|
|
320
|
+
chatter: chatterScore(text, df),
|
|
321
|
+
sessionId,
|
|
322
|
+
}));
|
|
323
|
+
}
|
|
324
|
+
const labelMs = Math.round(performance.now() - labelStarted);
|
|
325
|
+
// 先把上一轮被 gate 挡下的行并回来(按 turn 去重),再决定这次能不能写
|
|
326
|
+
const carried = takePending(sessionId);
|
|
327
|
+
if (carried.length > 0) {
|
|
328
|
+
const byTurn = new Map(carried.map((row) => [row.turn, row]));
|
|
329
|
+
for (const row of added) byTurn.set(row.turn, row);
|
|
330
|
+
added.splice(0, added.length, ...[...byTurn.values()].sort((a, b) => Number(a.turn) - Number(b.turn)));
|
|
331
|
+
}
|
|
332
|
+
// 互斥 B:正在迁移期间不写盘,且**不推进 state / 不写 baseline**
|
|
333
|
+
// 被挡的行进 pendingBySession,下一轮先合并再写——否则这些轮次会永久丢失
|
|
334
|
+
if (gate && typeof gate.isBlocked === 'function' && gate.isBlocked(sessionId)) {
|
|
335
|
+
pushPending(sessionId, added);
|
|
336
|
+
return {
|
|
337
|
+
added: 0,
|
|
338
|
+
skipped: 'relabel',
|
|
339
|
+
blocked: true,
|
|
340
|
+
pending: (pendingBySession.get(sessionId) ?? []).length,
|
|
341
|
+
maxTurn,
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
const writeStarted = performance.now();
|
|
345
|
+
saveIndexJsonl(file, [...records, ...added]);
|
|
346
|
+
saveBaseline(resolvedBaselineFile, baseline);
|
|
347
|
+
const writeMs = Math.round(performance.now() - writeStarted);
|
|
348
|
+
const lastSeq = added.at(-1)?.seqEnd ?? afterSeq;
|
|
349
|
+
const lastTurnWritten = added.at(-1)?.turn ?? maxTurn;
|
|
350
|
+
writeState({ at: new Date().toISOString(), lastSeq, lastTurn: lastTurnWritten, maxTurn: lastTurnWritten });
|
|
351
|
+
const cost = {
|
|
352
|
+
at: new Date().toISOString(),
|
|
353
|
+
added: added.length,
|
|
354
|
+
ms: Math.round(performance.now() - started),
|
|
355
|
+
readMs,
|
|
356
|
+
labelMs,
|
|
357
|
+
writeMs,
|
|
358
|
+
totalMs: Math.round(performance.now() - started),
|
|
359
|
+
maxTurn: lastTurnWritten,
|
|
360
|
+
maxSeq: lastSeq,
|
|
361
|
+
remaining,
|
|
362
|
+
mode,
|
|
363
|
+
};
|
|
364
|
+
try { onCost?.(cost); } catch { /* ignore */ }
|
|
365
|
+
return { ...cost, records: added };
|
|
366
|
+
} catch (error) {
|
|
367
|
+
try {
|
|
368
|
+
onError?.(error);
|
|
369
|
+
} catch { /* ignore */ }
|
|
370
|
+
return { added: 0, error: String(error?.message ?? error) };
|
|
371
|
+
}
|
|
372
|
+
};
|
|
373
|
+
|
|
374
|
+
const updateIndex = async (payload = {}) => {
|
|
375
|
+
if (!sessionQuery || typeof sessionQuery.readSession !== 'function') return { added: 0, skipped: 'no-session' };
|
|
376
|
+
if (runningPromise) return { added: 0, skipped: 'busy' };
|
|
377
|
+
running = true;
|
|
378
|
+
runningPromise = runOnce(payload).finally(() => {
|
|
379
|
+
running = false;
|
|
380
|
+
runningPromise = null;
|
|
381
|
+
});
|
|
382
|
+
return runningPromise;
|
|
383
|
+
};
|
|
384
|
+
|
|
385
|
+
/** 等当前这一次写盘结束(迁移开始前必须 await)。 */
|
|
386
|
+
updateIndex.whenIdle = () => runningPromise ?? Promise.resolve();
|
|
387
|
+
/** 是否正有一次写盘在跑。 */
|
|
388
|
+
updateIndex.isRunning = () => runningPromise !== null;
|
|
389
|
+
/** 被 gate 挡下的行数(观测用)。 */
|
|
390
|
+
updateIndex.pendingCount = (sessionId) => (pendingBySession.get(sessionId) ?? []).length;
|
|
391
|
+
|
|
392
|
+
return updateIndex;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
export function defaultCostLogPath(home = process.env.USERPROFILE ?? process.env.HOME ?? '') {
|
|
396
|
+
return path.join(home, '.dsh', 'continuity', 'bc-index-cost.jsonl');
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
export function appendCostLog(file, cost) {
|
|
400
|
+
try {
|
|
401
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
402
|
+
fs.appendFileSync(file, JSON.stringify(cost) + '\n', 'utf8');
|
|
403
|
+
} catch { /* cost log must not break indexing */ }
|
|
404
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compact pointer index: one JSONL line per turn, no raw text.
|
|
3
|
+
* Raw text stays in the session log; the pointer is turn + seq range.
|
|
4
|
+
*/
|
|
5
|
+
import fs from 'node:fs';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
|
|
8
|
+
export function toPointer(record) {
|
|
9
|
+
return {
|
|
10
|
+
turn: record.turn,
|
|
11
|
+
seqStart: record.seqStart,
|
|
12
|
+
seqEnd: record.seqEnd,
|
|
13
|
+
generic: record.generic ?? [],
|
|
14
|
+
...(Array.isArray(record.genericTopic) ? { genericTopic: record.genericTopic } : {}),
|
|
15
|
+
...(Array.isArray(record.genericWorkflow) ? { genericWorkflow: record.genericWorkflow } : {}),
|
|
16
|
+
...(Number.isFinite(record.topicScore) ? { topicScore: record.topicScore } : {}),
|
|
17
|
+
...(typeof record.topicConfidence === 'string' && record.topicConfidence ? { topicConfidence: record.topicConfidence } : {}),
|
|
18
|
+
...(Number.isFinite(record.topicMinScore) ? { topicMinScore: record.topicMinScore } : {}),
|
|
19
|
+
...(typeof record.labelStamp === 'string' && record.labelStamp ? { labelStamp: record.labelStamp } : {}),
|
|
20
|
+
labelVersion: Number.isInteger(record.labelVersion) ? record.labelVersion : 1,
|
|
21
|
+
terms: record.terms ?? [],
|
|
22
|
+
keywords: record.keywords ?? [],
|
|
23
|
+
backfilled: record.backfilled ?? [],
|
|
24
|
+
conceptScores: record.conceptScores ?? {},
|
|
25
|
+
confidence: record.confidence ?? 0,
|
|
26
|
+
status: record.status ?? 'weak',
|
|
27
|
+
...(typeof record.vec === 'string' && record.vec ? { vec: record.vec } : {}),
|
|
28
|
+
...(Number.isFinite(record.chatter) ? { chatter: Number(record.chatter) } : {}),
|
|
29
|
+
...(typeof record.sessionId === 'string' && record.sessionId ? { sessionId: record.sessionId } : {}),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function saveIndexJsonl(file, records = []) {
|
|
34
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
35
|
+
const temp = `${file}.tmp-${process.pid}`;
|
|
36
|
+
fs.writeFileSync(temp, records.map((record) => JSON.stringify(toPointer(record))).join('\n') + (records.length > 0 ? '\n' : ''), 'utf8');
|
|
37
|
+
fs.renameSync(temp, file);
|
|
38
|
+
return file;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function loadIndexJsonl(file) {
|
|
42
|
+
if (!fs.existsSync(file)) return [];
|
|
43
|
+
return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter(Boolean)
|
|
44
|
+
.map((line) => {
|
|
45
|
+
const record = JSON.parse(line.replace(/^\uFEFF/, ''));
|
|
46
|
+
return { labelVersion: 1, ...record };
|
|
47
|
+
});
|
|
48
|
+
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B-layer labeling: frozen embedding + generic concept descriptions.
|
|
3
|
+
*
|
|
4
|
+
* No LLM, no network, no writes. One BGE load can label many texts.
|
|
5
|
+
*/
|
|
6
|
+
import { GENERIC_CONCEPTS } from './bc-recall.mjs';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* labeler 判定语义的版本号(手工维护)。
|
|
10
|
+
*
|
|
11
|
+
* 只有"会改变打标结果"的改动才 bump:
|
|
12
|
+
* · 分块规则 / 阈值默认值 / center 与 lead 判据
|
|
13
|
+
* · 概念描述的相似度计算方式
|
|
14
|
+
* · 文本拼接规则(user 在前、去 reasoning 等)
|
|
15
|
+
* 纯文档、纯日志、纯 UI 改动**不要** bump——否则 labelStamp 全员失效,
|
|
16
|
+
* 触发一次 30 分钟的全量重扫,与"增量/不扫"的目标冲突(创 2026-11-04 定)。
|
|
17
|
+
*
|
|
18
|
+
* marker.labelerVersion 只在回归测试时显式覆盖它,默认用本常量。
|
|
19
|
+
*/
|
|
20
|
+
export const LABELER_VERSION = 1;
|
|
21
|
+
|
|
22
|
+
export const GENERIC_LABELS = Object.freeze({
|
|
23
|
+
游戏: '游戏、桌游、跑团、角色扮演、掷骰、回合制',
|
|
24
|
+
故事: '故事、剧情、叙事、角色、对白、场景',
|
|
25
|
+
对话: '对话、聊天、讨论、交流',
|
|
26
|
+
技术: '技术、代码、程序、系统、开发、调试',
|
|
27
|
+
设备: '设备、硬件、机器、工具、图像',
|
|
28
|
+
规则: '规则、规定、制度、约束、流程',
|
|
29
|
+
记忆: '记忆、回忆、历史、上下文、连续性',
|
|
30
|
+
剧情: '剧情、情节、故事发展、角色行动',
|
|
31
|
+
病毒: '病毒、传染病、疫情、病原体、公共卫生',
|
|
32
|
+
公共卫生: '公共卫生、防疫、传播、疾病控制',
|
|
33
|
+
科学: '科学、研究、实验、论文、证据',
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
function normalizeVector(vector) {
|
|
37
|
+
const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) || 1;
|
|
38
|
+
return vector.map((value) => value / norm);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function cosine(a, b) {
|
|
42
|
+
let dot = 0;
|
|
43
|
+
for (let index = 0; index < Math.min(a.length, b.length); index += 1) dot += a[index] * b[index];
|
|
44
|
+
return dot;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function chunkText(text, { chunkChars = 600, maxChunks = 12 } = {}) {
|
|
48
|
+
const value = String(text ?? '');
|
|
49
|
+
if (value.length <= chunkChars) return [value];
|
|
50
|
+
const chunks = [];
|
|
51
|
+
for (let start = 0; start < value.length && chunks.length < maxChunks; start += chunkChars) {
|
|
52
|
+
chunks.push(value.slice(start, start + chunkChars));
|
|
53
|
+
}
|
|
54
|
+
return chunks;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function createBLabeler({ embed = null, concepts = GENERIC_CONCEPTS, labels = GENERIC_LABELS, kinds = null } = {}) {
|
|
58
|
+
if (typeof embed !== 'function') throw new Error('B labeler needs an embed function');
|
|
59
|
+
let ready = null;
|
|
60
|
+
const names = concepts.filter((concept) => labels[concept]);
|
|
61
|
+
const split = kinds !== null && typeof kinds === 'object' && Object.keys(kinds).length > 0;
|
|
62
|
+
const kindOf = (concept) => (kinds?.[concept] === 'workflow' ? 'workflow' : 'topic');
|
|
63
|
+
// topicConfidence is a display label; 0.4/0.5 are not write thresholds.
|
|
64
|
+
const confidenceOf = (score) => (score >= 0.5 ? 'normal' : score >= 0.4 ? 'low' : 'weak');
|
|
65
|
+
async function prepare() {
|
|
66
|
+
if (ready !== null) return ready;
|
|
67
|
+
const vectors = names.length > 0 ? await embed(names.map((concept) => labels[concept])) : [];
|
|
68
|
+
ready = { labels: names, vectors: vectors.map(normalizeVector) };
|
|
69
|
+
return ready;
|
|
70
|
+
}
|
|
71
|
+
function decide(scored, {
|
|
72
|
+
topK = 3,
|
|
73
|
+
minScore = 0.25,
|
|
74
|
+
topicMinScore = 0.33,
|
|
75
|
+
topicCenterMin = 0.02,
|
|
76
|
+
topicLead = 0.02,
|
|
77
|
+
topicTopK = 1,
|
|
78
|
+
workflowTopK = 2,
|
|
79
|
+
workflowCenterMin = 0.02,
|
|
80
|
+
workflowLead = 0.02,
|
|
81
|
+
topicBaseline = null,
|
|
82
|
+
topicBaselineFallback = null,
|
|
83
|
+
workflowBaseline = null,
|
|
84
|
+
workflowBaselineFallback = null,
|
|
85
|
+
vector = [],
|
|
86
|
+
chunks = 1,
|
|
87
|
+
} = {}) {
|
|
88
|
+
if (split) {
|
|
89
|
+
const topics = scored
|
|
90
|
+
.filter((item) => kindOf(item.concept) === 'topic')
|
|
91
|
+
.map((item) => {
|
|
92
|
+
const center = Number.isFinite(topicBaseline?.[item.concept])
|
|
93
|
+
? topicBaseline[item.concept]
|
|
94
|
+
: (Number.isFinite(topicBaselineFallback) ? topicBaselineFallback : 0);
|
|
95
|
+
return {
|
|
96
|
+
concept: item.concept,
|
|
97
|
+
score: item.score,
|
|
98
|
+
rawScore: item.score,
|
|
99
|
+
centered: item.score - center,
|
|
100
|
+
};
|
|
101
|
+
})
|
|
102
|
+
.sort((a, b) => b.centered - a.centered);
|
|
103
|
+
const topTopic = topics[0];
|
|
104
|
+
const secondTopic = topics[1];
|
|
105
|
+
const topicPasses = topTopic !== undefined
|
|
106
|
+
&& topTopic.rawScore >= topicMinScore
|
|
107
|
+
&& topTopic.centered >= topicCenterMin
|
|
108
|
+
&& (secondTopic === undefined || topTopic.centered - secondTopic.centered >= topicLead);
|
|
109
|
+
const topicCandidates = topicPasses
|
|
110
|
+
? topics.slice(0, Math.max(1, topicTopK))
|
|
111
|
+
: [];
|
|
112
|
+
const workflows = scored
|
|
113
|
+
.filter((item) => kindOf(item.concept) === 'workflow')
|
|
114
|
+
.map((item) => {
|
|
115
|
+
const center = Number.isFinite(workflowBaseline?.[item.concept])
|
|
116
|
+
? workflowBaseline[item.concept]
|
|
117
|
+
: (Number.isFinite(workflowBaselineFallback) ? workflowBaselineFallback : 0);
|
|
118
|
+
return {
|
|
119
|
+
concept: item.concept,
|
|
120
|
+
score: item.score,
|
|
121
|
+
rawScore: item.score,
|
|
122
|
+
centered: item.score - center,
|
|
123
|
+
};
|
|
124
|
+
})
|
|
125
|
+
.sort((a, b) => b.centered - a.centered);
|
|
126
|
+
const topWorkflow = workflows[0];
|
|
127
|
+
const secondWorkflow = workflows[1];
|
|
128
|
+
const workflowPasses = topWorkflow !== undefined
|
|
129
|
+
&& topWorkflow.rawScore >= minScore
|
|
130
|
+
&& topWorkflow.centered >= workflowCenterMin
|
|
131
|
+
&& (secondWorkflow === undefined || topWorkflow.centered - secondWorkflow.centered >= workflowLead);
|
|
132
|
+
const workflow = workflowPasses
|
|
133
|
+
? workflows
|
|
134
|
+
.slice(0, Math.max(0, workflowTopK))
|
|
135
|
+
.filter((item) => item.rawScore >= minScore && item.centered >= workflowCenterMin)
|
|
136
|
+
: [];
|
|
137
|
+
const candidates = [...topicCandidates, ...workflow]
|
|
138
|
+
.sort((a, b) => b.rawScore - a.rawScore)
|
|
139
|
+
.slice(0, topicCandidates.length + workflowTopK);
|
|
140
|
+
const scores = Object.fromEntries([...topicCandidates, ...workflow].map((item) => [item.concept, item.rawScore]));
|
|
141
|
+
return {
|
|
142
|
+
candidates,
|
|
143
|
+
topic: topicCandidates,
|
|
144
|
+
workflow,
|
|
145
|
+
scores,
|
|
146
|
+
topicScores: Object.fromEntries(topics.map((item) => [item.concept, item.rawScore])),
|
|
147
|
+
topicCenteredScores: Object.fromEntries(topics.map((item) => [item.concept, item.centered])),
|
|
148
|
+
workflowScores: Object.fromEntries(workflows.map((item) => [item.concept, item.rawScore])),
|
|
149
|
+
workflowCenteredScores: Object.fromEntries(workflows.map((item) => [item.concept, item.centered])),
|
|
150
|
+
topicScore: topicCandidates.length > 0 ? topTopic.rawScore : null,
|
|
151
|
+
topicCenteredScore: topicCandidates.length > 0 ? topTopic.centered : null,
|
|
152
|
+
topicConfidence: topicCandidates.length > 0 ? confidenceOf(topTopic.rawScore) : null,
|
|
153
|
+
topicMinScore,
|
|
154
|
+
topicCenterMin,
|
|
155
|
+
topicLead,
|
|
156
|
+
workflowCenterMin,
|
|
157
|
+
workflowLead,
|
|
158
|
+
topicBaseline,
|
|
159
|
+
topicBaselineFallback,
|
|
160
|
+
workflowBaseline,
|
|
161
|
+
workflowBaselineFallback,
|
|
162
|
+
chunks,
|
|
163
|
+
unknown: candidates.length === 0,
|
|
164
|
+
top: topicCandidates[0] ?? workflow[0] ?? null,
|
|
165
|
+
vector,
|
|
166
|
+
version: 2,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
const candidates = scored.filter((item) => item.score >= minScore).slice(0, topK);
|
|
170
|
+
return {
|
|
171
|
+
candidates,
|
|
172
|
+
unknown: candidates.length === 0 || candidates[0].score < minScore,
|
|
173
|
+
top: candidates[0] ?? null,
|
|
174
|
+
vector,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
return {
|
|
178
|
+
async label(text, options = {}) {
|
|
179
|
+
const prepared = await prepare();
|
|
180
|
+
const [rawVector] = await embed([String(text ?? '')]);
|
|
181
|
+
const vector = normalizeVector(rawVector);
|
|
182
|
+
if (prepared.labels.length === 0) {
|
|
183
|
+
return { candidates: [], unknown: true, top: null, vector, ...(split ? { version: 2, topic: [], workflow: [], scores: {} } : {}) };
|
|
184
|
+
}
|
|
185
|
+
const scored = prepared.labels.map((concept, index) => ({
|
|
186
|
+
concept,
|
|
187
|
+
score: cosine(vector, prepared.vectors[index]),
|
|
188
|
+
})).sort((a, b) => b.score - a.score);
|
|
189
|
+
return decide(scored, { ...options, vector, chunks: 1 });
|
|
190
|
+
},
|
|
191
|
+
async labelChunked(text, options = {}) {
|
|
192
|
+
const prepared = await prepare();
|
|
193
|
+
const chunks = chunkText(text, options);
|
|
194
|
+
if (prepared.labels.length === 0) {
|
|
195
|
+
return { candidates: [], unknown: true, top: null, vector: [], ...(split ? { version: 2, topic: [], workflow: [], scores: {} } : {}) };
|
|
196
|
+
}
|
|
197
|
+
const best = new Map();
|
|
198
|
+
const vectors = [];
|
|
199
|
+
for (const chunk of chunks) {
|
|
200
|
+
const [rawVector] = await embed([chunk]);
|
|
201
|
+
const vector = normalizeVector(rawVector);
|
|
202
|
+
vectors.push(vector);
|
|
203
|
+
for (let index = 0; index < prepared.labels.length; index += 1) {
|
|
204
|
+
const concept = prepared.labels[index];
|
|
205
|
+
const score = cosine(vector, prepared.vectors[index]);
|
|
206
|
+
if (!best.has(concept) || score > best.get(concept)) best.set(concept, score);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
const summed = new Array(vectors[0]?.length ?? 0).fill(0);
|
|
210
|
+
for (const vector of vectors) {
|
|
211
|
+
for (let index = 0; index < summed.length; index += 1) summed[index] += vector[index] ?? 0;
|
|
212
|
+
}
|
|
213
|
+
const vector = normalizeVector(summed);
|
|
214
|
+
const scored = [...best.entries()]
|
|
215
|
+
.map(([concept, score]) => ({ concept, score }))
|
|
216
|
+
.sort((a, b) => b.score - a.score);
|
|
217
|
+
return decide(scored, { ...options, vector, chunks: chunks.length });
|
|
218
|
+
},
|
|
219
|
+
};
|
|
220
|
+
}
|