@aiwayds/dsh-topics-memory 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/README.zh.md +5 -0
- package/lib/commands.js +27 -11
- package/lib/config.d.ts +54 -1
- package/lib/config.js +38 -1
- package/lib/digest.d.ts +73 -0
- package/lib/digest.js +384 -0
- package/lib/distill.d.ts +39 -0
- package/lib/distill.js +250 -15
- package/lib/embedding.d.ts +113 -0
- package/lib/embedding.js +208 -0
- package/lib/ilog.d.ts +145 -0
- package/lib/ilog.js +236 -0
- package/lib/index.d.ts +8 -0
- package/lib/index.js +23 -6
- package/lib/pipeline-events.d.ts +36 -0
- package/lib/pipeline-events.js +121 -0
- package/lib/quality.d.ts +47 -7
- package/lib/quality.js +124 -20
- package/lib/retrieval.d.ts +119 -0
- package/lib/retrieval.js +290 -0
- package/lib/service.d.ts +60 -3
- package/lib/service.js +320 -44
- package/lib/store.d.ts +15 -12
- package/lib/store.js +43 -3
- package/lib/tools.js +6 -2
- package/package.json +2 -2
- package/src/commands.ts +35 -16
- package/src/config.ts +47 -1
- package/src/digest.ts +416 -0
- package/src/distill.ts +275 -14
- package/src/embedding.ts +294 -0
- package/src/ilog.ts +306 -0
- package/src/index.ts +23 -5
- package/src/pipeline-events.ts +132 -0
- package/src/quality.ts +145 -21
- package/src/retrieval.ts +357 -0
- package/src/service.ts +326 -52
- package/src/store.ts +46 -10
- package/src/tools.ts +6 -2
package/lib/service.js
CHANGED
|
@@ -6,18 +6,58 @@
|
|
|
6
6
|
* @module service
|
|
7
7
|
*/
|
|
8
8
|
import { readFile, stat } from 'node:fs/promises';
|
|
9
|
-
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
9
|
+
import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
10
10
|
import { join } from 'node:path';
|
|
11
11
|
import * as okf from "./okf.js";
|
|
12
|
-
import { searchTopics, scoreTopic, passesGate, tokenize } from "./retrieval.js";
|
|
13
|
-
import { assembleInjection, assemblePointer, POINTER_PER_TOPIC, POINTER_TOTAL } from "./digest.js";
|
|
14
|
-
import {
|
|
12
|
+
import { searchTopics, scoreTopic, passesGate, tokenize, sanitizeQuery, loadEmbeddingIndex, rerankNearFloor, ensureIndexFresh, saveEmbeddingIndex } from "./retrieval.js";
|
|
13
|
+
import { assembleInjection, assemblePointer, assemblePackage, POINTER_PER_TOPIC, POINTER_TOTAL } from "./digest.js";
|
|
14
|
+
import { embedTexts, embedQuerySync, embedderWith, resolveEmbedModel } from "./embedding.js";
|
|
15
|
+
import { resolveBundleRoot } from "./paths.js";
|
|
16
|
+
import { aggregateStats, querySample, aggregateUsage, aggregateZeroOpen, ZERO_OPEN_TAIL, USAGE_WINDOW_DAYS, attributeOpen, OPEN_ATTRIBUTION_TAIL } from "./ilog.js";
|
|
15
17
|
import { fileHistory, fileAtRev } from "./git.js";
|
|
18
|
+
/** Tolerant JSONL line read: undefined = unreadable file. Unlike an empty
|
|
19
|
+
* array, undefined tells the caller to fail open AND skip caching — a
|
|
20
|
+
* successful stat paired with a failed read must not freeze an empty
|
|
21
|
+
* snapshot into the cache until the next append. */
|
|
22
|
+
function readLogLinesSync(file) {
|
|
23
|
+
try {
|
|
24
|
+
return readFileSync(file, 'utf8').split('\n');
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return undefined;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** Parse already-split lines back-to-front, keeping the last `tail` parsed
|
|
31
|
+
* records in order: blanks and torn JSON lines are skipped (and never
|
|
32
|
+
* consume tail budget), so a torn tail line cannot crowd out a good one.
|
|
33
|
+
* Shared by the usage/zero-open cache — one tail parse feeds both. */
|
|
34
|
+
function parseJsonlLinesSync(lines, tail = Number.POSITIVE_INFINITY) {
|
|
35
|
+
const out = [];
|
|
36
|
+
for (let i = lines.length - 1; i >= 0 && out.length < tail; i -= 1) {
|
|
37
|
+
const line = lines[i];
|
|
38
|
+
if (line.trim() === '')
|
|
39
|
+
continue;
|
|
40
|
+
try {
|
|
41
|
+
out.push(JSON.parse(line));
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
continue; // torn tail line
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
out.reverse();
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
16
50
|
export class TopicsService {
|
|
17
51
|
cache = new Map();
|
|
18
52
|
/** mtime-keyed parse of the observations log, grouped by session. */
|
|
19
53
|
echoCache;
|
|
20
|
-
|
|
54
|
+
/** Single shared cache over the two feedback JSONL logs (S2): the key is
|
|
55
|
+
* the (mtime,size) quartet of injections.jsonl + opens.jsonl, one read
|
|
56
|
+
* and one tail parse feed BOTH aggregates — usageSignalsSync and
|
|
57
|
+
* zeroOpenSlugsSync used to each keep a full-parse cache of their own.
|
|
58
|
+
* Invalidation stays append-driven: the store only ever appends, so any
|
|
59
|
+
* write moves mtime+size. Never populated when a read failed (N1). */
|
|
60
|
+
logSignalsCache;
|
|
21
61
|
store;
|
|
22
62
|
getConfig;
|
|
23
63
|
sync;
|
|
@@ -92,7 +132,7 @@ export class TopicsService {
|
|
|
92
132
|
invalidate() {
|
|
93
133
|
this.cache.clear();
|
|
94
134
|
this.echoCache = undefined;
|
|
95
|
-
this.
|
|
135
|
+
this.logSignalsCache = undefined;
|
|
96
136
|
}
|
|
97
137
|
/**
|
|
98
138
|
* Slugs distilled from THIS session's own turns — the echo set (2026-09
|
|
@@ -151,10 +191,34 @@ export class TopicsService {
|
|
|
151
191
|
* Usage signals (ADR 0015): mtime-cached aggregate over the injection log
|
|
152
192
|
* and the opens log, rolling USAGE_WINDOW_DAYS. Synchronous like
|
|
153
193
|
* echoSlugsSync — the retrieval hot path is sync — and cheap when warm:
|
|
154
|
-
* a fresh cache costs two stat() calls
|
|
194
|
+
* a fresh cache costs two stat() calls, one read of each log, and feeds
|
|
195
|
+
* the zero-open set from the same parse. Unreadable logs fail open to an
|
|
155
196
|
* empty map (no boost), never an error.
|
|
156
197
|
*/
|
|
157
198
|
usageSignalsSync() {
|
|
199
|
+
return this.logSignalsSync().signals;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Zero-open slug set for the decay (v5): the shared aggregateZeroOpen
|
|
203
|
+
* rule over the injection tail and the whole opens log (compacted, so a
|
|
204
|
+
* full read stays small). Shares one cache and one parse with
|
|
205
|
+
* usageSignalsSync; unreadable logs fail open to an empty set (no
|
|
206
|
+
* decay), never an error. Only called when zeroOpenDecay is on.
|
|
207
|
+
*/
|
|
208
|
+
zeroOpenSlugsSync() {
|
|
209
|
+
return this.logSignalsSync().zeroOpenSlugs;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* The one shared feedback-log read behind both signals: stat both files
|
|
213
|
+
* (mtime+size quartet key), parse the injections TAIL only
|
|
214
|
+
* (ZERO_OPEN_TAIL rounds comfortably cover both 30-day windows) and the
|
|
215
|
+
* opens log whole (the zero-open rule is whole-history), then feed the
|
|
216
|
+
* two pure aggregates from that single parse. A failed read still fails
|
|
217
|
+
* open to empty inputs but NEVER populates the cache (N1) — otherwise a
|
|
218
|
+
* stat-success/read-failure race would freeze an empty snapshot until
|
|
219
|
+
* the next append.
|
|
220
|
+
*/
|
|
221
|
+
logSignalsSync() {
|
|
158
222
|
const injFile = join(this.store.metaDir(), 'injections.jsonl');
|
|
159
223
|
const opensFile = join(this.store.metaDir(), 'opens.jsonl');
|
|
160
224
|
const statOf = (file) => {
|
|
@@ -168,42 +232,111 @@ export class TopicsService {
|
|
|
168
232
|
};
|
|
169
233
|
const injSt = statOf(injFile);
|
|
170
234
|
const opensSt = statOf(opensFile);
|
|
171
|
-
const cached = this.
|
|
235
|
+
const cached = this.logSignalsCache;
|
|
172
236
|
if (cached !== undefined &&
|
|
173
237
|
cached.injMs === injSt.mtimeMs &&
|
|
174
238
|
cached.injSize === injSt.size &&
|
|
175
239
|
cached.opensMs === opensSt.mtimeMs &&
|
|
176
240
|
cached.opensSize === opensSt.size) {
|
|
177
|
-
return cached
|
|
241
|
+
return cached;
|
|
242
|
+
}
|
|
243
|
+
const injLines = readLogLinesSync(injFile);
|
|
244
|
+
const opensLines = readLogLinesSync(opensFile);
|
|
245
|
+
const injections = parseJsonlLinesSync(injLines ?? [], ZERO_OPEN_TAIL);
|
|
246
|
+
const opens = parseJsonlLinesSync(opensLines ?? []);
|
|
247
|
+
const now = Date.now();
|
|
248
|
+
const signals = aggregateUsage(injections, opens, USAGE_WINDOW_DAYS, now);
|
|
249
|
+
const zeroOpenSlugs = new Set(aggregateZeroOpen(injections, opens, now).keys());
|
|
250
|
+
if (injLines !== undefined && opensLines !== undefined) {
|
|
251
|
+
this.logSignalsCache = {
|
|
252
|
+
injMs: injSt.mtimeMs,
|
|
253
|
+
injSize: injSt.size,
|
|
254
|
+
opensMs: opensSt.mtimeMs,
|
|
255
|
+
opensSize: opensSt.size,
|
|
256
|
+
signals,
|
|
257
|
+
zeroOpenSlugs,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
return { signals, zeroOpenSlugs };
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* zeroOpenDecay fields for scoring calls: empty when the gate is off —
|
|
264
|
+
* no file reads, zero behavior change — and the shared mtime-cached set
|
|
265
|
+
* when on. Spread into every searchTopics/scoreTopic cfg that already
|
|
266
|
+
* carries the usage signals (injection paths + explicit topic_search;
|
|
267
|
+
* the slow lane's recall band stays untouched, mirroring usageBoost).
|
|
268
|
+
*/
|
|
269
|
+
zeroOpenCfg() {
|
|
270
|
+
if (this.cfg.zeroOpenDecay !== true)
|
|
271
|
+
return {};
|
|
272
|
+
return { zeroOpenDecay: true, zeroOpenSlugs: this.zeroOpenSlugsSync() };
|
|
273
|
+
}
|
|
274
|
+
/** mtime-cached view of meta/embeddings.json (sync read — hot-lane caller).
|
|
275
|
+
* Missing/corrupt index = {} → the reranker degrades to lexical order. */
|
|
276
|
+
embedIndexCache;
|
|
277
|
+
embeddingIndexCacheGet() {
|
|
278
|
+
const root = resolveBundleRoot();
|
|
279
|
+
const file = join(root, 'meta', 'embeddings.json');
|
|
280
|
+
try {
|
|
281
|
+
const st = statSync(file);
|
|
282
|
+
if (this.embedIndexCache !== undefined && this.embedIndexCache.mtimeMs === st.mtimeMs && this.embedIndexCache.size === st.size) {
|
|
283
|
+
return this.embedIndexCache.index;
|
|
284
|
+
}
|
|
285
|
+
const index = loadEmbeddingIndex(root);
|
|
286
|
+
this.embedIndexCache = { mtimeMs: st.mtimeMs, size: st.size, index };
|
|
287
|
+
return index;
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
return this.embedIndexCache?.index ?? {};
|
|
178
291
|
}
|
|
179
|
-
|
|
180
|
-
|
|
292
|
+
}
|
|
293
|
+
/** Shadow bypass ledger (spec §8.3): one line per shadow round — feature
|
|
294
|
+
* fields only, never query text (jev-log discipline). Silent on failure;
|
|
295
|
+
* 512KB truncation keeps the newest three quarters. */
|
|
296
|
+
appendPackageShadowLedger(entry) {
|
|
297
|
+
try {
|
|
298
|
+
const file = join(resolveBundleRoot(), 'meta', 'package-shadow.jsonl');
|
|
299
|
+
let lines = [];
|
|
181
300
|
try {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
out.push(JSON.parse(line));
|
|
187
|
-
}
|
|
188
|
-
catch {
|
|
189
|
-
continue; // torn tail line
|
|
190
|
-
}
|
|
301
|
+
const st = statSync(file);
|
|
302
|
+
if (st.size > 512 * 1024) {
|
|
303
|
+
const all = readFileSync(file, 'utf8').split('\n').filter((l) => l !== '');
|
|
304
|
+
lines = all.slice(Math.floor(all.length / 4));
|
|
191
305
|
}
|
|
192
306
|
}
|
|
193
|
-
catch {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
307
|
+
catch { /* first write */ }
|
|
308
|
+
lines.push(JSON.stringify(entry));
|
|
309
|
+
writeFileSync(file, lines.join('\n') + '\n');
|
|
310
|
+
}
|
|
311
|
+
catch { /* ledger must never break the lane */ }
|
|
312
|
+
}
|
|
313
|
+
/** Boot-time embedding index maintenance (spec §3: boot 校验+增量重建).
|
|
314
|
+
* Provider '' or packageMode off → no-op (default-off byte-for-byte).
|
|
315
|
+
* Full-roster embed on rebuild (A1 §5: ~9.5s for 137 topics). */
|
|
316
|
+
async refreshEmbeddingIndex() {
|
|
317
|
+
const provider = (this.cfg.embeddingProvider ?? '').trim();
|
|
318
|
+
if (provider === '' || this.cfg.packageMode === undefined || this.cfg.packageMode === 'off')
|
|
319
|
+
return;
|
|
320
|
+
const embedder = embedderWith({ provider, model: this.cfg.embeddingModel });
|
|
321
|
+
const topics = await this.roster();
|
|
322
|
+
const root = resolveBundleRoot();
|
|
323
|
+
const index = loadEmbeddingIndex(root);
|
|
324
|
+
const stale = ensureIndexFresh(index, topics, embedder);
|
|
325
|
+
if (stale.length === 0)
|
|
326
|
+
return;
|
|
327
|
+
const docs = await Promise.all(stale.map(async (slug) => (await this.readDoc(slug)) ?? ''));
|
|
328
|
+
const texts = stale.map((slug, i) => `${slug}\n${docs[i] ?? ''}`);
|
|
329
|
+
const r = await embedTexts(texts, { provider, model: this.cfg.embeddingModel });
|
|
330
|
+
if (!r.ok)
|
|
331
|
+
return;
|
|
332
|
+
const next = { ...index };
|
|
333
|
+
stale.forEach((slug, i) => {
|
|
334
|
+
const vec = r.vectors[i];
|
|
335
|
+
if (vec !== undefined)
|
|
336
|
+
next[slug] = { model: r.model, dim: vec.length, vec };
|
|
337
|
+
});
|
|
338
|
+
saveEmbeddingIndex(root, next);
|
|
339
|
+
this.embedIndexCache = undefined;
|
|
207
340
|
}
|
|
208
341
|
/**
|
|
209
342
|
* Full retrieval round: search → assemble → log (ADR 0006/0007). The
|
|
@@ -230,6 +363,7 @@ export class TopicsService {
|
|
|
230
363
|
conflicts,
|
|
231
364
|
usageBoost: cfg.usageBoost,
|
|
232
365
|
usage: this.usageSignalsSync(),
|
|
366
|
+
...this.zeroOpenCfg(),
|
|
233
367
|
});
|
|
234
368
|
const bySlug = new Map(roster.map((r) => [r.slug, r]));
|
|
235
369
|
const entries = [];
|
|
@@ -324,8 +458,12 @@ export class TopicsService {
|
|
|
324
458
|
}
|
|
325
459
|
return { text: '', outcome: empty, deduped: [], echoed: [], included: [], slowIncluded: [] };
|
|
326
460
|
}
|
|
461
|
+
// A-line gate (A3/A4): query cleaning + package + semantic rerank ride
|
|
462
|
+
// behind packageMode (default off = byte-for-byte legacy path).
|
|
463
|
+
const pkgMode = cfg.packageMode === 'shadow' || cfg.packageMode === 'on' ? cfg.packageMode : undefined;
|
|
464
|
+
const effectiveQuery = pkgMode !== undefined ? sanitizeQuery(query) : query;
|
|
327
465
|
const conflicts = this.store.getConflictsSync();
|
|
328
|
-
const outcome = searchTopics(
|
|
466
|
+
const outcome = searchTopics(effectiveQuery, roster, {
|
|
329
467
|
threshold: cfg.matchThreshold,
|
|
330
468
|
topK: cfg.topK,
|
|
331
469
|
tagBoost: cfg.tagBoost,
|
|
@@ -334,7 +472,38 @@ export class TopicsService {
|
|
|
334
472
|
conflicts,
|
|
335
473
|
usageBoost: cfg.usageBoost,
|
|
336
474
|
usage: this.usageSignalsSync(),
|
|
475
|
+
...this.zeroOpenCfg(),
|
|
337
476
|
});
|
|
477
|
+
// ---- A-line semantic rerank (A4 决议 2, spec §3.1) ----
|
|
478
|
+
// Reorder ONLY the 0.20–0.30 nearFloor band (the 93% near-miss mass);
|
|
479
|
+
// thresholds/gateScore/hits untouched. Any failure degrades to the
|
|
480
|
+
// original order — the fast lane never blocks on the semantic layer.
|
|
481
|
+
let nearFloorRerank;
|
|
482
|
+
if (pkgMode !== undefined) {
|
|
483
|
+
const bandLo = 0.2;
|
|
484
|
+
const band = outcome.nearMisses.filter((h) => h.score >= bandLo && h.score < cfg.matchThreshold);
|
|
485
|
+
const orderBefore = band.map((h) => h.slug);
|
|
486
|
+
if (band.length > 1) {
|
|
487
|
+
const index = this.embeddingIndexCacheGet();
|
|
488
|
+
const models = new Set(Object.values(index).map((e) => e.model));
|
|
489
|
+
const embedModel = resolveEmbedModel(cfg.embeddingModel);
|
|
490
|
+
if (models.size <= 1 && (models.size === 0 || models.has(embedModel))) {
|
|
491
|
+
const qv = embedQuerySync(effectiveQuery, { provider: cfg.embeddingProvider, model: cfg.embeddingModel });
|
|
492
|
+
const rr = rerankNearFloor(effectiveQuery, qv.ok ? qv.vector : null, band, index);
|
|
493
|
+
if (rr.reranked) {
|
|
494
|
+
const inBand = (h) => h.score >= bandLo && h.score < cfg.matchThreshold;
|
|
495
|
+
const rrSlugs = new Set(rr.hits.map((h) => h.slug));
|
|
496
|
+
const merged = [...rr.hits, ...band.filter((h) => !rrSlugs.has(h.slug))];
|
|
497
|
+
let mi = 0;
|
|
498
|
+
outcome.nearMisses = outcome.nearMisses.map((h) => (inBand(h) ? merged[mi++] : h));
|
|
499
|
+
nearFloorRerank = { reranked: true, orderBefore, orderAfter: merged.map((h) => h.slug) };
|
|
500
|
+
}
|
|
501
|
+
else {
|
|
502
|
+
nearFloorRerank = { reranked: false, passthroughReason: rr.passthroughReason, orderBefore, orderAfter: orderBefore };
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
338
507
|
const exclude = dedup?.exclude;
|
|
339
508
|
const echo = dedup?.echo;
|
|
340
509
|
const deduped = exclude === undefined ? [] : outcome.hits.filter((h) => exclude.has(h.slug)).map((h) => h.slug);
|
|
@@ -365,7 +534,7 @@ export class TopicsService {
|
|
|
365
534
|
let slowIncluded = [];
|
|
366
535
|
const consumedAt = new Date().toISOString();
|
|
367
536
|
if (slow !== undefined) {
|
|
368
|
-
const queryTokens = new Set(tokenize(
|
|
537
|
+
const queryTokens = new Set(tokenize(effectiveQuery));
|
|
369
538
|
for (const item of slow.items) {
|
|
370
539
|
if (exclude?.has(item.slug)) {
|
|
371
540
|
if (!deduped.includes(item.slug))
|
|
@@ -397,6 +566,7 @@ export class TopicsService {
|
|
|
397
566
|
structuralGate: true,
|
|
398
567
|
usageBoost: cfg.usageBoost,
|
|
399
568
|
usage: this.usageSignalsSync(),
|
|
569
|
+
...this.zeroOpenCfg(),
|
|
400
570
|
});
|
|
401
571
|
shadow.push({
|
|
402
572
|
slug: item.slug,
|
|
@@ -415,9 +585,70 @@ export class TopicsService {
|
|
|
415
585
|
const pointerMode = cfg.injectMode !== 'digest';
|
|
416
586
|
// Pointer budget: per-entry fixed at the design's 80, total clamped to the
|
|
417
587
|
// fast-lane upper bound of 600 (I4) — totalBudget stays tunable downward.
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
588
|
+
let packageSkipped = false;
|
|
589
|
+
let packageShadow;
|
|
590
|
+
let injection;
|
|
591
|
+
if (!pointerMode) {
|
|
592
|
+
injection = assembleInjection(entries, { perTopicBudget: cfg.perTopicBudget, totalBudget: cfg.totalBudget }, slowPointers);
|
|
593
|
+
}
|
|
594
|
+
else if (pkgMode === undefined) {
|
|
595
|
+
injection = assemblePointer(entries, { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) }, slowPointers);
|
|
596
|
+
}
|
|
597
|
+
else {
|
|
598
|
+
// A-line relevance package (A3, spec §2): direct = lexical hits,
|
|
599
|
+
// neighborhood = graph hits + slow picks, weak = reranked band head.
|
|
600
|
+
const budget = { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) };
|
|
601
|
+
const legacy = assemblePointer(entries, budget, slowPointers);
|
|
602
|
+
const direct = [];
|
|
603
|
+
const neighborhood = [];
|
|
604
|
+
for (const e of entries) {
|
|
605
|
+
const meta = bySlug.get(e.slug);
|
|
606
|
+
const cand = {
|
|
607
|
+
slug: e.slug,
|
|
608
|
+
title: meta?.title ?? e.slug,
|
|
609
|
+
status: meta?.status ?? 'draft',
|
|
610
|
+
description: meta?.description,
|
|
611
|
+
conclusion: firstContentLine(e.doc.body),
|
|
612
|
+
score: e.hit.score,
|
|
613
|
+
reasons: e.hit.reasons,
|
|
614
|
+
via: e.hit.viaGraph ? 'graph' : undefined,
|
|
615
|
+
};
|
|
616
|
+
(e.hit.viaGraph ? neighborhood : direct).push(cand);
|
|
617
|
+
}
|
|
618
|
+
for (const sp of slowPointers) {
|
|
619
|
+
neighborhood.push({ slug: sp.slug, title: sp.title, status: sp.status, description: sp.description, score: 0, reasons: [], via: 'slow-pick' });
|
|
620
|
+
}
|
|
621
|
+
const weak = outcome.nearMisses
|
|
622
|
+
.filter((h) => h.score >= 0.2 && h.score < cfg.matchThreshold)
|
|
623
|
+
.slice(0, 2)
|
|
624
|
+
.map((h) => {
|
|
625
|
+
const meta = bySlug.get(h.slug);
|
|
626
|
+
return { slug: h.slug, title: meta?.title ?? h.slug, status: meta?.status ?? 'draft', score: h.score, reasons: h.reasons, via: 'weak' };
|
|
627
|
+
});
|
|
628
|
+
const usage = this.usageSignalsSync();
|
|
629
|
+
const usageBySlug = new Map([...usage].map(([slug, sig]) => [slug, sig.opens * 3 + sig.hits]));
|
|
630
|
+
const pkgInput = { query: effectiveQuery, queryTokens: tokenize(effectiveQuery), direct, neighborhood, weak, usageBySlug };
|
|
631
|
+
const pkg = assemblePackage(pkgInput);
|
|
632
|
+
if (pkgMode === 'shadow') {
|
|
633
|
+
packageShadow = pkg;
|
|
634
|
+
injection = legacy;
|
|
635
|
+
}
|
|
636
|
+
else if (pkg === null) {
|
|
637
|
+
// Trivial-query downgrade (A3 决议 5): emit nothing, log the why.
|
|
638
|
+
packageSkipped = true;
|
|
639
|
+
injection = { text: '', usedTokens: 0, included: [], dropped: [] };
|
|
640
|
+
}
|
|
641
|
+
else {
|
|
642
|
+
const slowSet = new Set(slowPointers.map((sp) => sp.slug));
|
|
643
|
+
injection = {
|
|
644
|
+
text: pkg.text,
|
|
645
|
+
usedTokens: pkg.usedTokens,
|
|
646
|
+
included: pkg.included,
|
|
647
|
+
dropped: pkg.dropped,
|
|
648
|
+
slowIncluded: pkg.included.filter((slug) => slowSet.has(slug)),
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
}
|
|
421
652
|
slowIncluded = injection.slowIncluded ?? [];
|
|
422
653
|
for (const pointer of slowPointers) {
|
|
423
654
|
if (slowIncluded.includes(pointer.slug))
|
|
@@ -425,8 +656,8 @@ export class TopicsService {
|
|
|
425
656
|
}
|
|
426
657
|
const record = {
|
|
427
658
|
at: new Date().toISOString(),
|
|
428
|
-
queryTokenCount: tokenize(
|
|
429
|
-
querySample: querySample(
|
|
659
|
+
queryTokenCount: tokenize(effectiveQuery).length,
|
|
660
|
+
querySample: querySample(effectiveQuery),
|
|
430
661
|
rosterSize: outcome.rosterSize,
|
|
431
662
|
hits: outcome.hits.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons, viaGraph: h.viaGraph, strong: h.strong, bodyHits: h.bodyHits })),
|
|
432
663
|
// Reasons ride along: the gate-blocked marker is the replay evidence
|
|
@@ -441,10 +672,28 @@ export class TopicsService {
|
|
|
441
672
|
record.deduped = deduped;
|
|
442
673
|
if (echoed.length > 0)
|
|
443
674
|
record.echoed = echoed;
|
|
675
|
+
if (packageSkipped)
|
|
676
|
+
record.why = 'package-skipped-trivial';
|
|
444
677
|
if (unreadable.length > 0) {
|
|
445
678
|
record.dropped = [...(record.dropped ?? []), ...unreadable.map((slug) => ({ slug, reason: 'doc-unreadable' }))];
|
|
446
679
|
}
|
|
680
|
+
if (pkgMode === 'shadow' && pointerMode) {
|
|
681
|
+
this.appendPackageShadowLedger({
|
|
682
|
+
at: record.at,
|
|
683
|
+
...(sessionId !== undefined ? { sessionId } : {}),
|
|
684
|
+
queryTokenCount: record.queryTokenCount,
|
|
685
|
+
emitted: { tokens: injection.usedTokens, included: injection.included },
|
|
686
|
+
package: packageShadow === undefined
|
|
687
|
+
? null
|
|
688
|
+
: packageShadow === null
|
|
689
|
+
? { skipped: 'trivial-or-empty' }
|
|
690
|
+
: { tokens: packageShadow.usedTokens, included: packageShadow.included, layers: packageShadow.layers },
|
|
691
|
+
nearFloorRerank: nearFloorRerank ?? null,
|
|
692
|
+
});
|
|
693
|
+
}
|
|
447
694
|
if (slow !== undefined) {
|
|
695
|
+
if (slow.outcome !== undefined)
|
|
696
|
+
record.slowOutcome = slow.outcome;
|
|
448
697
|
// Any round whose pending was consumed is a slow-participating round —
|
|
449
698
|
// lane never stays undefined when the lane family is present (a
|
|
450
699
|
// delivered pick with fast pointers is 'mixed'; a consumed-but-empty
|
|
@@ -501,9 +750,14 @@ export class TopicsService {
|
|
|
501
750
|
if (doc === undefined)
|
|
502
751
|
return { found: false, slug };
|
|
503
752
|
// Awaited (tool path, not hot): a settled write makes the pointer-open
|
|
504
|
-
// stat reliable; a failed log write must not fail the tool.
|
|
753
|
+
// stat reliable; a failed log write must not fail the tool. Source
|
|
754
|
+
// attribution rides the same write: a cheap injections-tail scan decides
|
|
755
|
+
// pointer vs search and, for pointers, carries the hit score and the
|
|
756
|
+
// open→injection lag (ilog.attributeOpen).
|
|
757
|
+
const at = new Date().toISOString();
|
|
758
|
+
const attribution = await this.attributeOpen(slug, sessionId, Date.parse(at));
|
|
505
759
|
await this.store
|
|
506
|
-
.appendOpenRecord({ slug, at
|
|
760
|
+
.appendOpenRecord({ slug, at, ...(sessionId !== undefined ? { sessionId } : {}), ...attribution })
|
|
507
761
|
.catch(() => undefined);
|
|
508
762
|
// Optional fields are OMITTED, never present-as-undefined: the host's
|
|
509
763
|
// tool-output validation rejects any object carrying an undefined-valued
|
|
@@ -523,6 +777,18 @@ export class TopicsService {
|
|
|
523
777
|
found.description = doc.fm.description;
|
|
524
778
|
return found;
|
|
525
779
|
}
|
|
780
|
+
/**
|
|
781
|
+
* Resolve the open's source attribution: the injections tail (bounded by
|
|
782
|
+
* OPEN_ATTRIBUTION_TAIL) scanned for a same-session hit within the
|
|
783
|
+
* attribution window. Read failures degrade to 'search' — attribution is
|
|
784
|
+
* telemetry, never a blocker.
|
|
785
|
+
*/
|
|
786
|
+
async attributeOpen(slug, sessionId, nowMs) {
|
|
787
|
+
if (sessionId === undefined)
|
|
788
|
+
return { source: 'search' };
|
|
789
|
+
const tail = await this.store.readInjectionRecords(OPEN_ATTRIBUTION_TAIL).catch(() => []);
|
|
790
|
+
return attributeOpen(tail, slug, sessionId, nowMs);
|
|
791
|
+
}
|
|
526
792
|
/** Sync roster read (same mtime cache as roster()). */
|
|
527
793
|
rosterSync() {
|
|
528
794
|
let files;
|
|
@@ -634,7 +900,7 @@ export class TopicsService {
|
|
|
634
900
|
fm.triggers = triggers;
|
|
635
901
|
if (existing !== undefined && existing.fm.verified !== undefined)
|
|
636
902
|
fm.verified = existing.fm.verified;
|
|
637
|
-
const result = await this.store.saveTopic({ slug, doc: { fm, body } }, { message: existing === undefined ? `topics(topic): create ${slug}` : `topics(topic): update ${slug}`, created: existing === undefined });
|
|
903
|
+
const result = await this.store.saveTopic({ slug, doc: { fm, body }, links: input.links }, { message: existing === undefined ? `topics(topic): create ${slug}` : `topics(topic): update ${slug}`, created: existing === undefined });
|
|
638
904
|
this.invalidate();
|
|
639
905
|
void this.sync?.schedulePush();
|
|
640
906
|
return { ...result, slug: result.slug };
|
|
@@ -656,6 +922,7 @@ export class TopicsService {
|
|
|
656
922
|
conflicts: await this.store.getConflicts(),
|
|
657
923
|
usageBoost: cfg.usageBoost,
|
|
658
924
|
usage: this.usageSignalsSync(),
|
|
925
|
+
...this.zeroOpenCfg(),
|
|
659
926
|
});
|
|
660
927
|
}
|
|
661
928
|
async history(slug, limit = 20) {
|
|
@@ -720,6 +987,15 @@ function dedupePaths(paths) {
|
|
|
720
987
|
}
|
|
721
988
|
return out;
|
|
722
989
|
}
|
|
990
|
+
function firstContentLine(doc) {
|
|
991
|
+
for (const line of doc.split('\n')) {
|
|
992
|
+
const t = line.trim();
|
|
993
|
+
if (t === '' || t.startsWith('#'))
|
|
994
|
+
continue;
|
|
995
|
+
return t.slice(0, 200);
|
|
996
|
+
}
|
|
997
|
+
return '';
|
|
998
|
+
}
|
|
723
999
|
function sanitizeTriggers(triggers) {
|
|
724
1000
|
const out = [];
|
|
725
1001
|
for (const t of triggers) {
|
package/lib/store.d.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* @module store
|
|
10
10
|
*/
|
|
11
11
|
import * as okf from './okf.ts';
|
|
12
|
-
import type { InjectionRecord } from './ilog.ts';
|
|
12
|
+
import type { InjectionRecord, OpenRecord } from './ilog.ts';
|
|
13
13
|
export interface TopicMeta {
|
|
14
14
|
slug: string;
|
|
15
15
|
title: string;
|
|
@@ -110,10 +110,19 @@ export declare class BundleStore {
|
|
|
110
110
|
* Write a topic (atomic), regenerate the index, and commit both — one
|
|
111
111
|
* commit per conclusion change (ADR 0003). `slug` in the doc's depends is
|
|
112
112
|
* NOT normalized here; callers pass bundle-relative paths.
|
|
113
|
+
*
|
|
114
|
+
* `links` (A2, optional): whitelist-surviving semantic targets from the
|
|
115
|
+
* distill lane. Applied INSIDE this one write transaction — the topic and
|
|
116
|
+
* its edges land together (先落 topic 再建边); the Links section lands as
|
|
117
|
+
* body wikilinks and the index/backlinks regeneration below (same
|
|
118
|
+
* transaction) recomputes the edge index over them. Normalized and
|
|
119
|
+
* self/dedup-guarded here too, so any caller benefits from the same guard
|
|
120
|
+
* the distill lane already ran.
|
|
113
121
|
*/
|
|
114
122
|
saveTopic(input: {
|
|
115
123
|
slug: string;
|
|
116
124
|
doc: okf.TopicDoc;
|
|
125
|
+
links?: readonly string[];
|
|
117
126
|
}, opts: {
|
|
118
127
|
message: string;
|
|
119
128
|
actor?: string;
|
|
@@ -171,18 +180,12 @@ export declare class BundleStore {
|
|
|
171
180
|
private compactInjectionsIfNeeded;
|
|
172
181
|
readInjectionRecords(limit?: number): Promise<InjectionRecord[]>;
|
|
173
182
|
private opensPath;
|
|
174
|
-
appendOpenRecord(record:
|
|
175
|
-
slug: string;
|
|
176
|
-
at: string;
|
|
177
|
-
sessionId?: string;
|
|
178
|
-
}): Promise<void>;
|
|
183
|
+
appendOpenRecord(record: OpenRecord): Promise<void>;
|
|
179
184
|
private compactOpensIfNeeded;
|
|
180
|
-
/** Opens in the recent window; tolerant of a torn tail like the other JSONLs.
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
sessionId?: string;
|
|
185
|
-
}[]>;
|
|
185
|
+
/** Opens in the recent window; tolerant of a torn tail like the other JSONLs.
|
|
186
|
+
* Enrichment fields (sessionId/source/score/sinceInjectionMs) are optional:
|
|
187
|
+
* pre-enrichment lines carry only slug/at and read back without them. */
|
|
188
|
+
readOpenRecords(limit?: number): Promise<OpenRecord[]>;
|
|
186
189
|
getConflicts(): Promise<Set<string>>;
|
|
187
190
|
/** Sync variant for the injection hot path. */
|
|
188
191
|
getConflictsSync(): Set<string>;
|
package/lib/store.js
CHANGED
|
@@ -42,6 +42,14 @@ function dedupePaths(paths) {
|
|
|
42
42
|
function sameStrings(a, b) {
|
|
43
43
|
return a.length === b.length && a.every((v, i) => v === b[i]);
|
|
44
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Body heading for caller-supplied semantic links (A2). Links are rendered as
|
|
47
|
+
* `[[wikilinks]]` under this section so the EXISTING link-edge machinery
|
|
48
|
+
* (okf.bodyLinkSlugs → computeBacklinks → retrieval graph walk) indexes them
|
|
49
|
+
* as `via: 'link'` edges — the edge representation gains no new type; the
|
|
50
|
+
* body landing spot is what makes them first-class links for free.
|
|
51
|
+
*/
|
|
52
|
+
const LINKS_HEADING = 'Links';
|
|
45
53
|
export class BundleStore {
|
|
46
54
|
root;
|
|
47
55
|
actor;
|
|
@@ -285,6 +293,14 @@ export class BundleStore {
|
|
|
285
293
|
* Write a topic (atomic), regenerate the index, and commit both — one
|
|
286
294
|
* commit per conclusion change (ADR 0003). `slug` in the doc's depends is
|
|
287
295
|
* NOT normalized here; callers pass bundle-relative paths.
|
|
296
|
+
*
|
|
297
|
+
* `links` (A2, optional): whitelist-surviving semantic targets from the
|
|
298
|
+
* distill lane. Applied INSIDE this one write transaction — the topic and
|
|
299
|
+
* its edges land together (先落 topic 再建边); the Links section lands as
|
|
300
|
+
* body wikilinks and the index/backlinks regeneration below (same
|
|
301
|
+
* transaction) recomputes the edge index over them. Normalized and
|
|
302
|
+
* self/dedup-guarded here too, so any caller benefits from the same guard
|
|
303
|
+
* the distill lane already ran.
|
|
288
304
|
*/
|
|
289
305
|
async saveTopic(input, opts) {
|
|
290
306
|
return this.enqueue(async () => {
|
|
@@ -292,9 +308,24 @@ export class BundleStore {
|
|
|
292
308
|
const file = this.topicPath(slug);
|
|
293
309
|
const created = opts.created ?? !(await existsSync(file));
|
|
294
310
|
const now = new Date().toISOString();
|
|
311
|
+
let body = input.doc.body;
|
|
312
|
+
if (input.links !== undefined && input.links.length > 0) {
|
|
313
|
+
const targets = [];
|
|
314
|
+
for (const raw of input.links) {
|
|
315
|
+
if (typeof raw !== 'string')
|
|
316
|
+
continue;
|
|
317
|
+
const target = okf.normalizeLinkTarget(raw);
|
|
318
|
+
if (target === undefined || target === slug || targets.includes(target))
|
|
319
|
+
continue;
|
|
320
|
+
targets.push(target);
|
|
321
|
+
}
|
|
322
|
+
if (targets.length > 0) {
|
|
323
|
+
body = okf.setSection(body, LINKS_HEADING, targets.map((t) => `- [[${t}]]`).join('\n'));
|
|
324
|
+
}
|
|
325
|
+
}
|
|
295
326
|
const doc = {
|
|
296
327
|
fm: { ...input.doc.fm, generated: { by: opts.actor ?? this.actor, at: now } },
|
|
297
|
-
body
|
|
328
|
+
body,
|
|
298
329
|
};
|
|
299
330
|
const raw = okf.serializeTopicDoc(doc);
|
|
300
331
|
await atomicWrite(file, raw);
|
|
@@ -544,7 +575,8 @@ export class BundleStore {
|
|
|
544
575
|
}
|
|
545
576
|
// ------------------------------------------------------------------
|
|
546
577
|
// Pointer-open log (v4 §4.3) — one line per topic_open call; feeds the
|
|
547
|
-
// 「指针打开率」 stat. Features only (slug/time), never
|
|
578
|
+
// 「指针打开率」 stat. Features only (slug/time + attribution), never
|
|
579
|
+
// conversation text.
|
|
548
580
|
// ------------------------------------------------------------------
|
|
549
581
|
opensPath() {
|
|
550
582
|
return join(this.metaDir(), 'opens.jsonl');
|
|
@@ -568,7 +600,9 @@ export class BundleStore {
|
|
|
568
600
|
const lines = (await readFile(file, 'utf8')).split('\n').filter((l) => l.trim() !== '');
|
|
569
601
|
await atomicWrite(file, lines.slice(-Math.max(1, Math.floor(lines.length / 4))).map((l) => `${l}\n`).join(''));
|
|
570
602
|
}
|
|
571
|
-
/** Opens in the recent window; tolerant of a torn tail like the other JSONLs.
|
|
603
|
+
/** Opens in the recent window; tolerant of a torn tail like the other JSONLs.
|
|
604
|
+
* Enrichment fields (sessionId/source/score/sinceInjectionMs) are optional:
|
|
605
|
+
* pre-enrichment lines carry only slug/at and read back without them. */
|
|
572
606
|
async readOpenRecords(limit = 2000) {
|
|
573
607
|
let raw;
|
|
574
608
|
try {
|
|
@@ -587,6 +621,12 @@ export class BundleStore {
|
|
|
587
621
|
const rec = { slug: parsed.slug, at: parsed.at };
|
|
588
622
|
if (typeof parsed.sessionId === 'string')
|
|
589
623
|
rec.sessionId = parsed.sessionId;
|
|
624
|
+
if (parsed.source === 'pointer' || parsed.source === 'search')
|
|
625
|
+
rec.source = parsed.source;
|
|
626
|
+
if (typeof parsed.score === 'number')
|
|
627
|
+
rec.score = parsed.score;
|
|
628
|
+
if (typeof parsed.sinceInjectionMs === 'number')
|
|
629
|
+
rec.sinceInjectionMs = parsed.sinceInjectionMs;
|
|
590
630
|
out.push(rec);
|
|
591
631
|
}
|
|
592
632
|
}
|
package/lib/tools.js
CHANGED
|
@@ -108,8 +108,12 @@ export function buildTopicTools(service) {
|
|
|
108
108
|
},
|
|
109
109
|
],
|
|
110
110
|
},
|
|
111
|
-
async execute(args) {
|
|
112
|
-
|
|
111
|
+
async execute(args, exec) {
|
|
112
|
+
// The agent handle on the run context IS the session (Agent.id is the
|
|
113
|
+
// session id — the same key the injection records use). Absent in
|
|
114
|
+
// bare test/host contexts: attribution then degrades to 'search'.
|
|
115
|
+
const sessionId = exec?.agent !== undefined ? String(exec.agent.id) : undefined;
|
|
116
|
+
return await service.openTopic(args.slug, sessionId);
|
|
113
117
|
},
|
|
114
118
|
});
|
|
115
119
|
const topicSearch = defineTool({
|