@aiwayds/dsh-topics-memory 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +38 -1
- package/lib/config.js +29 -0
- package/lib/digest.d.ts +73 -0
- package/lib/digest.js +384 -0
- package/lib/distill.d.ts +39 -0
- package/lib/distill.js +250 -15
- package/lib/embedding.d.ts +113 -0
- package/lib/embedding.js +208 -0
- package/lib/ilog.d.ts +16 -0
- package/lib/index.d.ts +6 -0
- package/lib/index.js +23 -6
- package/lib/pipeline-events.d.ts +36 -0
- package/lib/pipeline-events.js +121 -0
- package/lib/quality.d.ts +47 -7
- package/lib/quality.js +124 -20
- package/lib/retrieval.d.ts +106 -0
- package/lib/retrieval.js +280 -0
- package/lib/service.d.ts +17 -1
- package/lib/service.js +202 -11
- package/lib/store.d.ts +9 -0
- package/lib/store.js +32 -1
- package/package.json +2 -2
- package/src/config.ts +34 -0
- package/src/digest.ts +416 -0
- package/src/distill.ts +275 -14
- package/src/embedding.ts +294 -0
- package/src/ilog.ts +17 -0
- package/src/index.ts +23 -5
- package/src/pipeline-events.ts +132 -0
- package/src/quality.ts +145 -21
- package/src/retrieval.ts +335 -0
- package/src/service.ts +199 -12
- package/src/store.ts +32 -2
package/lib/retrieval.js
CHANGED
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
*
|
|
12
12
|
* @module retrieval
|
|
13
13
|
*/
|
|
14
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { join } from 'node:path';
|
|
14
16
|
import { dependsSlugs } from "./okf.js";
|
|
15
17
|
/** Hard cap for the usage bonus — below threshold 0.3 so usage can never
|
|
16
18
|
* carry a candidate across the gate on its own (ADR 0015). */
|
|
@@ -256,3 +258,281 @@ export function searchTopics(query, roster, cfg = {}) {
|
|
|
256
258
|
}
|
|
257
259
|
return { hits, nearMisses, rosterSize: roster.length };
|
|
258
260
|
}
|
|
261
|
+
// ---------------------------------------------------------------------------
|
|
262
|
+
// Injection-quality line (wave/injection-quality, spec §2.1/§3.1/§3.3) —
|
|
263
|
+
// CAPABILITY ONLY. None of this is wired into searchTopics or any service
|
|
264
|
+
// path; the caller gates it on packageMode ≠ 'off' (Phase 2 wiring). With
|
|
265
|
+
// the config keys at their defaults every code path below is unreachable
|
|
266
|
+
// from the hot lane, so default-off behavior stays byte-for-byte.
|
|
267
|
+
// ---------------------------------------------------------------------------
|
|
268
|
+
// --- ① Query sanitization (A4 决议 4, spec §3.3) ---------------------------
|
|
269
|
+
// Pure, LLM-free, millisecond-scale. Treats the queryTokenCount p90=462
|
|
270
|
+
// paste-pollution tail: fenced code blocks, URLs, line-leading shell
|
|
271
|
+
// prompts, over-long lines, and a hard token cap.
|
|
272
|
+
/**
|
|
273
|
+
* Token cap in tokenize() units — the SAME unit injections.jsonl logs as
|
|
274
|
+
* queryTokenCount (p90=462 baseline; real-query p50=153 stays untouched).
|
|
275
|
+
* Raised or lowered via SanitizeOptions.maxTokens.
|
|
276
|
+
*/
|
|
277
|
+
export const QUERY_TOKEN_CAP = 96;
|
|
278
|
+
/**
|
|
279
|
+
* Lines longer than this are folded to their head (log-paste guard). Sized
|
|
280
|
+
* so the 96-token cap is the binding constraint, not the fold: 96 latin
|
|
281
|
+
* tokens ≈ 650+ chars, so a legitimate long single-line query passes the
|
|
282
|
+
* fold intact and only the token cap trims it. (实现期定 constant.)
|
|
283
|
+
*/
|
|
284
|
+
export const MAX_LINE_CHARS = 720;
|
|
285
|
+
const FENCE_OPEN = /^[ \t]*(```|~~~)/;
|
|
286
|
+
// Line-leading shell prompt: `user@host:~/repo$ `, `C:\> `, or a bare
|
|
287
|
+
// `$ `/`# `/`% `/`> `/`❯ ` marker. Requires trailing whitespace so words
|
|
288
|
+
// like `$PATH` or `#tag` survive untouched.
|
|
289
|
+
const LINE_PROMPT = /^[ \t]*(?:[\w.@:/\\~-]*[$#%>❯➜])[ \t]+/;
|
|
290
|
+
const URL_RE = /\b(?:https?:\/\/|www\.)[^\s<>"'`)\]]+/gi;
|
|
291
|
+
// Chunks the cap walks: one latin/digit word (or hyphenated compound) per
|
|
292
|
+
// match, or one CJK run (sliced when a run alone busts the cap).
|
|
293
|
+
const CHUNK_RUN = new RegExp(`${WORD_RUN.source}|${CJK_RUN.source}`, 'gu');
|
|
294
|
+
/** Remove fenced code blocks wholesale (markers AND body): fenced code is
|
|
295
|
+
* the paste-pollution payload; the prose around it is the actual query. */
|
|
296
|
+
function stripFencedBlocks(text) {
|
|
297
|
+
const out = [];
|
|
298
|
+
let open = null;
|
|
299
|
+
for (const line of text.split('\n')) {
|
|
300
|
+
const m = line.match(FENCE_OPEN);
|
|
301
|
+
if (open === null) {
|
|
302
|
+
if (m !== null)
|
|
303
|
+
open = m[1];
|
|
304
|
+
else
|
|
305
|
+
out.push(line);
|
|
306
|
+
}
|
|
307
|
+
else if (m !== null && m[1] === open) {
|
|
308
|
+
open = null; // closing fence — block fully dropped
|
|
309
|
+
}
|
|
310
|
+
// inside an open fence: drop the line
|
|
311
|
+
}
|
|
312
|
+
return out.join('\n');
|
|
313
|
+
}
|
|
314
|
+
/** Cap the sanitized text to ≤ cap distinct retrieval tokens. Fast path:
|
|
315
|
+
* at or under the cap the text passes through VERBATIM (no re-chunking —
|
|
316
|
+
* hyphens, punctuation and spacing survive byte-exact; retrieval scores
|
|
317
|
+
* tokens, not bytes, but the string may be echoed into logs/injection).
|
|
318
|
+
* Only an over-cap text rebuilds from chunks: head-first (a query's intent
|
|
319
|
+
* leads; pollution trails), with CJK runs sliced at the character level so
|
|
320
|
+
* a single giant run truncates instead of being dropped whole. */
|
|
321
|
+
function capTokens(text, cap) {
|
|
322
|
+
const total = tokenize(text).length;
|
|
323
|
+
if (total === 0)
|
|
324
|
+
return ''; // nothing lexical survives — caller sees a zero-token query
|
|
325
|
+
if (total <= cap)
|
|
326
|
+
return text;
|
|
327
|
+
const seen = new Set();
|
|
328
|
+
const kept = [];
|
|
329
|
+
for (const chunk of text.match(CHUNK_RUN) ?? []) {
|
|
330
|
+
const fresh = tokenize(chunk).filter((t) => !seen.has(t));
|
|
331
|
+
if (fresh.length === 0)
|
|
332
|
+
continue; // already-covered chunk adds nothing
|
|
333
|
+
if (seen.size + fresh.length <= cap) {
|
|
334
|
+
for (const t of fresh)
|
|
335
|
+
seen.add(t);
|
|
336
|
+
kept.push(chunk);
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
const room = cap - seen.size;
|
|
340
|
+
if (room <= 0)
|
|
341
|
+
break;
|
|
342
|
+
// Over budget: CJK runs truncate exactly (slice room+1 chars → ≤ room
|
|
343
|
+
// fresh bigrams, so the cap invariant holds). Latin chunks are single
|
|
344
|
+
// tokens by construction — they never reach here with room > 0, but if
|
|
345
|
+
// one ever does, dropping it beats busting the cap.
|
|
346
|
+
if (/^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(chunk)) {
|
|
347
|
+
kept.push(chunk.slice(0, room + 1));
|
|
348
|
+
}
|
|
349
|
+
break;
|
|
350
|
+
}
|
|
351
|
+
return kept.join(' ');
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Sanitize a raw user query for retrieval (spec §3.3): strip fenced code
|
|
355
|
+
* blocks and URLs, fold line-leading shell prompts, fold over-long lines to
|
|
356
|
+
* their head, cap the distinct-token count. Returns '' when nothing lexical
|
|
357
|
+
* survives — the caller treats that as a zero-token query (natural skip).
|
|
358
|
+
* Pure function; the hot lane only reaches it behind packageMode.
|
|
359
|
+
*/
|
|
360
|
+
export function sanitizeQuery(raw, opts = {}) {
|
|
361
|
+
const cap = opts.maxTokens ?? QUERY_TOKEN_CAP;
|
|
362
|
+
const maxLineChars = opts.maxLineChars ?? MAX_LINE_CHARS;
|
|
363
|
+
if (raw === '')
|
|
364
|
+
return '';
|
|
365
|
+
let text = stripFencedBlocks(raw);
|
|
366
|
+
text = text.replace(URL_RE, ' ');
|
|
367
|
+
text = text
|
|
368
|
+
.split('\n')
|
|
369
|
+
.map((line) => {
|
|
370
|
+
let l = line.replace(LINE_PROMPT, '');
|
|
371
|
+
if (l.length > maxLineChars)
|
|
372
|
+
l = l.slice(0, maxLineChars);
|
|
373
|
+
return l;
|
|
374
|
+
})
|
|
375
|
+
.join('\n');
|
|
376
|
+
text = text.replace(/\s+/g, ' ').trim();
|
|
377
|
+
if (text === '')
|
|
378
|
+
return '';
|
|
379
|
+
return capTokens(text, cap);
|
|
380
|
+
}
|
|
381
|
+
export function embeddingIndexPath(bundleDir) {
|
|
382
|
+
return join(bundleDir, 'meta', 'embeddings.json');
|
|
383
|
+
}
|
|
384
|
+
function parseEmbeddingIndexEntry(value) {
|
|
385
|
+
if (typeof value !== 'object' || value === null)
|
|
386
|
+
return null;
|
|
387
|
+
const e = value;
|
|
388
|
+
if (typeof e.model !== 'string' || e.model === '')
|
|
389
|
+
return null;
|
|
390
|
+
if (typeof e.dim !== 'number' || !Number.isInteger(e.dim) || e.dim <= 0)
|
|
391
|
+
return null;
|
|
392
|
+
if (!Array.isArray(e.vec) || e.vec.length !== e.dim)
|
|
393
|
+
return null;
|
|
394
|
+
if (!e.vec.every((n) => typeof n === 'number' && Number.isFinite(n)))
|
|
395
|
+
return null;
|
|
396
|
+
return { model: e.model, dim: e.dim, vec: e.vec };
|
|
397
|
+
}
|
|
398
|
+
/** Load meta/embeddings.json from a bundle dir. Missing file, corrupt JSON,
|
|
399
|
+
* or malformed entries are skipped/empty — never throws (a broken index
|
|
400
|
+
* must degrade to the lexical path, not break retrieval). */
|
|
401
|
+
export function loadEmbeddingIndex(bundleDir) {
|
|
402
|
+
let raw;
|
|
403
|
+
try {
|
|
404
|
+
raw = readFileSync(embeddingIndexPath(bundleDir), 'utf8');
|
|
405
|
+
}
|
|
406
|
+
catch {
|
|
407
|
+
return {};
|
|
408
|
+
}
|
|
409
|
+
try {
|
|
410
|
+
const json = JSON.parse(raw);
|
|
411
|
+
if (typeof json !== 'object' || json === null || Array.isArray(json))
|
|
412
|
+
return {};
|
|
413
|
+
const out = {};
|
|
414
|
+
for (const [slug, entry] of Object.entries(json)) {
|
|
415
|
+
if (slug === '')
|
|
416
|
+
continue;
|
|
417
|
+
const parsed = parseEmbeddingIndexEntry(entry);
|
|
418
|
+
if (parsed !== null)
|
|
419
|
+
out[slug] = parsed;
|
|
420
|
+
}
|
|
421
|
+
return out;
|
|
422
|
+
}
|
|
423
|
+
catch {
|
|
424
|
+
return {};
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
/** Persist the index in the canonical shape (the rebuild's write side —
|
|
428
|
+
* embedding/rebuilding itself stays with the caller). Offline rebuild
|
|
429
|
+
* path, not the hot lane. */
|
|
430
|
+
export function saveEmbeddingIndex(bundleDir, index, mkdir = true) {
|
|
431
|
+
if (mkdir)
|
|
432
|
+
mkdirSync(join(bundleDir, 'meta'), { recursive: true });
|
|
433
|
+
writeFileSync(embeddingIndexPath(bundleDir), JSON.stringify(index));
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Freshness audit: which slugs need (re)embedding — missing from the index,
|
|
437
|
+
* built by a different model, or shape-broken (dim/vec disagree). Returns
|
|
438
|
+
* them in roster order; an EMPTY list means the index covers the roster.
|
|
439
|
+
* A '' embedder model means the semantic layer is off → nothing to maintain.
|
|
440
|
+
*
|
|
441
|
+
* NOTE the freshness boundary: content-change invalidation needs a stored
|
|
442
|
+
* content hash, which the pinned {model,dim,vec} entry shape does not carry
|
|
443
|
+
* — the rebuild trigger policy is spec-实现期定 and stays with the caller.
|
|
444
|
+
*/
|
|
445
|
+
export function ensureIndexFresh(index, topics, embedFn) {
|
|
446
|
+
if ((embedFn.model ?? '') === '')
|
|
447
|
+
return [];
|
|
448
|
+
const stale = [];
|
|
449
|
+
for (const t of topics) {
|
|
450
|
+
const e = index[t.slug];
|
|
451
|
+
if (e === undefined || e.model !== embedFn.model || e.dim !== e.vec.length || !(e.dim > 0))
|
|
452
|
+
stale.push(t.slug);
|
|
453
|
+
}
|
|
454
|
+
return stale;
|
|
455
|
+
}
|
|
456
|
+
// --- ③ nearFloor semantic rerank (A4 决议 2, spec §3.1) ---------------------
|
|
457
|
+
// Cosine reordering of the nearFloor band ONLY (0.20–0.30 — the 93%
|
|
458
|
+
// near-miss mass). Thresholds, gateScore and the structural gate are
|
|
459
|
+
// untouched: the reranker reorders and tags, never re-scores.
|
|
460
|
+
export const RERANK_FLOOR = 0.2;
|
|
461
|
+
export const RERANK_CEILING = 0.3;
|
|
462
|
+
export const SEMANTIC_RERANK_REASON = 'semantic-rerank';
|
|
463
|
+
/** Cosine similarity; 0 for dim mismatch or zero vectors (never throws). */
|
|
464
|
+
export function cosineSimilarity(a, b) {
|
|
465
|
+
if (a.length === 0 || a.length !== b.length)
|
|
466
|
+
return 0;
|
|
467
|
+
let dot = 0;
|
|
468
|
+
let na = 0;
|
|
469
|
+
let nb = 0;
|
|
470
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
471
|
+
dot += a[i] * b[i];
|
|
472
|
+
na += a[i] * a[i];
|
|
473
|
+
nb += b[i] * b[i];
|
|
474
|
+
}
|
|
475
|
+
if (na === 0 || nb === 0)
|
|
476
|
+
return 0;
|
|
477
|
+
return dot / Math.sqrt(na * nb);
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Reorder the nearFloor band (0.20–0.30 by gateScore) by cosine similarity
|
|
481
|
+
* against the query vector. Band members with a usable index vector sort by
|
|
482
|
+
* similarity (stable — ties keep input order) and gain the
|
|
483
|
+
* `semantic-rerank` reason plus their similarity; everyone else — out of
|
|
484
|
+
* band, unindexed, model-mismatched — passes through UNCHANGED at the tail
|
|
485
|
+
* in input order. Scores are never touched.
|
|
486
|
+
*
|
|
487
|
+
* Degradation is explicit and total: provider/model unconfigured, probe
|
|
488
|
+
* failure, missing query vector or missing index all return the input as-is
|
|
489
|
+
* with `reranked:false` and a passthroughReason (原序透传,不许静默 — the
|
|
490
|
+
* caller logs it into the degraded status). `queryText` rides the signature
|
|
491
|
+
* for provenance/debug parity with the caller's logging; the v1 math is
|
|
492
|
+
* vector-only.
|
|
493
|
+
*/
|
|
494
|
+
export function rerankNearFloor(queryText, queryVec, nearFloorCandidates, index, opts = {}) {
|
|
495
|
+
void queryText; // v1 math is vector-only — see doc above
|
|
496
|
+
if (nearFloorCandidates.length === 0) {
|
|
497
|
+
return { hits: [], reranked: false, passthroughReason: 'no-candidates' };
|
|
498
|
+
}
|
|
499
|
+
if (!Array.isArray(queryVec) || queryVec.length === 0) {
|
|
500
|
+
return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'no-query-vector' };
|
|
501
|
+
}
|
|
502
|
+
if (Object.keys(index).length === 0) {
|
|
503
|
+
return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'empty-index' };
|
|
504
|
+
}
|
|
505
|
+
const floor = opts.floor ?? RERANK_FLOOR;
|
|
506
|
+
const ceiling = opts.ceiling ?? RERANK_CEILING;
|
|
507
|
+
const simOf = new Map();
|
|
508
|
+
for (const hit of nearFloorCandidates) {
|
|
509
|
+
const entry = index[hit.slug];
|
|
510
|
+
const usable = entry !== undefined &&
|
|
511
|
+
entry.dim === entry.vec.length &&
|
|
512
|
+
entry.dim > 0 &&
|
|
513
|
+
entry.dim === queryVec.length &&
|
|
514
|
+
(opts.expectedModel === undefined || entry.model === opts.expectedModel);
|
|
515
|
+
simOf.set(hit, usable ? cosineSimilarity(queryVec, entry.vec) : null);
|
|
516
|
+
}
|
|
517
|
+
const band = nearFloorCandidates.filter((h) => {
|
|
518
|
+
const sim = simOf.get(h);
|
|
519
|
+
return sim !== null && sim !== undefined && h.gateScore !== undefined && h.gateScore >= floor && h.gateScore < ceiling;
|
|
520
|
+
});
|
|
521
|
+
if (band.length === 0) {
|
|
522
|
+
return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'no-rerankable-candidates' };
|
|
523
|
+
}
|
|
524
|
+
const rerankedSet = new Set(band);
|
|
525
|
+
const tail = nearFloorCandidates.filter((h) => !rerankedSet.has(h));
|
|
526
|
+
// Stable sort: V8's Array#sort keeps input order for equal similarities.
|
|
527
|
+
const ordered = [...band].sort((x, y) => simOf.get(y) - simOf.get(x));
|
|
528
|
+
return {
|
|
529
|
+
hits: [
|
|
530
|
+
...ordered.map((h) => {
|
|
531
|
+
const sim = simOf.get(h);
|
|
532
|
+
return { ...h, reasons: [...h.reasons, SEMANTIC_RERANK_REASON], similarity: Math.round(sim * 10_000) / 10_000 };
|
|
533
|
+
}),
|
|
534
|
+
...tail.map((h) => ({ ...h })),
|
|
535
|
+
],
|
|
536
|
+
reranked: true,
|
|
537
|
+
};
|
|
538
|
+
}
|
package/lib/service.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { type RetrievableTopic, type SearchOutcome } from './retrieval.ts';
|
|
|
10
10
|
import { type AssembleResult } from './digest.ts';
|
|
11
11
|
import type { BundleStore, Observation, SaveResult } from './store.ts';
|
|
12
12
|
import type { TopicsConfigValue } from './config.ts';
|
|
13
|
-
import { type AggregateStats, type QueryBuildShape, type UsageSignal } from './ilog.ts';
|
|
13
|
+
import { type AggregateStats, type QueryBuildShape, type SlowOutcome, type UsageSignal } from './ilog.ts';
|
|
14
14
|
import type { Sync } from './sync.ts';
|
|
15
15
|
export interface RetrieveOutcome {
|
|
16
16
|
outcome: SearchOutcome;
|
|
@@ -33,6 +33,8 @@ export interface SlowDelivery {
|
|
|
33
33
|
queryBuild: QueryBuildShape;
|
|
34
34
|
model: string;
|
|
35
35
|
ms: number;
|
|
36
|
+
/** Producing run's disposition (A4 全留痕) — copied into ilog.slowOutcome. */
|
|
37
|
+
outcome?: SlowOutcome;
|
|
36
38
|
}
|
|
37
39
|
export declare class TopicsService {
|
|
38
40
|
private cache;
|
|
@@ -101,6 +103,18 @@ export declare class TopicsService {
|
|
|
101
103
|
* the slow lane's recall band stays untouched, mirroring usageBoost).
|
|
102
104
|
*/
|
|
103
105
|
private zeroOpenCfg;
|
|
106
|
+
/** mtime-cached view of meta/embeddings.json (sync read — hot-lane caller).
|
|
107
|
+
* Missing/corrupt index = {} → the reranker degrades to lexical order. */
|
|
108
|
+
private embedIndexCache;
|
|
109
|
+
private embeddingIndexCacheGet;
|
|
110
|
+
/** Shadow bypass ledger (spec §8.3): one line per shadow round — feature
|
|
111
|
+
* fields only, never query text (jev-log discipline). Silent on failure;
|
|
112
|
+
* 512KB truncation keeps the newest three quarters. */
|
|
113
|
+
private appendPackageShadowLedger;
|
|
114
|
+
/** Boot-time embedding index maintenance (spec §3: boot 校验+增量重建).
|
|
115
|
+
* Provider '' or packageMode off → no-op (default-off byte-for-byte).
|
|
116
|
+
* Full-roster embed on rebuild (A1 §5: ~9.5s for 137 topics). */
|
|
117
|
+
refreshEmbeddingIndex(): Promise<void>;
|
|
104
118
|
/**
|
|
105
119
|
* Full retrieval round: search → assemble → log (ADR 0006/0007). The
|
|
106
120
|
* injection record is written even for zero-hit rounds — the hit-rate
|
|
@@ -178,6 +192,8 @@ export declare class TopicsService {
|
|
|
178
192
|
depends?: string[];
|
|
179
193
|
openQuestions?: string[];
|
|
180
194
|
impact?: string[];
|
|
195
|
+
/** A2 semantic-link targets; store.saveTopic lands them as body wikilinks. */
|
|
196
|
+
links?: string[];
|
|
181
197
|
status?: okf.TopicStatus;
|
|
182
198
|
conclusion?: string;
|
|
183
199
|
recommendations?: string;
|
package/lib/service.js
CHANGED
|
@@ -6,11 +6,13 @@
|
|
|
6
6
|
* @module service
|
|
7
7
|
*/
|
|
8
8
|
import { readFile, stat } from 'node:fs/promises';
|
|
9
|
-
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
9
|
+
import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
10
10
|
import { join } from 'node:path';
|
|
11
11
|
import * as okf from "./okf.js";
|
|
12
|
-
import { searchTopics, scoreTopic, passesGate, tokenize } from "./retrieval.js";
|
|
13
|
-
import { assembleInjection, assemblePointer, POINTER_PER_TOPIC, POINTER_TOTAL } from "./digest.js";
|
|
12
|
+
import { searchTopics, scoreTopic, passesGate, tokenize, sanitizeQuery, loadEmbeddingIndex, rerankNearFloor, ensureIndexFresh, saveEmbeddingIndex } from "./retrieval.js";
|
|
13
|
+
import { assembleInjection, assemblePointer, assemblePackage, POINTER_PER_TOPIC, POINTER_TOTAL } from "./digest.js";
|
|
14
|
+
import { embedTexts, embedQuerySync, embedderWith, resolveEmbedModel } from "./embedding.js";
|
|
15
|
+
import { resolveBundleRoot } from "./paths.js";
|
|
14
16
|
import { aggregateStats, querySample, aggregateUsage, aggregateZeroOpen, ZERO_OPEN_TAIL, USAGE_WINDOW_DAYS, attributeOpen, OPEN_ATTRIBUTION_TAIL } from "./ilog.js";
|
|
15
17
|
import { fileHistory, fileAtRev } from "./git.js";
|
|
16
18
|
/** Tolerant JSONL line read: undefined = unreadable file. Unlike an empty
|
|
@@ -269,6 +271,73 @@ export class TopicsService {
|
|
|
269
271
|
return {};
|
|
270
272
|
return { zeroOpenDecay: true, zeroOpenSlugs: this.zeroOpenSlugsSync() };
|
|
271
273
|
}
|
|
274
|
+
/** mtime-cached view of meta/embeddings.json (sync read — hot-lane caller).
|
|
275
|
+
* Missing/corrupt index = {} → the reranker degrades to lexical order. */
|
|
276
|
+
embedIndexCache;
|
|
277
|
+
embeddingIndexCacheGet() {
|
|
278
|
+
const root = resolveBundleRoot();
|
|
279
|
+
const file = join(root, 'meta', 'embeddings.json');
|
|
280
|
+
try {
|
|
281
|
+
const st = statSync(file);
|
|
282
|
+
if (this.embedIndexCache !== undefined && this.embedIndexCache.mtimeMs === st.mtimeMs && this.embedIndexCache.size === st.size) {
|
|
283
|
+
return this.embedIndexCache.index;
|
|
284
|
+
}
|
|
285
|
+
const index = loadEmbeddingIndex(root);
|
|
286
|
+
this.embedIndexCache = { mtimeMs: st.mtimeMs, size: st.size, index };
|
|
287
|
+
return index;
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
return this.embedIndexCache?.index ?? {};
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
/** Shadow bypass ledger (spec §8.3): one line per shadow round — feature
|
|
294
|
+
* fields only, never query text (jev-log discipline). Silent on failure;
|
|
295
|
+
* 512KB truncation keeps the newest three quarters. */
|
|
296
|
+
appendPackageShadowLedger(entry) {
|
|
297
|
+
try {
|
|
298
|
+
const file = join(resolveBundleRoot(), 'meta', 'package-shadow.jsonl');
|
|
299
|
+
let lines = [];
|
|
300
|
+
try {
|
|
301
|
+
const st = statSync(file);
|
|
302
|
+
if (st.size > 512 * 1024) {
|
|
303
|
+
const all = readFileSync(file, 'utf8').split('\n').filter((l) => l !== '');
|
|
304
|
+
lines = all.slice(Math.floor(all.length / 4));
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
catch { /* first write */ }
|
|
308
|
+
lines.push(JSON.stringify(entry));
|
|
309
|
+
writeFileSync(file, lines.join('\n') + '\n');
|
|
310
|
+
}
|
|
311
|
+
catch { /* ledger must never break the lane */ }
|
|
312
|
+
}
|
|
313
|
+
/** Boot-time embedding index maintenance (spec §3: boot 校验+增量重建).
|
|
314
|
+
* Provider '' or packageMode off → no-op (default-off byte-for-byte).
|
|
315
|
+
* Full-roster embed on rebuild (A1 §5: ~9.5s for 137 topics). */
|
|
316
|
+
async refreshEmbeddingIndex() {
|
|
317
|
+
const provider = (this.cfg.embeddingProvider ?? '').trim();
|
|
318
|
+
if (provider === '' || this.cfg.packageMode === undefined || this.cfg.packageMode === 'off')
|
|
319
|
+
return;
|
|
320
|
+
const embedder = embedderWith({ provider, model: this.cfg.embeddingModel });
|
|
321
|
+
const topics = await this.roster();
|
|
322
|
+
const root = resolveBundleRoot();
|
|
323
|
+
const index = loadEmbeddingIndex(root);
|
|
324
|
+
const stale = ensureIndexFresh(index, topics, embedder);
|
|
325
|
+
if (stale.length === 0)
|
|
326
|
+
return;
|
|
327
|
+
const docs = await Promise.all(stale.map(async (slug) => (await this.readDoc(slug)) ?? ''));
|
|
328
|
+
const texts = stale.map((slug, i) => `${slug}\n${docs[i] ?? ''}`);
|
|
329
|
+
const r = await embedTexts(texts, { provider, model: this.cfg.embeddingModel });
|
|
330
|
+
if (!r.ok)
|
|
331
|
+
return;
|
|
332
|
+
const next = { ...index };
|
|
333
|
+
stale.forEach((slug, i) => {
|
|
334
|
+
const vec = r.vectors[i];
|
|
335
|
+
if (vec !== undefined)
|
|
336
|
+
next[slug] = { model: r.model, dim: vec.length, vec };
|
|
337
|
+
});
|
|
338
|
+
saveEmbeddingIndex(root, next);
|
|
339
|
+
this.embedIndexCache = undefined;
|
|
340
|
+
}
|
|
272
341
|
/**
|
|
273
342
|
* Full retrieval round: search → assemble → log (ADR 0006/0007). The
|
|
274
343
|
* injection record is written even for zero-hit rounds — the hit-rate
|
|
@@ -389,8 +458,12 @@ export class TopicsService {
|
|
|
389
458
|
}
|
|
390
459
|
return { text: '', outcome: empty, deduped: [], echoed: [], included: [], slowIncluded: [] };
|
|
391
460
|
}
|
|
461
|
+
// A-line gate (A3/A4): query cleaning + package + semantic rerank ride
|
|
462
|
+
// behind packageMode (default off = byte-for-byte legacy path).
|
|
463
|
+
const pkgMode = cfg.packageMode === 'shadow' || cfg.packageMode === 'on' ? cfg.packageMode : undefined;
|
|
464
|
+
const effectiveQuery = pkgMode !== undefined ? sanitizeQuery(query) : query;
|
|
392
465
|
const conflicts = this.store.getConflictsSync();
|
|
393
|
-
const outcome = searchTopics(
|
|
466
|
+
const outcome = searchTopics(effectiveQuery, roster, {
|
|
394
467
|
threshold: cfg.matchThreshold,
|
|
395
468
|
topK: cfg.topK,
|
|
396
469
|
tagBoost: cfg.tagBoost,
|
|
@@ -401,6 +474,36 @@ export class TopicsService {
|
|
|
401
474
|
usage: this.usageSignalsSync(),
|
|
402
475
|
...this.zeroOpenCfg(),
|
|
403
476
|
});
|
|
477
|
+
// ---- A-line semantic rerank (A4 决议 2, spec §3.1) ----
|
|
478
|
+
// Reorder ONLY the 0.20–0.30 nearFloor band (the 93% near-miss mass);
|
|
479
|
+
// thresholds/gateScore/hits untouched. Any failure degrades to the
|
|
480
|
+
// original order — the fast lane never blocks on the semantic layer.
|
|
481
|
+
let nearFloorRerank;
|
|
482
|
+
if (pkgMode !== undefined) {
|
|
483
|
+
const bandLo = 0.2;
|
|
484
|
+
const band = outcome.nearMisses.filter((h) => h.score >= bandLo && h.score < cfg.matchThreshold);
|
|
485
|
+
const orderBefore = band.map((h) => h.slug);
|
|
486
|
+
if (band.length > 1) {
|
|
487
|
+
const index = this.embeddingIndexCacheGet();
|
|
488
|
+
const models = new Set(Object.values(index).map((e) => e.model));
|
|
489
|
+
const embedModel = resolveEmbedModel(cfg.embeddingModel);
|
|
490
|
+
if (models.size <= 1 && (models.size === 0 || models.has(embedModel))) {
|
|
491
|
+
const qv = embedQuerySync(effectiveQuery, { provider: cfg.embeddingProvider, model: cfg.embeddingModel });
|
|
492
|
+
const rr = rerankNearFloor(effectiveQuery, qv.ok ? qv.vector : null, band, index);
|
|
493
|
+
if (rr.reranked) {
|
|
494
|
+
const inBand = (h) => h.score >= bandLo && h.score < cfg.matchThreshold;
|
|
495
|
+
const rrSlugs = new Set(rr.hits.map((h) => h.slug));
|
|
496
|
+
const merged = [...rr.hits, ...band.filter((h) => !rrSlugs.has(h.slug))];
|
|
497
|
+
let mi = 0;
|
|
498
|
+
outcome.nearMisses = outcome.nearMisses.map((h) => (inBand(h) ? merged[mi++] : h));
|
|
499
|
+
nearFloorRerank = { reranked: true, orderBefore, orderAfter: merged.map((h) => h.slug) };
|
|
500
|
+
}
|
|
501
|
+
else {
|
|
502
|
+
nearFloorRerank = { reranked: false, passthroughReason: rr.passthroughReason, orderBefore, orderAfter: orderBefore };
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
404
507
|
const exclude = dedup?.exclude;
|
|
405
508
|
const echo = dedup?.echo;
|
|
406
509
|
const deduped = exclude === undefined ? [] : outcome.hits.filter((h) => exclude.has(h.slug)).map((h) => h.slug);
|
|
@@ -431,7 +534,7 @@ export class TopicsService {
|
|
|
431
534
|
let slowIncluded = [];
|
|
432
535
|
const consumedAt = new Date().toISOString();
|
|
433
536
|
if (slow !== undefined) {
|
|
434
|
-
const queryTokens = new Set(tokenize(
|
|
537
|
+
const queryTokens = new Set(tokenize(effectiveQuery));
|
|
435
538
|
for (const item of slow.items) {
|
|
436
539
|
if (exclude?.has(item.slug)) {
|
|
437
540
|
if (!deduped.includes(item.slug))
|
|
@@ -482,9 +585,70 @@ export class TopicsService {
|
|
|
482
585
|
const pointerMode = cfg.injectMode !== 'digest';
|
|
483
586
|
// Pointer budget: per-entry fixed at the design's 80, total clamped to the
|
|
484
587
|
// fast-lane upper bound of 600 (I4) — totalBudget stays tunable downward.
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
588
|
+
let packageSkipped = false;
|
|
589
|
+
let packageShadow;
|
|
590
|
+
let injection;
|
|
591
|
+
if (!pointerMode) {
|
|
592
|
+
injection = assembleInjection(entries, { perTopicBudget: cfg.perTopicBudget, totalBudget: cfg.totalBudget }, slowPointers);
|
|
593
|
+
}
|
|
594
|
+
else if (pkgMode === undefined) {
|
|
595
|
+
injection = assemblePointer(entries, { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) }, slowPointers);
|
|
596
|
+
}
|
|
597
|
+
else {
|
|
598
|
+
// A-line relevance package (A3, spec §2): direct = lexical hits,
|
|
599
|
+
// neighborhood = graph hits + slow picks, weak = reranked band head.
|
|
600
|
+
const budget = { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) };
|
|
601
|
+
const legacy = assemblePointer(entries, budget, slowPointers);
|
|
602
|
+
const direct = [];
|
|
603
|
+
const neighborhood = [];
|
|
604
|
+
for (const e of entries) {
|
|
605
|
+
const meta = bySlug.get(e.slug);
|
|
606
|
+
const cand = {
|
|
607
|
+
slug: e.slug,
|
|
608
|
+
title: meta?.title ?? e.slug,
|
|
609
|
+
status: meta?.status ?? 'draft',
|
|
610
|
+
description: meta?.description,
|
|
611
|
+
conclusion: firstContentLine(e.doc.body),
|
|
612
|
+
score: e.hit.score,
|
|
613
|
+
reasons: e.hit.reasons,
|
|
614
|
+
via: e.hit.viaGraph ? 'graph' : undefined,
|
|
615
|
+
};
|
|
616
|
+
(e.hit.viaGraph ? neighborhood : direct).push(cand);
|
|
617
|
+
}
|
|
618
|
+
for (const sp of slowPointers) {
|
|
619
|
+
neighborhood.push({ slug: sp.slug, title: sp.title, status: sp.status, description: sp.description, score: 0, reasons: [], via: 'slow-pick' });
|
|
620
|
+
}
|
|
621
|
+
const weak = outcome.nearMisses
|
|
622
|
+
.filter((h) => h.score >= 0.2 && h.score < cfg.matchThreshold)
|
|
623
|
+
.slice(0, 2)
|
|
624
|
+
.map((h) => {
|
|
625
|
+
const meta = bySlug.get(h.slug);
|
|
626
|
+
return { slug: h.slug, title: meta?.title ?? h.slug, status: meta?.status ?? 'draft', score: h.score, reasons: h.reasons, via: 'weak' };
|
|
627
|
+
});
|
|
628
|
+
const usage = this.usageSignalsSync();
|
|
629
|
+
const usageBySlug = new Map([...usage].map(([slug, sig]) => [slug, sig.opens * 3 + sig.hits]));
|
|
630
|
+
const pkgInput = { query: effectiveQuery, queryTokens: tokenize(effectiveQuery), direct, neighborhood, weak, usageBySlug };
|
|
631
|
+
const pkg = assemblePackage(pkgInput);
|
|
632
|
+
if (pkgMode === 'shadow') {
|
|
633
|
+
packageShadow = pkg;
|
|
634
|
+
injection = legacy;
|
|
635
|
+
}
|
|
636
|
+
else if (pkg === null) {
|
|
637
|
+
// Trivial-query downgrade (A3 决议 5): emit nothing, log the why.
|
|
638
|
+
packageSkipped = true;
|
|
639
|
+
injection = { text: '', usedTokens: 0, included: [], dropped: [] };
|
|
640
|
+
}
|
|
641
|
+
else {
|
|
642
|
+
const slowSet = new Set(slowPointers.map((sp) => sp.slug));
|
|
643
|
+
injection = {
|
|
644
|
+
text: pkg.text,
|
|
645
|
+
usedTokens: pkg.usedTokens,
|
|
646
|
+
included: pkg.included,
|
|
647
|
+
dropped: pkg.dropped,
|
|
648
|
+
slowIncluded: pkg.included.filter((slug) => slowSet.has(slug)),
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
}
|
|
488
652
|
slowIncluded = injection.slowIncluded ?? [];
|
|
489
653
|
for (const pointer of slowPointers) {
|
|
490
654
|
if (slowIncluded.includes(pointer.slug))
|
|
@@ -492,8 +656,8 @@ export class TopicsService {
|
|
|
492
656
|
}
|
|
493
657
|
const record = {
|
|
494
658
|
at: new Date().toISOString(),
|
|
495
|
-
queryTokenCount: tokenize(
|
|
496
|
-
querySample: querySample(
|
|
659
|
+
queryTokenCount: tokenize(effectiveQuery).length,
|
|
660
|
+
querySample: querySample(effectiveQuery),
|
|
497
661
|
rosterSize: outcome.rosterSize,
|
|
498
662
|
hits: outcome.hits.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons, viaGraph: h.viaGraph, strong: h.strong, bodyHits: h.bodyHits })),
|
|
499
663
|
// Reasons ride along: the gate-blocked marker is the replay evidence
|
|
@@ -508,10 +672,28 @@ export class TopicsService {
|
|
|
508
672
|
record.deduped = deduped;
|
|
509
673
|
if (echoed.length > 0)
|
|
510
674
|
record.echoed = echoed;
|
|
675
|
+
if (packageSkipped)
|
|
676
|
+
record.why = 'package-skipped-trivial';
|
|
511
677
|
if (unreadable.length > 0) {
|
|
512
678
|
record.dropped = [...(record.dropped ?? []), ...unreadable.map((slug) => ({ slug, reason: 'doc-unreadable' }))];
|
|
513
679
|
}
|
|
680
|
+
if (pkgMode === 'shadow' && pointerMode) {
|
|
681
|
+
this.appendPackageShadowLedger({
|
|
682
|
+
at: record.at,
|
|
683
|
+
...(sessionId !== undefined ? { sessionId } : {}),
|
|
684
|
+
queryTokenCount: record.queryTokenCount,
|
|
685
|
+
emitted: { tokens: injection.usedTokens, included: injection.included },
|
|
686
|
+
package: packageShadow === undefined
|
|
687
|
+
? null
|
|
688
|
+
: packageShadow === null
|
|
689
|
+
? { skipped: 'trivial-or-empty' }
|
|
690
|
+
: { tokens: packageShadow.usedTokens, included: packageShadow.included, layers: packageShadow.layers },
|
|
691
|
+
nearFloorRerank: nearFloorRerank ?? null,
|
|
692
|
+
});
|
|
693
|
+
}
|
|
514
694
|
if (slow !== undefined) {
|
|
695
|
+
if (slow.outcome !== undefined)
|
|
696
|
+
record.slowOutcome = slow.outcome;
|
|
515
697
|
// Any round whose pending was consumed is a slow-participating round —
|
|
516
698
|
// lane never stays undefined when the lane family is present (a
|
|
517
699
|
// delivered pick with fast pointers is 'mixed'; a consumed-but-empty
|
|
@@ -718,7 +900,7 @@ export class TopicsService {
|
|
|
718
900
|
fm.triggers = triggers;
|
|
719
901
|
if (existing !== undefined && existing.fm.verified !== undefined)
|
|
720
902
|
fm.verified = existing.fm.verified;
|
|
721
|
-
const result = await this.store.saveTopic({ slug, doc: { fm, body } }, { message: existing === undefined ? `topics(topic): create ${slug}` : `topics(topic): update ${slug}`, created: existing === undefined });
|
|
903
|
+
const result = await this.store.saveTopic({ slug, doc: { fm, body }, links: input.links }, { message: existing === undefined ? `topics(topic): create ${slug}` : `topics(topic): update ${slug}`, created: existing === undefined });
|
|
722
904
|
this.invalidate();
|
|
723
905
|
void this.sync?.schedulePush();
|
|
724
906
|
return { ...result, slug: result.slug };
|
|
@@ -805,6 +987,15 @@ function dedupePaths(paths) {
|
|
|
805
987
|
}
|
|
806
988
|
return out;
|
|
807
989
|
}
|
|
990
|
+
function firstContentLine(doc) {
|
|
991
|
+
for (const line of doc.split('\n')) {
|
|
992
|
+
const t = line.trim();
|
|
993
|
+
if (t === '' || t.startsWith('#'))
|
|
994
|
+
continue;
|
|
995
|
+
return t.slice(0, 200);
|
|
996
|
+
}
|
|
997
|
+
return '';
|
|
998
|
+
}
|
|
808
999
|
function sanitizeTriggers(triggers) {
|
|
809
1000
|
const out = [];
|
|
810
1001
|
for (const t of triggers) {
|
package/lib/store.d.ts
CHANGED
|
@@ -110,10 +110,19 @@ export declare class BundleStore {
|
|
|
110
110
|
* Write a topic (atomic), regenerate the index, and commit both — one
|
|
111
111
|
* commit per conclusion change (ADR 0003). `slug` in the doc's depends is
|
|
112
112
|
* NOT normalized here; callers pass bundle-relative paths.
|
|
113
|
+
*
|
|
114
|
+
* `links` (A2, optional): whitelist-surviving semantic targets from the
|
|
115
|
+
* distill lane. Applied INSIDE this one write transaction — the topic and
|
|
116
|
+
* its edges land together (先落 topic 再建边); the Links section lands as
|
|
117
|
+
* body wikilinks and the index/backlinks regeneration below (same
|
|
118
|
+
* transaction) recomputes the edge index over them. Normalized and
|
|
119
|
+
* self/dedup-guarded here too, so any caller benefits from the same guard
|
|
120
|
+
* the distill lane already ran.
|
|
113
121
|
*/
|
|
114
122
|
saveTopic(input: {
|
|
115
123
|
slug: string;
|
|
116
124
|
doc: okf.TopicDoc;
|
|
125
|
+
links?: readonly string[];
|
|
117
126
|
}, opts: {
|
|
118
127
|
message: string;
|
|
119
128
|
actor?: string;
|