@aiwayds/dsh-topics-memory 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/README.zh.md +5 -0
- package/lib/commands.js +27 -11
- package/lib/config.d.ts +54 -1
- package/lib/config.js +38 -1
- package/lib/digest.d.ts +73 -0
- package/lib/digest.js +384 -0
- package/lib/distill.d.ts +39 -0
- package/lib/distill.js +250 -15
- package/lib/embedding.d.ts +113 -0
- package/lib/embedding.js +208 -0
- package/lib/ilog.d.ts +145 -0
- package/lib/ilog.js +236 -0
- package/lib/index.d.ts +8 -0
- package/lib/index.js +23 -6
- package/lib/pipeline-events.d.ts +36 -0
- package/lib/pipeline-events.js +121 -0
- package/lib/quality.d.ts +47 -7
- package/lib/quality.js +124 -20
- package/lib/retrieval.d.ts +119 -0
- package/lib/retrieval.js +290 -0
- package/lib/service.d.ts +60 -3
- package/lib/service.js +320 -44
- package/lib/store.d.ts +15 -12
- package/lib/store.js +43 -3
- package/lib/tools.js +6 -2
- package/package.json +2 -2
- package/src/commands.ts +35 -16
- package/src/config.ts +47 -1
- package/src/digest.ts +416 -0
- package/src/distill.ts +275 -14
- package/src/embedding.ts +294 -0
- package/src/ilog.ts +306 -0
- package/src/index.ts +23 -5
- package/src/pipeline-events.ts +132 -0
- package/src/quality.ts +145 -21
- package/src/retrieval.ts +357 -0
- package/src/service.ts +326 -52
- package/src/store.ts +46 -10
- package/src/tools.ts +6 -2
package/lib/retrieval.d.ts
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* @module retrieval
|
|
13
13
|
*/
|
|
14
14
|
import type { TopicMeta } from './store.ts';
|
|
15
|
+
import type { EmbedResult } from './embedding.ts';
|
|
15
16
|
export interface RetrievableTopic extends TopicMeta {
|
|
16
17
|
/** `# Conclusion` section text (may be empty). */
|
|
17
18
|
conclusion: string;
|
|
@@ -48,10 +49,23 @@ export interface RetrievalConfig {
|
|
|
48
49
|
hits: number;
|
|
49
50
|
opens: number;
|
|
50
51
|
}>;
|
|
52
|
+
/**
|
|
53
|
+
* Zero-open decay (v5, default off — config `zeroOpenDecay`): halve the
|
|
54
|
+
* score of slugs on the zero-open list (≥5 rendered injections in the
|
|
55
|
+
* rolling 30 days, zero opens ever) at score-finalization time, BEFORE
|
|
56
|
+
* the threshold cut — a decayed candidate that drops below the threshold
|
|
57
|
+
* lands in nearMisses with the `zero-open-decay` reason, so the log
|
|
58
|
+
* shows why it stopped injecting. Absent/off = zero behavior change.
|
|
59
|
+
*/
|
|
60
|
+
zeroOpenDecay?: boolean;
|
|
61
|
+
/** Zero-open slugs for this round (service-built from the ilog tail). */
|
|
62
|
+
zeroOpenSlugs?: ReadonlySet<string>;
|
|
51
63
|
}
|
|
52
64
|
/** Hard cap for the usage bonus — below threshold 0.3 so usage can never
|
|
53
65
|
* carry a candidate across the gate on its own (ADR 0015). */
|
|
54
66
|
export declare const USAGE_BOOST_CAP = 0.2;
|
|
67
|
+
/** Score multiplier for zero-open slugs when zeroOpenDecay is on (v5). */
|
|
68
|
+
export declare const ZERO_OPEN_DECAY_FACTOR = 0.5;
|
|
55
69
|
export declare const DEFAULT_RETRIEVAL: RetrievalConfig;
|
|
56
70
|
/** Trigger hits outrank every other lexical field (pi-llm-wiki: highest weight). */
|
|
57
71
|
export declare const TRIGGER_WEIGHT = 5;
|
|
@@ -107,3 +121,108 @@ export declare function passesGate(s: {
|
|
|
107
121
|
* evidence shows what the gate is costing.
|
|
108
122
|
*/
|
|
109
123
|
export declare function searchTopics(query: string, roster: readonly RetrievableTopic[], cfg?: Partial<RetrievalConfig>): SearchOutcome;
|
|
124
|
+
/**
|
|
125
|
+
* Token cap in tokenize() units — the SAME unit injections.jsonl logs as
|
|
126
|
+
* queryTokenCount (p90=462 baseline; real-query p50=153 stays untouched).
|
|
127
|
+
* Raised or lowered via SanitizeOptions.maxTokens.
|
|
128
|
+
*/
|
|
129
|
+
export declare const QUERY_TOKEN_CAP = 96;
|
|
130
|
+
/**
|
|
131
|
+
* Lines longer than this are folded to their head (log-paste guard). Sized
|
|
132
|
+
* so the 96-token cap is the binding constraint, not the fold: 96 latin
|
|
133
|
+
* tokens ≈ 650+ chars, so a legitimate long single-line query passes the
|
|
134
|
+
* fold intact and only the token cap trims it. (实现期定 constant.)
|
|
135
|
+
*/
|
|
136
|
+
export declare const MAX_LINE_CHARS = 720;
|
|
137
|
+
export interface SanitizeOptions {
|
|
138
|
+
maxTokens?: number;
|
|
139
|
+
maxLineChars?: number;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Sanitize a raw user query for retrieval (spec §3.3): strip fenced code
|
|
143
|
+
* blocks and URLs, fold line-leading shell prompts, fold over-long lines to
|
|
144
|
+
* their head, cap the distinct-token count. Returns '' when nothing lexical
|
|
145
|
+
* survives — the caller treats that as a zero-token query (natural skip).
|
|
146
|
+
* Pure function; the hot lane only reaches it behind packageMode.
|
|
147
|
+
*/
|
|
148
|
+
export declare function sanitizeQuery(raw: string, opts?: SanitizeOptions): string;
|
|
149
|
+
export interface EmbeddingIndexEntry {
|
|
150
|
+
model: string;
|
|
151
|
+
dim: number;
|
|
152
|
+
vec: number[];
|
|
153
|
+
}
|
|
154
|
+
/** slug -> entry, exactly as persisted in meta/embeddings.json. */
|
|
155
|
+
export type EmbeddingIndex = Record<string, EmbeddingIndexEntry>;
|
|
156
|
+
export declare function embeddingIndexPath(bundleDir: string): string;
|
|
157
|
+
/** Load meta/embeddings.json from a bundle dir. Missing file, corrupt JSON,
|
|
158
|
+
* or malformed entries are skipped/empty — never throws (a broken index
|
|
159
|
+
* must degrade to the lexical path, not break retrieval). */
|
|
160
|
+
export declare function loadEmbeddingIndex(bundleDir: string): EmbeddingIndex;
|
|
161
|
+
/** Persist the index in the canonical shape (the rebuild's write side —
|
|
162
|
+
* embedding/rebuilding itself stays with the caller). Offline rebuild
|
|
163
|
+
* path, not the hot lane. */
|
|
164
|
+
export declare function saveEmbeddingIndex(bundleDir: string, index: EmbeddingIndex, mkdir?: boolean): void;
|
|
165
|
+
/**
|
|
166
|
+
* An embed function bound to a config, carrying the RESOLVED model identity
|
|
167
|
+
* the index must match (build once via embedding.ts's `embedderWith`).
|
|
168
|
+
* ensureIndexFresh reads `.model` only — it never calls the function; the
|
|
169
|
+
* rebuild is the caller's job.
|
|
170
|
+
*/
|
|
171
|
+
export type EmbedderFn = ((texts: readonly string[]) => Promise<EmbedResult>) & {
|
|
172
|
+
model: string;
|
|
173
|
+
};
|
|
174
|
+
/**
|
|
175
|
+
* Freshness audit: which slugs need (re)embedding — missing from the index,
|
|
176
|
+
* built by a different model, or shape-broken (dim/vec disagree). Returns
|
|
177
|
+
* them in roster order; an EMPTY list means the index covers the roster.
|
|
178
|
+
* A '' embedder model means the semantic layer is off → nothing to maintain.
|
|
179
|
+
*
|
|
180
|
+
* NOTE the freshness boundary: content-change invalidation needs a stored
|
|
181
|
+
* content hash, which the pinned {model,dim,vec} entry shape does not carry
|
|
182
|
+
* — the rebuild trigger policy is spec-实现期定 and stays with the caller.
|
|
183
|
+
*/
|
|
184
|
+
export declare function ensureIndexFresh(index: EmbeddingIndex, topics: readonly {
|
|
185
|
+
slug: string;
|
|
186
|
+
}[], embedFn: EmbedderFn): string[];
|
|
187
|
+
export declare const RERANK_FLOOR = 0.2;
|
|
188
|
+
export declare const RERANK_CEILING = 0.3;
|
|
189
|
+
export declare const SEMANTIC_RERANK_REASON = "semantic-rerank";
|
|
190
|
+
export interface RerankedHit extends SearchHit {
|
|
191
|
+
/** Cosine(query, slug-vector), rounded 4dp. Present only on reranked
|
|
192
|
+
* hits; absent = the candidate rode through unchanged. */
|
|
193
|
+
similarity?: number;
|
|
194
|
+
}
|
|
195
|
+
export type RerankPassthroughReason = 'no-candidates' | 'no-query-vector' | 'empty-index' | 'no-rerankable-candidates';
|
|
196
|
+
export interface RerankOptions {
|
|
197
|
+
/** Expected index model; entries built by another model count as
|
|
198
|
+
* vector-less (they ride through at the band tail). */
|
|
199
|
+
expectedModel?: string;
|
|
200
|
+
floor?: number;
|
|
201
|
+
ceiling?: number;
|
|
202
|
+
}
|
|
203
|
+
export interface RerankResult {
|
|
204
|
+
/** Candidates in final order. On passthrough: the input array, same
|
|
205
|
+
* order, no tags — byte-identical ordering, explicit why. */
|
|
206
|
+
hits: RerankedHit[];
|
|
207
|
+
/** true = the semantic layer engaged (order may or may not have moved). */
|
|
208
|
+
reranked: boolean;
|
|
209
|
+
passthroughReason?: RerankPassthroughReason;
|
|
210
|
+
}
|
|
211
|
+
/** Cosine similarity; 0 for dim mismatch or zero vectors (never throws). */
|
|
212
|
+
export declare function cosineSimilarity(a: readonly number[], b: readonly number[]): number;
|
|
213
|
+
/**
|
|
214
|
+
* Reorder the nearFloor band (0.20–0.30 by gateScore) by cosine similarity
|
|
215
|
+
* against the query vector. Band members with a usable index vector sort by
|
|
216
|
+
* similarity (stable — ties keep input order) and gain the
|
|
217
|
+
* `semantic-rerank` reason plus their similarity; everyone else — out of
|
|
218
|
+
* band, unindexed, model-mismatched — passes through UNCHANGED at the tail
|
|
219
|
+
* in input order. Scores are never touched.
|
|
220
|
+
*
|
|
221
|
+
* Degradation is explicit and total: provider/model unconfigured, probe
|
|
222
|
+
* failure, missing query vector or missing index all return the input as-is
|
|
223
|
+
* with `reranked:false` and a passthroughReason (原序透传,不许静默 — the
|
|
224
|
+
* caller logs it into the degraded status). `queryText` rides the signature
|
|
225
|
+
* for provenance/debug parity with the caller's logging; the v1 math is
|
|
226
|
+
* vector-only.
|
|
227
|
+
*/
|
|
228
|
+
export declare function rerankNearFloor(queryText: string, queryVec: readonly number[] | null | undefined, nearFloorCandidates: readonly SearchHit[], index: EmbeddingIndex, opts?: RerankOptions): RerankResult;
|
package/lib/retrieval.js
CHANGED
|
@@ -11,10 +11,14 @@
|
|
|
11
11
|
*
|
|
12
12
|
* @module retrieval
|
|
13
13
|
*/
|
|
14
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { join } from 'node:path';
|
|
14
16
|
import { dependsSlugs } from "./okf.js";
|
|
15
17
|
/** Hard cap for the usage bonus — below threshold 0.3 so usage can never
|
|
16
18
|
* carry a candidate across the gate on its own (ADR 0015). */
|
|
17
19
|
export const USAGE_BOOST_CAP = 0.2;
|
|
20
|
+
/** Score multiplier for zero-open slugs when zeroOpenDecay is on (v5). */
|
|
21
|
+
export const ZERO_OPEN_DECAY_FACTOR = 0.5;
|
|
18
22
|
export const DEFAULT_RETRIEVAL = {
|
|
19
23
|
threshold: 0.3,
|
|
20
24
|
topK: 4,
|
|
@@ -149,6 +153,14 @@ export function scoreTopic(queryTokens, topic, cfg) {
|
|
|
149
153
|
score *= 0.3;
|
|
150
154
|
reasons.push('conflicted-demoted');
|
|
151
155
|
}
|
|
156
|
+
// Zero-open decay (v5): same demotion family as conflicts — applied to
|
|
157
|
+
// the content score BEFORE gateScore is taken, so the threshold cut and
|
|
158
|
+
// the near-miss band both see the halved score; recency still rides
|
|
159
|
+
// outside it as the ranking tiebreaker.
|
|
160
|
+
if (cfg.zeroOpenDecay === true && cfg.zeroOpenSlugs?.has(topic.slug) === true) {
|
|
161
|
+
score *= ZERO_OPEN_DECAY_FACTOR;
|
|
162
|
+
reasons.push('zero-open-decay');
|
|
163
|
+
}
|
|
152
164
|
const gateScore = score;
|
|
153
165
|
score += recencyBonus;
|
|
154
166
|
const bodySet = new Set([...description, ...conclusion]);
|
|
@@ -246,3 +258,281 @@ export function searchTopics(query, roster, cfg = {}) {
|
|
|
246
258
|
}
|
|
247
259
|
return { hits, nearMisses, rosterSize: roster.length };
|
|
248
260
|
}
|
|
261
|
+
// ---------------------------------------------------------------------------
|
|
262
|
+
// Injection-quality line (wave/injection-quality, spec §2.1/§3.1/§3.3) —
|
|
263
|
+
// CAPABILITY ONLY. None of this is wired into searchTopics or any service
|
|
264
|
+
// path; the caller gates it on packageMode ≠ 'off' (Phase 2 wiring). With
|
|
265
|
+
// the config keys at their defaults every code path below is unreachable
|
|
266
|
+
// from the hot lane, so default-off behavior stays byte-for-byte.
|
|
267
|
+
// ---------------------------------------------------------------------------
|
|
268
|
+
// --- ① Query sanitization (A4 决议 4, spec §3.3) ---------------------------
|
|
269
|
+
// Pure, LLM-free, millisecond-scale. Treats the queryTokenCount p90=462
|
|
270
|
+
// paste-pollution tail: fenced code blocks, URLs, line-leading shell
|
|
271
|
+
// prompts, over-long lines, and a hard token cap.
|
|
272
|
+
/**
|
|
273
|
+
* Token cap in tokenize() units — the SAME unit injections.jsonl logs as
|
|
274
|
+
* queryTokenCount (p90=462 baseline; real-query p50=153 stays untouched).
|
|
275
|
+
* Raised or lowered via SanitizeOptions.maxTokens.
|
|
276
|
+
*/
|
|
277
|
+
export const QUERY_TOKEN_CAP = 96;
|
|
278
|
+
/**
|
|
279
|
+
* Lines longer than this are folded to their head (log-paste guard). Sized
|
|
280
|
+
* so the 96-token cap is the binding constraint, not the fold: 96 latin
|
|
281
|
+
* tokens ≈ 650+ chars, so a legitimate long single-line query passes the
|
|
282
|
+
* fold intact and only the token cap trims it. (实现期定 constant.)
|
|
283
|
+
*/
|
|
284
|
+
export const MAX_LINE_CHARS = 720;
|
|
285
|
+
const FENCE_OPEN = /^[ \t]*(```|~~~)/;
|
|
286
|
+
// Line-leading shell prompt: `user@host:~/repo$ `, `C:\> `, or a bare
|
|
287
|
+
// `$ `/`# `/`% `/`> `/`❯ ` marker. Requires trailing whitespace so words
|
|
288
|
+
// like `$PATH` or `#tag` survive untouched.
|
|
289
|
+
const LINE_PROMPT = /^[ \t]*(?:[\w.@:/\\~-]*[$#%>❯➜])[ \t]+/;
|
|
290
|
+
const URL_RE = /\b(?:https?:\/\/|www\.)[^\s<>"'`)\]]+/gi;
|
|
291
|
+
// Chunks the cap walks: one latin/digit word (or hyphenated compound) per
|
|
292
|
+
// match, or one CJK run (sliced when a run alone busts the cap).
|
|
293
|
+
const CHUNK_RUN = new RegExp(`${WORD_RUN.source}|${CJK_RUN.source}`, 'gu');
|
|
294
|
+
/** Remove fenced code blocks wholesale (markers AND body): fenced code is
|
|
295
|
+
* the paste-pollution payload; the prose around it is the actual query. */
|
|
296
|
+
function stripFencedBlocks(text) {
|
|
297
|
+
const out = [];
|
|
298
|
+
let open = null;
|
|
299
|
+
for (const line of text.split('\n')) {
|
|
300
|
+
const m = line.match(FENCE_OPEN);
|
|
301
|
+
if (open === null) {
|
|
302
|
+
if (m !== null)
|
|
303
|
+
open = m[1];
|
|
304
|
+
else
|
|
305
|
+
out.push(line);
|
|
306
|
+
}
|
|
307
|
+
else if (m !== null && m[1] === open) {
|
|
308
|
+
open = null; // closing fence — block fully dropped
|
|
309
|
+
}
|
|
310
|
+
// inside an open fence: drop the line
|
|
311
|
+
}
|
|
312
|
+
return out.join('\n');
|
|
313
|
+
}
|
|
314
|
+
/** Cap the sanitized text to ≤ cap distinct retrieval tokens. Fast path:
|
|
315
|
+
* at or under the cap the text passes through VERBATIM (no re-chunking —
|
|
316
|
+
* hyphens, punctuation and spacing survive byte-exact; retrieval scores
|
|
317
|
+
* tokens, not bytes, but the string may be echoed into logs/injection).
|
|
318
|
+
* Only an over-cap text rebuilds from chunks: head-first (a query's intent
|
|
319
|
+
* leads; pollution trails), with CJK runs sliced at the character level so
|
|
320
|
+
* a single giant run truncates instead of being dropped whole. */
|
|
321
|
+
function capTokens(text, cap) {
|
|
322
|
+
const total = tokenize(text).length;
|
|
323
|
+
if (total === 0)
|
|
324
|
+
return ''; // nothing lexical survives — caller sees a zero-token query
|
|
325
|
+
if (total <= cap)
|
|
326
|
+
return text;
|
|
327
|
+
const seen = new Set();
|
|
328
|
+
const kept = [];
|
|
329
|
+
for (const chunk of text.match(CHUNK_RUN) ?? []) {
|
|
330
|
+
const fresh = tokenize(chunk).filter((t) => !seen.has(t));
|
|
331
|
+
if (fresh.length === 0)
|
|
332
|
+
continue; // already-covered chunk adds nothing
|
|
333
|
+
if (seen.size + fresh.length <= cap) {
|
|
334
|
+
for (const t of fresh)
|
|
335
|
+
seen.add(t);
|
|
336
|
+
kept.push(chunk);
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
const room = cap - seen.size;
|
|
340
|
+
if (room <= 0)
|
|
341
|
+
break;
|
|
342
|
+
// Over budget: CJK runs truncate exactly (slice room+1 chars → ≤ room
|
|
343
|
+
// fresh bigrams, so the cap invariant holds). Latin chunks are single
|
|
344
|
+
// tokens by construction — they never reach here with room > 0, but if
|
|
345
|
+
// one ever does, dropping it beats busting the cap.
|
|
346
|
+
if (/^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(chunk)) {
|
|
347
|
+
kept.push(chunk.slice(0, room + 1));
|
|
348
|
+
}
|
|
349
|
+
break;
|
|
350
|
+
}
|
|
351
|
+
return kept.join(' ');
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Sanitize a raw user query for retrieval (spec §3.3): strip fenced code
|
|
355
|
+
* blocks and URLs, fold line-leading shell prompts, fold over-long lines to
|
|
356
|
+
* their head, cap the distinct-token count. Returns '' when nothing lexical
|
|
357
|
+
* survives — the caller treats that as a zero-token query (natural skip).
|
|
358
|
+
* Pure function; the hot lane only reaches it behind packageMode.
|
|
359
|
+
*/
|
|
360
|
+
export function sanitizeQuery(raw, opts = {}) {
|
|
361
|
+
const cap = opts.maxTokens ?? QUERY_TOKEN_CAP;
|
|
362
|
+
const maxLineChars = opts.maxLineChars ?? MAX_LINE_CHARS;
|
|
363
|
+
if (raw === '')
|
|
364
|
+
return '';
|
|
365
|
+
let text = stripFencedBlocks(raw);
|
|
366
|
+
text = text.replace(URL_RE, ' ');
|
|
367
|
+
text = text
|
|
368
|
+
.split('\n')
|
|
369
|
+
.map((line) => {
|
|
370
|
+
let l = line.replace(LINE_PROMPT, '');
|
|
371
|
+
if (l.length > maxLineChars)
|
|
372
|
+
l = l.slice(0, maxLineChars);
|
|
373
|
+
return l;
|
|
374
|
+
})
|
|
375
|
+
.join('\n');
|
|
376
|
+
text = text.replace(/\s+/g, ' ').trim();
|
|
377
|
+
if (text === '')
|
|
378
|
+
return '';
|
|
379
|
+
return capTokens(text, cap);
|
|
380
|
+
}
|
|
381
|
+
export function embeddingIndexPath(bundleDir) {
|
|
382
|
+
return join(bundleDir, 'meta', 'embeddings.json');
|
|
383
|
+
}
|
|
384
|
+
function parseEmbeddingIndexEntry(value) {
|
|
385
|
+
if (typeof value !== 'object' || value === null)
|
|
386
|
+
return null;
|
|
387
|
+
const e = value;
|
|
388
|
+
if (typeof e.model !== 'string' || e.model === '')
|
|
389
|
+
return null;
|
|
390
|
+
if (typeof e.dim !== 'number' || !Number.isInteger(e.dim) || e.dim <= 0)
|
|
391
|
+
return null;
|
|
392
|
+
if (!Array.isArray(e.vec) || e.vec.length !== e.dim)
|
|
393
|
+
return null;
|
|
394
|
+
if (!e.vec.every((n) => typeof n === 'number' && Number.isFinite(n)))
|
|
395
|
+
return null;
|
|
396
|
+
return { model: e.model, dim: e.dim, vec: e.vec };
|
|
397
|
+
}
|
|
398
|
+
/** Load meta/embeddings.json from a bundle dir. Missing file, corrupt JSON,
|
|
399
|
+
* or malformed entries are skipped/empty — never throws (a broken index
|
|
400
|
+
* must degrade to the lexical path, not break retrieval). */
|
|
401
|
+
export function loadEmbeddingIndex(bundleDir) {
|
|
402
|
+
let raw;
|
|
403
|
+
try {
|
|
404
|
+
raw = readFileSync(embeddingIndexPath(bundleDir), 'utf8');
|
|
405
|
+
}
|
|
406
|
+
catch {
|
|
407
|
+
return {};
|
|
408
|
+
}
|
|
409
|
+
try {
|
|
410
|
+
const json = JSON.parse(raw);
|
|
411
|
+
if (typeof json !== 'object' || json === null || Array.isArray(json))
|
|
412
|
+
return {};
|
|
413
|
+
const out = {};
|
|
414
|
+
for (const [slug, entry] of Object.entries(json)) {
|
|
415
|
+
if (slug === '')
|
|
416
|
+
continue;
|
|
417
|
+
const parsed = parseEmbeddingIndexEntry(entry);
|
|
418
|
+
if (parsed !== null)
|
|
419
|
+
out[slug] = parsed;
|
|
420
|
+
}
|
|
421
|
+
return out;
|
|
422
|
+
}
|
|
423
|
+
catch {
|
|
424
|
+
return {};
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
/** Persist the index in the canonical shape (the rebuild's write side —
|
|
428
|
+
* embedding/rebuilding itself stays with the caller). Offline rebuild
|
|
429
|
+
* path, not the hot lane. */
|
|
430
|
+
export function saveEmbeddingIndex(bundleDir, index, mkdir = true) {
|
|
431
|
+
if (mkdir)
|
|
432
|
+
mkdirSync(join(bundleDir, 'meta'), { recursive: true });
|
|
433
|
+
writeFileSync(embeddingIndexPath(bundleDir), JSON.stringify(index));
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Freshness audit: which slugs need (re)embedding — missing from the index,
|
|
437
|
+
* built by a different model, or shape-broken (dim/vec disagree). Returns
|
|
438
|
+
* them in roster order; an EMPTY list means the index covers the roster.
|
|
439
|
+
* A '' embedder model means the semantic layer is off → nothing to maintain.
|
|
440
|
+
*
|
|
441
|
+
* NOTE the freshness boundary: content-change invalidation needs a stored
|
|
442
|
+
* content hash, which the pinned {model,dim,vec} entry shape does not carry
|
|
443
|
+
* — the rebuild trigger policy is spec-实现期定 and stays with the caller.
|
|
444
|
+
*/
|
|
445
|
+
export function ensureIndexFresh(index, topics, embedFn) {
|
|
446
|
+
if ((embedFn.model ?? '') === '')
|
|
447
|
+
return [];
|
|
448
|
+
const stale = [];
|
|
449
|
+
for (const t of topics) {
|
|
450
|
+
const e = index[t.slug];
|
|
451
|
+
if (e === undefined || e.model !== embedFn.model || e.dim !== e.vec.length || !(e.dim > 0))
|
|
452
|
+
stale.push(t.slug);
|
|
453
|
+
}
|
|
454
|
+
return stale;
|
|
455
|
+
}
|
|
456
|
+
// --- ③ nearFloor semantic rerank (A4 决议 2, spec §3.1) ---------------------
|
|
457
|
+
// Cosine reordering of the nearFloor band ONLY (0.20–0.30 — the 93%
|
|
458
|
+
// near-miss mass). Thresholds, gateScore and the structural gate are
|
|
459
|
+
// untouched: the reranker reorders and tags, never re-scores.
|
|
460
|
+
export const RERANK_FLOOR = 0.2;
|
|
461
|
+
export const RERANK_CEILING = 0.3;
|
|
462
|
+
export const SEMANTIC_RERANK_REASON = 'semantic-rerank';
|
|
463
|
+
/** Cosine similarity; 0 for dim mismatch or zero vectors (never throws). */
|
|
464
|
+
export function cosineSimilarity(a, b) {
|
|
465
|
+
if (a.length === 0 || a.length !== b.length)
|
|
466
|
+
return 0;
|
|
467
|
+
let dot = 0;
|
|
468
|
+
let na = 0;
|
|
469
|
+
let nb = 0;
|
|
470
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
471
|
+
dot += a[i] * b[i];
|
|
472
|
+
na += a[i] * a[i];
|
|
473
|
+
nb += b[i] * b[i];
|
|
474
|
+
}
|
|
475
|
+
if (na === 0 || nb === 0)
|
|
476
|
+
return 0;
|
|
477
|
+
return dot / Math.sqrt(na * nb);
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Reorder the nearFloor band (0.20–0.30 by gateScore) by cosine similarity
|
|
481
|
+
* against the query vector. Band members with a usable index vector sort by
|
|
482
|
+
* similarity (stable — ties keep input order) and gain the
|
|
483
|
+
* `semantic-rerank` reason plus their similarity; everyone else — out of
|
|
484
|
+
* band, unindexed, model-mismatched — passes through UNCHANGED at the tail
|
|
485
|
+
* in input order. Scores are never touched.
|
|
486
|
+
*
|
|
487
|
+
* Degradation is explicit and total: provider/model unconfigured, probe
|
|
488
|
+
* failure, missing query vector or missing index all return the input as-is
|
|
489
|
+
* with `reranked:false` and a passthroughReason (原序透传,不许静默 — the
|
|
490
|
+
* caller logs it into the degraded status). `queryText` rides the signature
|
|
491
|
+
* for provenance/debug parity with the caller's logging; the v1 math is
|
|
492
|
+
* vector-only.
|
|
493
|
+
*/
|
|
494
|
+
export function rerankNearFloor(queryText, queryVec, nearFloorCandidates, index, opts = {}) {
|
|
495
|
+
void queryText; // v1 math is vector-only — see doc above
|
|
496
|
+
if (nearFloorCandidates.length === 0) {
|
|
497
|
+
return { hits: [], reranked: false, passthroughReason: 'no-candidates' };
|
|
498
|
+
}
|
|
499
|
+
if (!Array.isArray(queryVec) || queryVec.length === 0) {
|
|
500
|
+
return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'no-query-vector' };
|
|
501
|
+
}
|
|
502
|
+
if (Object.keys(index).length === 0) {
|
|
503
|
+
return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'empty-index' };
|
|
504
|
+
}
|
|
505
|
+
const floor = opts.floor ?? RERANK_FLOOR;
|
|
506
|
+
const ceiling = opts.ceiling ?? RERANK_CEILING;
|
|
507
|
+
const simOf = new Map();
|
|
508
|
+
for (const hit of nearFloorCandidates) {
|
|
509
|
+
const entry = index[hit.slug];
|
|
510
|
+
const usable = entry !== undefined &&
|
|
511
|
+
entry.dim === entry.vec.length &&
|
|
512
|
+
entry.dim > 0 &&
|
|
513
|
+
entry.dim === queryVec.length &&
|
|
514
|
+
(opts.expectedModel === undefined || entry.model === opts.expectedModel);
|
|
515
|
+
simOf.set(hit, usable ? cosineSimilarity(queryVec, entry.vec) : null);
|
|
516
|
+
}
|
|
517
|
+
const band = nearFloorCandidates.filter((h) => {
|
|
518
|
+
const sim = simOf.get(h);
|
|
519
|
+
return sim !== null && sim !== undefined && h.gateScore !== undefined && h.gateScore >= floor && h.gateScore < ceiling;
|
|
520
|
+
});
|
|
521
|
+
if (band.length === 0) {
|
|
522
|
+
return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'no-rerankable-candidates' };
|
|
523
|
+
}
|
|
524
|
+
const rerankedSet = new Set(band);
|
|
525
|
+
const tail = nearFloorCandidates.filter((h) => !rerankedSet.has(h));
|
|
526
|
+
// Stable sort: V8's Array#sort keeps input order for equal similarities.
|
|
527
|
+
const ordered = [...band].sort((x, y) => simOf.get(y) - simOf.get(x));
|
|
528
|
+
return {
|
|
529
|
+
hits: [
|
|
530
|
+
...ordered.map((h) => {
|
|
531
|
+
const sim = simOf.get(h);
|
|
532
|
+
return { ...h, reasons: [...h.reasons, SEMANTIC_RERANK_REASON], similarity: Math.round(sim * 10_000) / 10_000 };
|
|
533
|
+
}),
|
|
534
|
+
...tail.map((h) => ({ ...h })),
|
|
535
|
+
],
|
|
536
|
+
reranked: true,
|
|
537
|
+
};
|
|
538
|
+
}
|
package/lib/service.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { type RetrievableTopic, type SearchOutcome } from './retrieval.ts';
|
|
|
10
10
|
import { type AssembleResult } from './digest.ts';
|
|
11
11
|
import type { BundleStore, Observation, SaveResult } from './store.ts';
|
|
12
12
|
import type { TopicsConfigValue } from './config.ts';
|
|
13
|
-
import { type AggregateStats, type QueryBuildShape, type UsageSignal } from './ilog.ts';
|
|
13
|
+
import { type AggregateStats, type QueryBuildShape, type SlowOutcome, type UsageSignal } from './ilog.ts';
|
|
14
14
|
import type { Sync } from './sync.ts';
|
|
15
15
|
export interface RetrieveOutcome {
|
|
16
16
|
outcome: SearchOutcome;
|
|
@@ -33,12 +33,20 @@ export interface SlowDelivery {
|
|
|
33
33
|
queryBuild: QueryBuildShape;
|
|
34
34
|
model: string;
|
|
35
35
|
ms: number;
|
|
36
|
+
/** Producing run's disposition (A4 全留痕) — copied into ilog.slowOutcome. */
|
|
37
|
+
outcome?: SlowOutcome;
|
|
36
38
|
}
|
|
37
39
|
export declare class TopicsService {
|
|
38
40
|
private cache;
|
|
39
41
|
/** mtime-keyed parse of the observations log, grouped by session. */
|
|
40
42
|
private echoCache;
|
|
41
|
-
|
|
43
|
+
/** Single shared cache over the two feedback JSONL logs (S2): the key is
|
|
44
|
+
* the (mtime,size) quartet of injections.jsonl + opens.jsonl, one read
|
|
45
|
+
* and one tail parse feed BOTH aggregates — usageSignalsSync and
|
|
46
|
+
* zeroOpenSlugsSync used to each keep a full-parse cache of their own.
|
|
47
|
+
* Invalidation stays append-driven: the store only ever appends, so any
|
|
48
|
+
* write moves mtime+size. Never populated when a read failed (N1). */
|
|
49
|
+
private logSignalsCache;
|
|
42
50
|
readonly store: BundleStore;
|
|
43
51
|
private readonly getConfig;
|
|
44
52
|
readonly sync?: Sync;
|
|
@@ -63,10 +71,50 @@ export declare class TopicsService {
|
|
|
63
71
|
* Usage signals (ADR 0015): mtime-cached aggregate over the injection log
|
|
64
72
|
* and the opens log, rolling USAGE_WINDOW_DAYS. Synchronous like
|
|
65
73
|
* echoSlugsSync — the retrieval hot path is sync — and cheap when warm:
|
|
66
|
-
* a fresh cache costs two stat() calls
|
|
74
|
+
* a fresh cache costs two stat() calls, one read of each log, and feeds
|
|
75
|
+
* the zero-open set from the same parse. Unreadable logs fail open to an
|
|
67
76
|
* empty map (no boost), never an error.
|
|
68
77
|
*/
|
|
69
78
|
usageSignalsSync(): Map<string, UsageSignal>;
|
|
79
|
+
/**
|
|
80
|
+
* Zero-open slug set for the decay (v5): the shared aggregateZeroOpen
|
|
81
|
+
* rule over the injection tail and the whole opens log (compacted, so a
|
|
82
|
+
* full read stays small). Shares one cache and one parse with
|
|
83
|
+
* usageSignalsSync; unreadable logs fail open to an empty set (no
|
|
84
|
+
* decay), never an error. Only called when zeroOpenDecay is on.
|
|
85
|
+
*/
|
|
86
|
+
zeroOpenSlugsSync(): ReadonlySet<string>;
|
|
87
|
+
/**
|
|
88
|
+
* The one shared feedback-log read behind both signals: stat both files
|
|
89
|
+
* (mtime+size quartet key), parse the injections TAIL only
|
|
90
|
+
* (ZERO_OPEN_TAIL rounds comfortably cover both 30-day windows) and the
|
|
91
|
+
* opens log whole (the zero-open rule is whole-history), then feed the
|
|
92
|
+
* two pure aggregates from that single parse. A failed read still fails
|
|
93
|
+
* open to empty inputs but NEVER populates the cache (N1) — otherwise a
|
|
94
|
+
* stat-success/read-failure race would freeze an empty snapshot until
|
|
95
|
+
* the next append.
|
|
96
|
+
*/
|
|
97
|
+
private logSignalsSync;
|
|
98
|
+
/**
|
|
99
|
+
* zeroOpenDecay fields for scoring calls: empty when the gate is off —
|
|
100
|
+
* no file reads, zero behavior change — and the shared mtime-cached set
|
|
101
|
+
* when on. Spread into every searchTopics/scoreTopic cfg that already
|
|
102
|
+
* carries the usage signals (injection paths + explicit topic_search;
|
|
103
|
+
* the slow lane's recall band stays untouched, mirroring usageBoost).
|
|
104
|
+
*/
|
|
105
|
+
private zeroOpenCfg;
|
|
106
|
+
/** mtime-cached view of meta/embeddings.json (sync read — hot-lane caller).
|
|
107
|
+
* Missing/corrupt index = {} → the reranker degrades to lexical order. */
|
|
108
|
+
private embedIndexCache;
|
|
109
|
+
private embeddingIndexCacheGet;
|
|
110
|
+
/** Shadow bypass ledger (spec §8.3): one line per shadow round — feature
|
|
111
|
+
* fields only, never query text (jev-log discipline). Silent on failure;
|
|
112
|
+
* 512KB truncation keeps the newest three quarters. */
|
|
113
|
+
private appendPackageShadowLedger;
|
|
114
|
+
/** Boot-time embedding index maintenance (spec §3: boot 校验+增量重建).
|
|
115
|
+
* Provider '' or packageMode off → no-op (default-off byte-for-byte).
|
|
116
|
+
* Full-roster embed on rebuild (A1 §5: ~9.5s for 137 topics). */
|
|
117
|
+
refreshEmbeddingIndex(): Promise<void>;
|
|
70
118
|
/**
|
|
71
119
|
* Full retrieval round: search → assemble → log (ADR 0006/0007). The
|
|
72
120
|
* injection record is written even for zero-hit rounds — the hit-rate
|
|
@@ -124,6 +172,13 @@ export declare class TopicsService {
|
|
|
124
172
|
openQuestions?: string[];
|
|
125
173
|
recommendations?: string;
|
|
126
174
|
}>;
|
|
175
|
+
/**
|
|
176
|
+
* Resolve the open's source attribution: the injections tail (bounded by
|
|
177
|
+
* OPEN_ATTRIBUTION_TAIL) scanned for a same-session hit within the
|
|
178
|
+
* attribution window. Read failures degrade to 'search' — attribution is
|
|
179
|
+
* telemetry, never a blocker.
|
|
180
|
+
*/
|
|
181
|
+
private attributeOpen;
|
|
127
182
|
/** Sync roster read (same mtime cache as roster()). */
|
|
128
183
|
rosterSync(): RetrievableTopic[];
|
|
129
184
|
private readDocSync;
|
|
@@ -137,6 +192,8 @@ export declare class TopicsService {
|
|
|
137
192
|
depends?: string[];
|
|
138
193
|
openQuestions?: string[];
|
|
139
194
|
impact?: string[];
|
|
195
|
+
/** A2 semantic-link targets; store.saveTopic lands them as body wikilinks. */
|
|
196
|
+
links?: string[];
|
|
140
197
|
status?: okf.TopicStatus;
|
|
141
198
|
conclusion?: string;
|
|
142
199
|
recommendations?: string;
|