@aiwayds/dsh-topics-memory 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commands.js +23 -2
- package/lib/config.d.ts +19 -5
- package/lib/config.js +21 -3
- package/lib/digest.d.ts +38 -1
- package/lib/digest.js +92 -11
- package/lib/distill.d.ts +1 -0
- package/lib/distill.js +3 -1
- package/lib/ilog.d.ts +36 -0
- package/lib/index.js +95 -23
- package/lib/observer.d.ts +17 -0
- package/lib/observer.js +31 -4
- package/lib/okf.d.ts +7 -0
- package/lib/okf.js +12 -1
- package/lib/quality.d.ts +90 -0
- package/lib/quality.js +315 -0
- package/lib/retrieval.d.ts +35 -3
- package/lib/retrieval.js +73 -19
- package/lib/service.d.ts +48 -7
- package/lib/service.js +198 -25
- package/lib/store.d.ts +13 -0
- package/lib/store.js +54 -0
- package/lib/tools.js +49 -1
- package/package.json +2 -2
- package/src/commands.ts +26 -2
- package/src/config.ts +21 -3
- package/src/digest.ts +118 -11
- package/src/distill.ts +4 -1
- package/src/ilog.ts +39 -2
- package/src/index.ts +87 -19
- package/src/observer.ts +42 -4
- package/src/okf.ts +17 -1
- package/src/quality.ts +335 -0
- package/src/retrieval.ts +93 -19
- package/src/service.ts +214 -25
- package/src/store.ts +53 -0
- package/src/tools.ts +51 -1
package/lib/retrieval.js
CHANGED
|
@@ -18,7 +18,10 @@ export const DEFAULT_RETRIEVAL = {
|
|
|
18
18
|
tagBoost: 0.15,
|
|
19
19
|
graphDepth: 2,
|
|
20
20
|
recencyWindowDays: 7,
|
|
21
|
+
structuralGate: true,
|
|
21
22
|
};
|
|
23
|
+
/** Trigger hits outrank every other lexical field (pi-llm-wiki: highest weight). */
|
|
24
|
+
export const TRIGGER_WEIGHT = 5;
|
|
22
25
|
const CJK_RUN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]+/gu;
|
|
23
26
|
const WORD_RUN = /[\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}\p{Nd}][\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}\p{Nd}+#.-]*/gu;
|
|
24
27
|
/**
|
|
@@ -60,20 +63,34 @@ function overlap(a, b) {
|
|
|
60
63
|
export function scoreTopic(queryTokens, topic, cfg) {
|
|
61
64
|
const title = new Set(tokenize(topic.title));
|
|
62
65
|
const slug = new Set(tokenize(topic.slug));
|
|
66
|
+
const triggers = new Set((topic.triggers ?? []).flatMap((t) => tokenize(t)));
|
|
63
67
|
const tags = new Set(topic.tags.flatMap((t) => tokenize(t)));
|
|
64
68
|
const description = new Set(tokenize(topic.description ?? ''));
|
|
65
69
|
const conclusion = new Set(tokenize(topic.conclusion));
|
|
66
70
|
const reasons = [];
|
|
67
71
|
let score = 0;
|
|
72
|
+
const cTriggers = containment(queryTokens, triggers);
|
|
73
|
+
if (cTriggers > 0) {
|
|
74
|
+
score += TRIGGER_WEIGHT * cTriggers;
|
|
75
|
+
reasons.push(`triggers:${cTriggers.toFixed(2)}`);
|
|
76
|
+
}
|
|
68
77
|
const cTitle = containment(queryTokens, title);
|
|
69
78
|
if (cTitle > 0) {
|
|
70
79
|
score += 3 * cTitle;
|
|
71
80
|
reasons.push(`title:${cTitle.toFixed(2)}`);
|
|
72
81
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
82
|
+
// Slug and tags are scored separately since the structural gate (v4): the
|
|
83
|
+
// slug is a strong field, tags are not. Weights unchanged (2) so historical
|
|
84
|
+
// score distributions stay comparable.
|
|
85
|
+
const cSlug = containment(queryTokens, slug);
|
|
86
|
+
if (cSlug > 0) {
|
|
87
|
+
score += 2 * cSlug;
|
|
88
|
+
reasons.push(`slug:${cSlug.toFixed(2)}`);
|
|
89
|
+
}
|
|
90
|
+
const cTags = containment(queryTokens, tags);
|
|
91
|
+
if (cTags > 0) {
|
|
92
|
+
score += 2 * cTags;
|
|
93
|
+
reasons.push(`tags:${cTags.toFixed(2)}`);
|
|
77
94
|
}
|
|
78
95
|
const cDesc = containment(queryTokens, description);
|
|
79
96
|
if (cDesc > 0) {
|
|
@@ -85,26 +102,29 @@ export function scoreTopic(queryTokens, topic, cfg) {
|
|
|
85
102
|
score += 0.8 * cConc;
|
|
86
103
|
reasons.push(`conclusion:${cConc.toFixed(2)}`);
|
|
87
104
|
}
|
|
88
|
-
// Exact tag hit boost, additive and capped (
|
|
89
|
-
//
|
|
105
|
+
// Exact tag hit boost, additive and capped at ONE boost's worth (v4 design:
|
|
106
|
+
// the old ×3 cap let shared tags mint a 0.45 floor — the priming main
|
|
107
|
+
// culprit). Never folded into the token set (dilution breaks the guard).
|
|
90
108
|
let tagHits = 0;
|
|
91
109
|
for (const t of tags) {
|
|
92
110
|
if (overlap(queryTokens, new Set([...tokenize(t)])))
|
|
93
111
|
tagHits += 1;
|
|
94
112
|
}
|
|
95
113
|
if (tagHits > 0) {
|
|
96
|
-
const boost = Math.min(cfg.tagBoost * tagHits, cfg.tagBoost
|
|
114
|
+
const boost = Math.min(cfg.tagBoost * tagHits, cfg.tagBoost);
|
|
97
115
|
score += boost;
|
|
98
116
|
reasons.push(`tag-boost:+${boost.toFixed(2)}`);
|
|
99
117
|
}
|
|
100
|
-
// Recency is a tiebreaker
|
|
101
|
-
//
|
|
118
|
+
// Recency is a RANKING tiebreaker only (v4): it pads `score` but is kept
|
|
119
|
+
// out of `gateScore`, so it can never carry a candidate across the numeric
|
|
120
|
+
// threshold. Conflict demotion applies to the pre-tiebreaker score.
|
|
121
|
+
let recencyBonus = 0;
|
|
102
122
|
if (score > 0 && cfg.recencyWindowDays > 0) {
|
|
103
123
|
const generated = Date.parse(topic.generatedAt);
|
|
104
124
|
if (!Number.isNaN(generated)) {
|
|
105
125
|
const ageDays = ((cfg.now?.getTime() ?? Date.now()) - generated) / 86_400_000;
|
|
106
126
|
if (ageDays >= 0 && ageDays <= cfg.recencyWindowDays) {
|
|
107
|
-
|
|
127
|
+
recencyBonus = 0.2;
|
|
108
128
|
reasons.push('recency');
|
|
109
129
|
}
|
|
110
130
|
}
|
|
@@ -113,12 +133,36 @@ export function scoreTopic(queryTokens, topic, cfg) {
|
|
|
113
133
|
score *= 0.3;
|
|
114
134
|
reasons.push('conflicted-demoted');
|
|
115
135
|
}
|
|
116
|
-
|
|
136
|
+
const gateScore = score;
|
|
137
|
+
score += recencyBonus;
|
|
138
|
+
const bodySet = new Set([...description, ...conclusion]);
|
|
139
|
+
let bodyHits = 0;
|
|
140
|
+
for (const t of queryTokens)
|
|
141
|
+
if (bodySet.has(t))
|
|
142
|
+
bodyHits += 1;
|
|
143
|
+
const strong = cTriggers > 0 || cTitle > 0 || cSlug > 0;
|
|
144
|
+
return {
|
|
145
|
+
slug: topic.slug,
|
|
146
|
+
score: Math.round(score * 1000) / 1000,
|
|
147
|
+
reasons,
|
|
148
|
+
strong,
|
|
149
|
+
bodyHits,
|
|
150
|
+
gateScore: Math.round(gateScore * 1000) / 1000,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
/** Structural gate v0 verdict: strong-field hit, or ≥2 distinct body terms. */
|
|
154
|
+
export function passesGate(s) {
|
|
155
|
+
return s.strong || s.bodyHits >= 2;
|
|
117
156
|
}
|
|
118
157
|
/**
|
|
119
|
-
* Search the roster: score every topic,
|
|
120
|
-
*
|
|
121
|
-
*
|
|
158
|
+
* Search the roster: score every topic, apply the numeric threshold to the
|
|
159
|
+
* gate score (recency excluded), split by the structural gate, then expand
|
|
160
|
+
* gate-passed seeds through the `depends` graph (both directions, geometric
|
|
161
|
+
* decay, depth ≤ cfg.graphDepth). Expanded entries enter at the back of the
|
|
162
|
+
* hits and are exempt from the structural gate — they carry no lexical
|
|
163
|
+
* evidence by design, only graph adjacency. Threshold-passing candidates the
|
|
164
|
+
* gate rejects lead the near-misses (reason `gate-blocked`) so the replay
|
|
165
|
+
* evidence shows what the gate is costing.
|
|
122
166
|
*/
|
|
123
167
|
export function searchTopics(query, roster, cfg = {}) {
|
|
124
168
|
const c = { ...DEFAULT_RETRIEVAL, ...cfg };
|
|
@@ -127,9 +171,17 @@ export function searchTopics(query, roster, cfg = {}) {
|
|
|
127
171
|
return { hits: [], nearMisses: [], rosterSize: roster.length };
|
|
128
172
|
}
|
|
129
173
|
const scored = roster.map((t) => scoreTopic(queryTokens, t, c)).sort((a, b) => b.score - a.score);
|
|
130
|
-
const
|
|
131
|
-
const
|
|
132
|
-
const
|
|
174
|
+
const passing = scored.filter((s) => s.gateScore >= c.threshold);
|
|
175
|
+
const directHits = passing.filter((s) => !c.structuralGate || passesGate(s)).slice(0, c.topK);
|
|
176
|
+
const gateBlocked = c.structuralGate
|
|
177
|
+
? passing.filter((s) => !passesGate(s)).slice(0, c.topK)
|
|
178
|
+
: [];
|
|
179
|
+
const nearFloor = scored.filter((s) => s.gateScore < c.threshold && s.gateScore >= c.threshold * 0.5).slice(0, 8);
|
|
180
|
+
const hits = directHits.map((s) => ({ ...s, viaGraph: false }));
|
|
181
|
+
const nearMisses = [
|
|
182
|
+
...gateBlocked.map((s) => ({ ...s, reasons: [...s.reasons, 'gate-blocked'], viaGraph: false })),
|
|
183
|
+
...nearFloor.map((s) => ({ ...s, viaGraph: false })),
|
|
184
|
+
].slice(0, 8);
|
|
133
185
|
// Graph expansion (ADR 0005 graph-shaped bundle): walk depends + body
|
|
134
186
|
// links, both directions, from the seeds; include up to topK+2 slots at
|
|
135
187
|
// decaying score.
|
|
@@ -153,7 +205,9 @@ export function searchTopics(query, roster, cfg = {}) {
|
|
|
153
205
|
}
|
|
154
206
|
}
|
|
155
207
|
const seen = new Set(hits.map((h) => h.slug));
|
|
156
|
-
|
|
208
|
+
// Decay starts from the gateScore: recency must not re-enter through the
|
|
209
|
+
// graph channel after being banned from the direct gate.
|
|
210
|
+
let frontier = hits.map((h) => ({ slug: h.slug, score: h.gateScore ?? h.score }));
|
|
157
211
|
for (let depth = 1; depth <= c.graphDepth && frontier.length > 0; depth += 1) {
|
|
158
212
|
const next = [];
|
|
159
213
|
for (const node of frontier) {
|
|
@@ -174,5 +228,5 @@ export function searchTopics(query, roster, cfg = {}) {
|
|
|
174
228
|
frontier = next;
|
|
175
229
|
}
|
|
176
230
|
}
|
|
177
|
-
return { hits, nearMisses
|
|
231
|
+
return { hits, nearMisses, rosterSize: roster.length };
|
|
178
232
|
}
|
package/lib/service.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { type RetrievableTopic, type SearchOutcome } from './retrieval.ts';
|
|
|
10
10
|
import { type AssembleResult } from './digest.ts';
|
|
11
11
|
import type { BundleStore, Observation, SaveResult } from './store.ts';
|
|
12
12
|
import type { TopicsConfigValue } from './config.ts';
|
|
13
|
-
import { type AggregateStats } from './ilog.ts';
|
|
13
|
+
import { type AggregateStats, type QueryBuildShape } from './ilog.ts';
|
|
14
14
|
import type { Sync } from './sync.ts';
|
|
15
15
|
export interface RetrieveOutcome {
|
|
16
16
|
outcome: SearchOutcome;
|
|
@@ -18,6 +18,22 @@ export interface RetrieveOutcome {
|
|
|
18
18
|
/** Roster was empty or injection disabled — no record written for pure no-ops. */
|
|
19
19
|
recorded: boolean;
|
|
20
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* Slow-lane payload handed to the hot path at consumption time (v4 §4.2).
|
|
23
|
+
* `items` are the rerank picks (0-2, why lines included); metadata feeds the
|
|
24
|
+
* ilog lane field family. The hot path resolves titles, filters dedup,
|
|
25
|
+
* renders, and takes the log-only shadow verdicts.
|
|
26
|
+
*/
|
|
27
|
+
export interface SlowDelivery {
|
|
28
|
+
items: readonly {
|
|
29
|
+
slug: string;
|
|
30
|
+
why: string;
|
|
31
|
+
}[];
|
|
32
|
+
computedAt: string;
|
|
33
|
+
queryBuild: QueryBuildShape;
|
|
34
|
+
model: string;
|
|
35
|
+
ms: number;
|
|
36
|
+
}
|
|
21
37
|
export declare class TopicsService {
|
|
22
38
|
private cache;
|
|
23
39
|
readonly store: BundleStore;
|
|
@@ -43,20 +59,44 @@ export declare class TopicsService {
|
|
|
43
59
|
* would always lose the race and inject nothing. Reads files with sync fs
|
|
44
60
|
* behind the mtime cache; the log record is written fire-and-forget.
|
|
45
61
|
*
|
|
46
|
-
* `dedup.exclude` (session-level injection dedup) filters hits
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* as injected by the caller; budget-dropped
|
|
62
|
+
* `dedup.exclude` (session-level injection dedup) filters hits AND slow
|
|
63
|
+
* picks before assembly; excluded slugs are reported back as `deduped`
|
|
64
|
+
* (and logged as `record.deduped`), never packed. `included` mirrors the
|
|
65
|
+
* fast pointers that entered the context and `slowIncluded` the slow ones
|
|
66
|
+
* — only these may be marked as injected by the caller; budget-dropped
|
|
67
|
+
* slugs stay injectable later.
|
|
68
|
+
*
|
|
69
|
+
* `slow` (v4 §4.2) carries the consumed pending when one exists: picks are
|
|
70
|
+
* packed after the fast pointers under the same budget, take log-only
|
|
71
|
+
* shadow re-gate verdicts against the CURRENT query (B3: record, never
|
|
72
|
+
* block), and fill the record's lane field family.
|
|
51
73
|
*/
|
|
52
74
|
retrieveSync(query: string, sessionId?: string, dedup?: {
|
|
53
75
|
exclude?: ReadonlySet<string>;
|
|
54
|
-
}): {
|
|
76
|
+
}, slow?: SlowDelivery, slowExpired?: 'ttl' | 'turn-lag'): {
|
|
55
77
|
text: string;
|
|
56
78
|
outcome: SearchOutcome;
|
|
57
79
|
deduped: string[];
|
|
58
80
|
included: string[];
|
|
81
|
+
slowIncluded: string[];
|
|
59
82
|
};
|
|
83
|
+
/**
|
|
84
|
+
* topic_open tool path (v4 §4.1): full conclusion / open questions /
|
|
85
|
+
* recommendations plus the staleness notice timestamp (fm.generated.at —
|
|
86
|
+
* the store stamps it on every write, so it IS the last-updated instant).
|
|
87
|
+
* Logs an open record for the pointer-open-rate stat.
|
|
88
|
+
*/
|
|
89
|
+
openTopic(slug: string, sessionId?: string): Promise<{
|
|
90
|
+
found: boolean;
|
|
91
|
+
slug: string;
|
|
92
|
+
title?: string;
|
|
93
|
+
status?: okf.TopicStatus;
|
|
94
|
+
updatedAt?: string;
|
|
95
|
+
description?: string;
|
|
96
|
+
conclusion?: string;
|
|
97
|
+
openQuestions?: string[];
|
|
98
|
+
recommendations?: string;
|
|
99
|
+
}>;
|
|
60
100
|
/** Sync roster read (same mtime cache as roster()). */
|
|
61
101
|
rosterSync(): RetrievableTopic[];
|
|
62
102
|
private readDocSync;
|
|
@@ -66,6 +106,7 @@ export declare class TopicsService {
|
|
|
66
106
|
title: string;
|
|
67
107
|
description?: string;
|
|
68
108
|
tags?: string[];
|
|
109
|
+
triggers?: string[];
|
|
69
110
|
depends?: string[];
|
|
70
111
|
openQuestions?: string[];
|
|
71
112
|
impact?: string[];
|
package/lib/service.js
CHANGED
|
@@ -9,8 +9,8 @@ import { readFile, stat } from 'node:fs/promises';
|
|
|
9
9
|
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
10
10
|
import { join } from 'node:path';
|
|
11
11
|
import * as okf from "./okf.js";
|
|
12
|
-
import { searchTopics, tokenize } from "./retrieval.js";
|
|
13
|
-
import { assembleInjection } from "./digest.js";
|
|
12
|
+
import { searchTopics, scoreTopic, passesGate, tokenize } from "./retrieval.js";
|
|
13
|
+
import { assembleInjection, assemblePointer, POINTER_PER_TOPIC, POINTER_TOTAL } from "./digest.js";
|
|
14
14
|
import { aggregateStats, querySample } from "./ilog.js";
|
|
15
15
|
import { fileHistory, fileAtRev } from "./git.js";
|
|
16
16
|
export class TopicsService {
|
|
@@ -70,6 +70,7 @@ export class TopicsService {
|
|
|
70
70
|
description: doc.fm.description,
|
|
71
71
|
status: doc.fm.status,
|
|
72
72
|
tags: doc.fm.tags,
|
|
73
|
+
triggers: doc.fm.triggers,
|
|
73
74
|
depends: doc.fm.depends,
|
|
74
75
|
links: okf.bodyLinkSlugs(doc.body),
|
|
75
76
|
generatedAt: doc.fm.generated.at,
|
|
@@ -116,10 +117,12 @@ export class TopicsService {
|
|
|
116
117
|
if (doc !== undefined)
|
|
117
118
|
entries.push({ slug: hit.slug, doc, hit });
|
|
118
119
|
}
|
|
119
|
-
const injection =
|
|
120
|
-
perTopicBudget: cfg.
|
|
121
|
-
|
|
122
|
-
|
|
120
|
+
const injection = cfg.injectMode !== 'digest'
|
|
121
|
+
? assemblePointer(entries, { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) })
|
|
122
|
+
: assembleInjection(entries, {
|
|
123
|
+
perTopicBudget: cfg.perTopicBudget,
|
|
124
|
+
totalBudget: cfg.totalBudget,
|
|
125
|
+
});
|
|
123
126
|
const record = {
|
|
124
127
|
at: new Date().toISOString(),
|
|
125
128
|
queryTokenCount: tokenize(query).length,
|
|
@@ -146,18 +149,52 @@ export class TopicsService {
|
|
|
146
149
|
* would always lose the race and inject nothing. Reads files with sync fs
|
|
147
150
|
* behind the mtime cache; the log record is written fire-and-forget.
|
|
148
151
|
*
|
|
149
|
-
* `dedup.exclude` (session-level injection dedup) filters hits
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
* as injected by the caller; budget-dropped
|
|
152
|
+
* `dedup.exclude` (session-level injection dedup) filters hits AND slow
|
|
153
|
+
* picks before assembly; excluded slugs are reported back as `deduped`
|
|
154
|
+
* (and logged as `record.deduped`), never packed. `included` mirrors the
|
|
155
|
+
* fast pointers that entered the context and `slowIncluded` the slow ones
|
|
156
|
+
* — only these may be marked as injected by the caller; budget-dropped
|
|
157
|
+
* slugs stay injectable later.
|
|
158
|
+
*
|
|
159
|
+
* `slow` (v4 §4.2) carries the consumed pending when one exists: picks are
|
|
160
|
+
* packed after the fast pointers under the same budget, take log-only
|
|
161
|
+
* shadow re-gate verdicts against the CURRENT query (B3: record, never
|
|
162
|
+
* block), and fill the record's lane field family.
|
|
154
163
|
*/
|
|
155
|
-
retrieveSync(query, sessionId, dedup) {
|
|
164
|
+
retrieveSync(query, sessionId, dedup, slow, slowExpired) {
|
|
156
165
|
const cfg = this.cfg;
|
|
157
166
|
const roster = this.rosterSync();
|
|
158
167
|
const empty = { hits: [], nearMisses: [], rosterSize: roster.length };
|
|
159
168
|
if (!cfg.autoInject || query.trim() === '' || roster.length === 0) {
|
|
160
|
-
|
|
169
|
+
// A consumed/expired pending must always leave an ilog trace (ADR 0014:
|
|
170
|
+
// 过期/消费都写 ilog) — even when the fast path early-returns here
|
|
171
|
+
// because the roster vanished between production and consumption.
|
|
172
|
+
if (cfg.autoInject && query.trim() !== '' && (slow !== undefined || slowExpired !== undefined)) {
|
|
173
|
+
const trace = {
|
|
174
|
+
at: new Date().toISOString(),
|
|
175
|
+
queryTokenCount: tokenize(query).length,
|
|
176
|
+
querySample: querySample(query),
|
|
177
|
+
rosterSize: roster.length,
|
|
178
|
+
hits: [],
|
|
179
|
+
nearMisses: [],
|
|
180
|
+
injected: false,
|
|
181
|
+
why: slowExpired !== undefined ? `slow-expired-${slowExpired}` : 'slow-no-roster',
|
|
182
|
+
lane: 'slow',
|
|
183
|
+
};
|
|
184
|
+
if (sessionId !== undefined)
|
|
185
|
+
trace.sessionId = sessionId;
|
|
186
|
+
if (slowExpired !== undefined)
|
|
187
|
+
trace.slowExpired = slowExpired;
|
|
188
|
+
if (slow !== undefined) {
|
|
189
|
+
trace.computedAt = slow.computedAt;
|
|
190
|
+
trace.queryBuild = slow.queryBuild;
|
|
191
|
+
trace.slowModel = slow.model;
|
|
192
|
+
trace.slowMs = slow.ms;
|
|
193
|
+
trace.slow = [];
|
|
194
|
+
}
|
|
195
|
+
void this.store.appendInjectionRecord(trace).catch(() => undefined);
|
|
196
|
+
}
|
|
197
|
+
return { text: '', outcome: empty, deduped: [], included: [], slowIncluded: [] };
|
|
161
198
|
}
|
|
162
199
|
const conflicts = this.store.getConflictsSync();
|
|
163
200
|
const outcome = searchTopics(query, roster, {
|
|
@@ -170,25 +207,81 @@ export class TopicsService {
|
|
|
170
207
|
});
|
|
171
208
|
const exclude = dedup?.exclude;
|
|
172
209
|
const deduped = exclude === undefined ? [] : outcome.hits.filter((h) => exclude.has(h.slug)).map((h) => h.slug);
|
|
210
|
+
const bySlug = new Map(roster.map((r) => [r.slug, r]));
|
|
173
211
|
const entries = [];
|
|
212
|
+
const unreadable = [];
|
|
174
213
|
for (const hit of outcome.hits) {
|
|
175
214
|
if (exclude?.has(hit.slug))
|
|
176
215
|
continue;
|
|
177
216
|
const doc = this.readDocSync(hit.slug);
|
|
178
|
-
if (doc
|
|
179
|
-
|
|
217
|
+
if (doc === undefined) {
|
|
218
|
+
// The hit has no readable doc — without this accounting it used to
|
|
219
|
+
// vanish from `included` AND `dropped` (v4 design §2 疑点).
|
|
220
|
+
unreadable.push(hit.slug);
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
entries.push({ slug: hit.slug, doc, hit });
|
|
224
|
+
}
|
|
225
|
+
// ---- Slow-lane merge (log-only shadow gate; consumption bookkeeping) ----
|
|
226
|
+
const slowPointers = [];
|
|
227
|
+
const slowDelivered = [];
|
|
228
|
+
const shadow = [];
|
|
229
|
+
let slowIncluded = [];
|
|
230
|
+
const consumedAt = new Date().toISOString();
|
|
231
|
+
if (slow !== undefined) {
|
|
232
|
+
const queryTokens = new Set(tokenize(query));
|
|
233
|
+
for (const item of slow.items) {
|
|
234
|
+
if (exclude?.has(item.slug)) {
|
|
235
|
+
if (!deduped.includes(item.slug))
|
|
236
|
+
deduped.push(item.slug);
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
const meta = bySlug.get(item.slug);
|
|
240
|
+
if (meta === undefined)
|
|
241
|
+
continue; // vanished between production and consumption
|
|
242
|
+
const verdict = scoreTopic(queryTokens, meta, {
|
|
243
|
+
threshold: cfg.matchThreshold,
|
|
244
|
+
topK: cfg.topK,
|
|
245
|
+
tagBoost: cfg.tagBoost,
|
|
246
|
+
graphDepth: cfg.graphDepth,
|
|
247
|
+
recencyWindowDays: cfg.recencyWindowDays,
|
|
248
|
+
conflicts,
|
|
249
|
+
structuralGate: true,
|
|
250
|
+
});
|
|
251
|
+
shadow.push({
|
|
252
|
+
slug: item.slug,
|
|
253
|
+
pass: passesGate(verdict),
|
|
254
|
+
why: verdict.strong ? `strong:${verdict.reasons[0] ?? 'field'}` : `body-hits:${verdict.bodyHits}`,
|
|
255
|
+
});
|
|
256
|
+
slowPointers.push({
|
|
257
|
+
slug: item.slug,
|
|
258
|
+
title: meta.title,
|
|
259
|
+
status: meta.status,
|
|
260
|
+
description: meta.description,
|
|
261
|
+
why: item.why,
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
const pointerMode = cfg.injectMode !== 'digest';
|
|
266
|
+
// Pointer budget: per-entry fixed at the design's 80, total clamped to the
|
|
267
|
+
// fast-lane upper bound of 600 (I4) — totalBudget stays tunable downward.
|
|
268
|
+
const injection = pointerMode
|
|
269
|
+
? assemblePointer(entries, { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) }, slowPointers)
|
|
270
|
+
: assembleInjection(entries, { perTopicBudget: cfg.perTopicBudget, totalBudget: cfg.totalBudget }, slowPointers);
|
|
271
|
+
slowIncluded = injection.slowIncluded ?? [];
|
|
272
|
+
for (const pointer of slowPointers) {
|
|
273
|
+
if (slowIncluded.includes(pointer.slug))
|
|
274
|
+
slowDelivered.push({ slug: pointer.slug, why: pointer.why });
|
|
180
275
|
}
|
|
181
|
-
const injection = assembleInjection(entries, {
|
|
182
|
-
perTopicBudget: cfg.perTopicBudget,
|
|
183
|
-
totalBudget: cfg.totalBudget,
|
|
184
|
-
});
|
|
185
276
|
const record = {
|
|
186
277
|
at: new Date().toISOString(),
|
|
187
278
|
queryTokenCount: tokenize(query).length,
|
|
188
279
|
querySample: querySample(query),
|
|
189
280
|
rosterSize: outcome.rosterSize,
|
|
190
|
-
hits: outcome.hits.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons, viaGraph: h.viaGraph })),
|
|
191
|
-
|
|
281
|
+
hits: outcome.hits.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons, viaGraph: h.viaGraph, strong: h.strong, bodyHits: h.bodyHits })),
|
|
282
|
+
// Reasons ride along: the gate-blocked marker is the replay evidence
|
|
283
|
+
// the P3 threshold decision needs (ADR 0014).
|
|
284
|
+
nearMisses: outcome.nearMisses.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons })),
|
|
192
285
|
injected: injection.text !== '',
|
|
193
286
|
usedTokens: injection.usedTokens,
|
|
194
287
|
};
|
|
@@ -196,14 +289,73 @@ export class TopicsService {
|
|
|
196
289
|
record.sessionId = sessionId;
|
|
197
290
|
if (deduped.length > 0)
|
|
198
291
|
record.deduped = deduped;
|
|
199
|
-
if (
|
|
292
|
+
if (unreadable.length > 0) {
|
|
293
|
+
record.dropped = [...(record.dropped ?? []), ...unreadable.map((slug) => ({ slug, reason: 'doc-unreadable' }))];
|
|
294
|
+
}
|
|
295
|
+
if (slow !== undefined) {
|
|
296
|
+
// Any round whose pending was consumed is a slow-participating round —
|
|
297
|
+
// lane never stays undefined when the lane family is present (a
|
|
298
|
+
// delivered pick with fast pointers is 'mixed'; a consumed-but-empty
|
|
299
|
+
// round beside fast pointers stays 'fast'; everything else 'slow').
|
|
300
|
+
record.lane = slowDelivered.length > 0 ? (injection.included.length > 0 ? 'mixed' : 'slow') : injection.included.length > 0 ? 'fast' : 'slow';
|
|
301
|
+
record.computedAt = slow.computedAt;
|
|
302
|
+
record.consumedAt = consumedAt;
|
|
303
|
+
if (shadow.length > 0)
|
|
304
|
+
record.shadowVerdict = shadow;
|
|
305
|
+
record.queryBuild = slow.queryBuild;
|
|
306
|
+
record.slowModel = slow.model;
|
|
307
|
+
record.slowMs = slow.ms;
|
|
308
|
+
record.slow = slowDelivered;
|
|
309
|
+
}
|
|
310
|
+
if (slowExpired !== undefined) {
|
|
311
|
+
// Expiry is recorded INDEPENDENT of whether the fast lane injected —
|
|
312
|
+
// the 赶上率 denominator must not lose rounds to a coincidental hit.
|
|
313
|
+
record.slowExpired = slowExpired;
|
|
314
|
+
if (record.lane === undefined)
|
|
315
|
+
record.lane = injection.text !== '' ? 'fast' : 'slow';
|
|
316
|
+
}
|
|
317
|
+
if (injection.text === '' && (outcome.hits.length > 0 || slowPointers.length > 0)) {
|
|
200
318
|
// All hits deduped is a different story than a budget squeeze — say so.
|
|
201
|
-
record.why = deduped.length === outcome.hits.length ? 'dedup' : 'below-budget-or-dropped';
|
|
319
|
+
record.why = deduped.length === outcome.hits.length && outcome.hits.length > 0 ? 'dedup' : 'below-budget-or-dropped';
|
|
320
|
+
}
|
|
321
|
+
else if (injection.text === '' && slowExpired !== undefined) {
|
|
322
|
+
// The pending died at its hard bounds (TTL / turn-lag) without ever
|
|
323
|
+
// being delivered — the denominator the 赶上率 needs (v4 B4).
|
|
324
|
+
record.why = `slow-expired-${slowExpired}`;
|
|
202
325
|
}
|
|
326
|
+
if (slowExpired !== undefined && injection.text === '')
|
|
327
|
+
record.lane = 'slow';
|
|
203
328
|
if (injection.dropped.length > 0)
|
|
204
|
-
record.dropped = injection.dropped;
|
|
329
|
+
record.dropped = [...(record.dropped ?? []), ...injection.dropped];
|
|
205
330
|
void this.store.appendInjectionRecord(record).catch(() => undefined);
|
|
206
|
-
return { text: injection.text, outcome, deduped, included: injection.included };
|
|
331
|
+
return { text: injection.text, outcome, deduped, included: injection.included, slowIncluded };
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* topic_open tool path (v4 §4.1): full conclusion / open questions /
|
|
335
|
+
* recommendations plus the staleness notice timestamp (fm.generated.at —
|
|
336
|
+
* the store stamps it on every write, so it IS the last-updated instant).
|
|
337
|
+
* Logs an open record for the pointer-open-rate stat.
|
|
338
|
+
*/
|
|
339
|
+
async openTopic(slug, sessionId) {
|
|
340
|
+
const doc = await this.store.readTopic(slug).catch(() => undefined);
|
|
341
|
+
if (doc === undefined)
|
|
342
|
+
return { found: false, slug };
|
|
343
|
+
// Awaited (tool path, not hot): a settled write makes the pointer-open
|
|
344
|
+
// stat reliable; a failed log write must not fail the tool.
|
|
345
|
+
await this.store
|
|
346
|
+
.appendOpenRecord({ slug, at: new Date().toISOString(), ...(sessionId !== undefined ? { sessionId } : {}) })
|
|
347
|
+
.catch(() => undefined);
|
|
348
|
+
return {
|
|
349
|
+
found: true,
|
|
350
|
+
slug,
|
|
351
|
+
title: doc.fm.title,
|
|
352
|
+
status: doc.fm.status,
|
|
353
|
+
updatedAt: doc.fm.generated.at,
|
|
354
|
+
description: doc.fm.description,
|
|
355
|
+
conclusion: okf.sectionOf(doc.body, okf.CONCLUSION_HEADING) ?? '',
|
|
356
|
+
openQuestions: doc.fm.open_questions,
|
|
357
|
+
recommendations: okf.sectionOf(doc.body, okf.RECOMMENDATIONS_HEADING) ?? '',
|
|
358
|
+
};
|
|
207
359
|
}
|
|
208
360
|
/** Sync roster read (same mtime cache as roster()). */
|
|
209
361
|
rosterSync() {
|
|
@@ -248,6 +400,7 @@ export class TopicsService {
|
|
|
248
400
|
description: doc.fm.description,
|
|
249
401
|
status: doc.fm.status,
|
|
250
402
|
tags: doc.fm.tags,
|
|
403
|
+
triggers: doc.fm.triggers,
|
|
251
404
|
depends: doc.fm.depends,
|
|
252
405
|
links: okf.bodyLinkSlugs(doc.body),
|
|
253
406
|
generatedAt: doc.fm.generated.at,
|
|
@@ -301,6 +454,11 @@ export class TopicsService {
|
|
|
301
454
|
};
|
|
302
455
|
if (input.description !== undefined && input.description !== '')
|
|
303
456
|
fm.description = input.description;
|
|
457
|
+
// Triggers pass through on create; on update the caller's value wins, else
|
|
458
|
+
// the stored list survives (same preserve-on-update semantics as tags).
|
|
459
|
+
const triggers = sanitizeTriggers(input.triggers ?? existing?.fm.triggers ?? []);
|
|
460
|
+
if (triggers.length > 0)
|
|
461
|
+
fm.triggers = triggers;
|
|
304
462
|
if (existing !== undefined && existing.fm.verified !== undefined)
|
|
305
463
|
fm.verified = existing.fm.verified;
|
|
306
464
|
const result = await this.store.saveTopic({ slug, doc: { fm, body } }, { message: existing === undefined ? `topics(topic): create ${slug}` : `topics(topic): update ${slug}`, created: existing === undefined });
|
|
@@ -319,6 +477,9 @@ export class TopicsService {
|
|
|
319
477
|
tagBoost: cfg.tagBoost,
|
|
320
478
|
graphDepth: cfg.graphDepth,
|
|
321
479
|
recencyWindowDays: cfg.recencyWindowDays,
|
|
480
|
+
// Explicit tool: the model asked for recall, so the injection-only
|
|
481
|
+
// structural gate must not hide results.
|
|
482
|
+
structuralGate: false,
|
|
322
483
|
conflicts: await this.store.getConflicts(),
|
|
323
484
|
});
|
|
324
485
|
}
|
|
@@ -373,3 +534,15 @@ function dedupeLower(tags) {
|
|
|
373
534
|
}
|
|
374
535
|
return out;
|
|
375
536
|
}
|
|
537
|
+
function sanitizeTriggers(triggers) {
|
|
538
|
+
const out = [];
|
|
539
|
+
for (const t of triggers) {
|
|
540
|
+
if (typeof t !== 'string')
|
|
541
|
+
continue;
|
|
542
|
+
const key = t.trim();
|
|
543
|
+
if (key === '')
|
|
544
|
+
continue;
|
|
545
|
+
out.push(key);
|
|
546
|
+
}
|
|
547
|
+
return out;
|
|
548
|
+
}
|
package/lib/store.d.ts
CHANGED
|
@@ -135,6 +135,19 @@ export declare class BundleStore {
|
|
|
135
135
|
appendInjectionRecord(record: unknown): Promise<void>;
|
|
136
136
|
private compactInjectionsIfNeeded;
|
|
137
137
|
readInjectionRecords(limit?: number): Promise<InjectionRecord[]>;
|
|
138
|
+
private opensPath;
|
|
139
|
+
appendOpenRecord(record: {
|
|
140
|
+
slug: string;
|
|
141
|
+
at: string;
|
|
142
|
+
sessionId?: string;
|
|
143
|
+
}): Promise<void>;
|
|
144
|
+
private compactOpensIfNeeded;
|
|
145
|
+
/** Opens in the recent window; tolerant of a torn tail like the other JSONLs. */
|
|
146
|
+
readOpenRecords(limit?: number): Promise<{
|
|
147
|
+
slug: string;
|
|
148
|
+
at: string;
|
|
149
|
+
sessionId?: string;
|
|
150
|
+
}[]>;
|
|
138
151
|
getConflicts(): Promise<Set<string>>;
|
|
139
152
|
/** Sync variant for the injection hot path. */
|
|
140
153
|
getConflictsSync(): Set<string>;
|
package/lib/store.js
CHANGED
|
@@ -425,6 +425,60 @@ export class BundleStore {
|
|
|
425
425
|
return out.slice(-limit);
|
|
426
426
|
}
|
|
427
427
|
// ------------------------------------------------------------------
|
|
428
|
+
// Pointer-open log (v4 §4.3) — one line per topic_open call; feeds the
|
|
429
|
+
// 「指针打开率」 stat. Features only (slug/time), never conversation text.
|
|
430
|
+
// ------------------------------------------------------------------
|
|
431
|
+
opensPath() {
|
|
432
|
+
return join(this.metaDir(), 'opens.jsonl');
|
|
433
|
+
}
|
|
434
|
+
async appendOpenRecord(record) {
|
|
435
|
+
await mkdir(this.metaDir(), { recursive: true });
|
|
436
|
+
await appendFile(this.opensPath(), `${JSON.stringify(record)}\n`, 'utf8');
|
|
437
|
+
await this.compactOpensIfNeeded();
|
|
438
|
+
}
|
|
439
|
+
async compactOpensIfNeeded() {
|
|
440
|
+
const file = this.opensPath();
|
|
441
|
+
let size = 0;
|
|
442
|
+
try {
|
|
443
|
+
size = (await readFile(file, 'utf8')).length;
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
if (size <= 512 * 1024)
|
|
449
|
+
return;
|
|
450
|
+
const lines = (await readFile(file, 'utf8')).split('\n').filter((l) => l.trim() !== '');
|
|
451
|
+
await atomicWrite(file, lines.slice(-Math.max(1, Math.floor(lines.length / 4))).map((l) => `${l}\n`).join(''));
|
|
452
|
+
}
|
|
453
|
+
/** Opens in the recent window; tolerant of a torn tail like the other JSONLs. */
|
|
454
|
+
async readOpenRecords(limit = 2000) {
|
|
455
|
+
let raw;
|
|
456
|
+
try {
|
|
457
|
+
raw = await readFile(this.opensPath(), 'utf8');
|
|
458
|
+
}
|
|
459
|
+
catch {
|
|
460
|
+
return [];
|
|
461
|
+
}
|
|
462
|
+
const out = [];
|
|
463
|
+
for (const line of raw.split('\n')) {
|
|
464
|
+
if (line.trim() === '')
|
|
465
|
+
continue;
|
|
466
|
+
try {
|
|
467
|
+
const parsed = JSON.parse(line);
|
|
468
|
+
if (typeof parsed.slug === 'string' && typeof parsed.at === 'string') {
|
|
469
|
+
const rec = { slug: parsed.slug, at: parsed.at };
|
|
470
|
+
if (typeof parsed.sessionId === 'string')
|
|
471
|
+
rec.sessionId = parsed.sessionId;
|
|
472
|
+
out.push(rec);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
catch {
|
|
476
|
+
// Torn tail line tolerated.
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return out.slice(-limit);
|
|
480
|
+
}
|
|
481
|
+
// ------------------------------------------------------------------
|
|
428
482
|
// Conflicted topics (ADR 0003) — retrieval demotes these
|
|
429
483
|
// ------------------------------------------------------------------
|
|
430
484
|
async getConflicts() {
|