@aiwayds/dsh-topics-memory 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/README.zh.md +6 -1
- package/lib/commands.js +27 -11
- package/lib/config.d.ts +17 -1
- package/lib/config.js +9 -1
- package/lib/ilog.d.ts +129 -0
- package/lib/ilog.js +236 -0
- package/lib/index.d.ts +2 -0
- package/lib/retrieval.d.ts +13 -0
- package/lib/retrieval.js +10 -0
- package/lib/service.d.ts +43 -2
- package/lib/service.js +121 -36
- package/lib/store.d.ts +6 -12
- package/lib/store.js +11 -2
- package/lib/tools.js +6 -2
- package/package.json +25 -25
- package/src/commands.ts +35 -16
- package/src/config.ts +13 -1
- package/src/ilog.ts +289 -0
- package/src/retrieval.ts +22 -0
- package/src/service.ts +130 -43
- package/src/store.ts +14 -8
- package/src/tools.ts +6 -2
package/src/ilog.ts
CHANGED
|
@@ -62,6 +62,24 @@ export interface InjectionRecord {
|
|
|
62
62
|
slow?: SlowItem[]
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
/**
|
|
66
|
+
* One topic_open event (pointer-open log, v4 §4.3). Everything beyond
|
|
67
|
+
* `slug`/`at` is optional and late-added: older lines carry only the
|
|
68
|
+
* original pair, and every reader must tolerate the rest being absent.
|
|
69
|
+
*/
|
|
70
|
+
export interface OpenRecord {
|
|
71
|
+
slug: string
|
|
72
|
+
at: string
|
|
73
|
+
/** Session that issued the topic_open, when the host surface exposed one. */
|
|
74
|
+
sessionId?: string
|
|
75
|
+
/** Attribution: opened off an injected pointer, or off a topic_search hit. */
|
|
76
|
+
source?: 'pointer' | 'search'
|
|
77
|
+
/** Retrieval score of the matched injection hit (source='pointer' only). */
|
|
78
|
+
score?: number
|
|
79
|
+
/** open time − the matched injection record's at, in ms (source='pointer' only). */
|
|
80
|
+
sinceInjectionMs?: number
|
|
81
|
+
}
|
|
82
|
+
|
|
65
83
|
export interface AggregateStats {
|
|
66
84
|
rounds: number
|
|
67
85
|
injectedRounds: number
|
|
@@ -191,3 +209,274 @@ export function aggregateUsage(
|
|
|
191
209
|
}
|
|
192
210
|
return out
|
|
193
211
|
}
|
|
212
|
+
|
|
213
|
+
// ---------------------------------------------------------------------
|
|
214
|
+
// Injection-vs-open metrics (v5) — open enrichment attribution + the
|
|
215
|
+
// /topics stats panel aggregate. All pure: file I/O stays in store/service.
|
|
216
|
+
// ---------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
/** How many tail injection records the cheap open attribution scans. */
|
|
219
|
+
export const OPEN_ATTRIBUTION_TAIL = 50
|
|
220
|
+
/** An injected pointer stays attributable for 48h (a work session's span). */
|
|
221
|
+
export const OPEN_ATTRIBUTION_WINDOW_MS = 48 * 3_600_000
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Cheap source attribution for one open event: scan the injection tail
|
|
225
|
+
* (newest first) for a same-session record within the attribution window
|
|
226
|
+
* where the slug was truly delivered to the model's eyes. That is hits
|
|
227
|
+
* minus echoed/dropped (echo suppression and budget drops never rendered),
|
|
228
|
+
* PLUS slow-lane deliveries, PLUS deduped hits — dedup only blocks
|
|
229
|
+
* re-packing THIS round; the pointer rendered earlier in the same session
|
|
230
|
+
* and the model saw it (deliberately wider than renderedPointers: the panel
|
|
231
|
+
* denominator asks "did it render this round", attribution asks "did the
|
|
232
|
+
* model ever see it"). A fast-hit match carries that hit's score; a
|
|
233
|
+
* slow-only match carries none (slow pointers have no lexical score).
|
|
234
|
+
* Anything else (no session id, other session, expired, never rendered)
|
|
235
|
+
* is `search`.
|
|
236
|
+
*/
|
|
237
|
+
export function attributeOpen(
|
|
238
|
+
injections: readonly InjectionRecord[],
|
|
239
|
+
slug: string,
|
|
240
|
+
sessionId: string | undefined,
|
|
241
|
+
nowMs: number,
|
|
242
|
+
): { source: 'pointer' | 'search'; score?: number; sinceInjectionMs?: number } {
|
|
243
|
+
if (sessionId === undefined) return { source: 'search' }
|
|
244
|
+
for (let i = injections.length - 1; i >= 0; i -= 1) {
|
|
245
|
+
const record = injections[i]
|
|
246
|
+
if (record.sessionId !== sessionId) continue
|
|
247
|
+
const atMs = Date.parse(record.at)
|
|
248
|
+
if (!Number.isFinite(atMs) || nowMs - atMs > OPEN_ATTRIBUTION_WINDOW_MS) continue
|
|
249
|
+
const sinceInjectionMs = Math.max(0, nowMs - atMs)
|
|
250
|
+
const suppressed = new Set<string>([...(record.echoed ?? []), ...(record.dropped ?? []).map((d) => d.slug)])
|
|
251
|
+
const hit = record.hits.find((h) => h.slug === slug && !suppressed.has(h.slug))
|
|
252
|
+
if (hit !== undefined) return { source: 'pointer', score: hit.score, sinceInjectionMs }
|
|
253
|
+
if ((record.slow ?? []).some((s) => s.slug === slug)) return { source: 'pointer', sinceInjectionMs }
|
|
254
|
+
}
|
|
255
|
+
return { source: 'search' }
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Score bands for the open-rate-by-score cut of the stats panel. */
|
|
259
|
+
export const SCORE_BANDS: readonly { label: string; lo: number; hi: number }[] = [
|
|
260
|
+
{ label: '[0.3,0.5)', lo: 0.3, hi: 0.5 },
|
|
261
|
+
{ label: '[0.5,1)', lo: 0.5, hi: 1 },
|
|
262
|
+
{ label: '[1,2)', lo: 1, hi: 2 },
|
|
263
|
+
{ label: '[2,5)', lo: 2, hi: 5 },
|
|
264
|
+
{ label: '[5,∞)', lo: 5, hi: Number.POSITIVE_INFINITY },
|
|
265
|
+
]
|
|
266
|
+
|
|
267
|
+
/** Zero-open listing threshold: injections at or above this with zero opens. */
|
|
268
|
+
export const ZERO_OPEN_MIN_INJECTIONS = 5
|
|
269
|
+
/** Rolling window of the zero-open rule: only rendered injections inside it count. */
|
|
270
|
+
export const ZERO_OPEN_WINDOW_DAYS = 30
|
|
271
|
+
/** Injection-tail rows the decay read scans — a bound comfortably covering
|
|
272
|
+
* the 30-day window at observed round volumes. */
|
|
273
|
+
export const ZERO_OPEN_TAIL = 500
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Slugs that truly rendered in one round, in delivery order: raw hits minus
|
|
277
|
+
* the round's exclusions (deduped/echoed/dropped were never assembled), plus
|
|
278
|
+
* delivered slow pointers; a slug the fast lane already delivered counts
|
|
279
|
+
* once even when the slow lane also reports it delivered ("not packed
|
|
280
|
+
* twice"). Scores ride along from the fast hit only — slow pointers carry
|
|
281
|
+
* no lexical score. Shared by the stats panel and the zero-open rule.
|
|
282
|
+
*/
|
|
283
|
+
export function renderedPointers(record: InjectionRecord): { slug: string; score?: number }[] {
|
|
284
|
+
const excluded = new Set<string>([
|
|
285
|
+
...(record.deduped ?? []),
|
|
286
|
+
...(record.echoed ?? []),
|
|
287
|
+
...(record.dropped ?? []).map((d) => d.slug),
|
|
288
|
+
])
|
|
289
|
+
const seen = new Set<string>()
|
|
290
|
+
const out: { slug: string; score?: number }[] = []
|
|
291
|
+
for (const h of record.hits) {
|
|
292
|
+
if (excluded.has(h.slug) || seen.has(h.slug)) continue
|
|
293
|
+
seen.add(h.slug)
|
|
294
|
+
out.push({ slug: h.slug, score: h.score })
|
|
295
|
+
}
|
|
296
|
+
for (const s of record.slow ?? []) {
|
|
297
|
+
if (seen.has(s.slug)) continue
|
|
298
|
+
seen.add(s.slug)
|
|
299
|
+
out.push({ slug: s.slug })
|
|
300
|
+
}
|
|
301
|
+
return out
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Zero-open slug set — the single shared 零打开高频 rule (v5): slugs whose
|
|
306
|
+
* truly-rendered pointer count within the rolling ZERO_OPEN_WINDOW_DAYS
|
|
307
|
+
* reaches ZERO_OPEN_MIN_INJECTIONS while the WHOLE open log has never
|
|
308
|
+
* recorded an open (wide scope — an open long after the last injection
|
|
309
|
+
* still disqualifies). Returns slug → windowed rendered-pointer count for
|
|
310
|
+
* the qualifying slugs; `/topics stats` (the panel's zero-open list) and
|
|
311
|
+
* the retrieval decay (`zeroOpenDecay`) both consume this one
|
|
312
|
+
* implementation. Records with unparsable `at` cannot prove window
|
|
313
|
+
* membership and are skipped (same tolerance as aggregateUsage).
|
|
314
|
+
*/
|
|
315
|
+
export function aggregateZeroOpen(
|
|
316
|
+
injections: readonly InjectionRecord[],
|
|
317
|
+
opens: readonly { slug: string }[],
|
|
318
|
+
nowMs: number,
|
|
319
|
+
): Map<string, number> {
|
|
320
|
+
const cutoff = nowMs - ZERO_OPEN_WINDOW_DAYS * 86_400_000
|
|
321
|
+
const counts = new Map<string, number>()
|
|
322
|
+
for (const record of injections) {
|
|
323
|
+
const at = Date.parse(record.at)
|
|
324
|
+
if (!Number.isFinite(at) || at < cutoff) continue
|
|
325
|
+
for (const p of renderedPointers(record)) {
|
|
326
|
+
counts.set(p.slug, (counts.get(p.slug) ?? 0) + 1)
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
const opened = new Set(opens.map((o) => o.slug))
|
|
330
|
+
const out = new Map<string, number>()
|
|
331
|
+
for (const [slug, n] of counts) {
|
|
332
|
+
if (n >= ZERO_OPEN_MIN_INJECTIONS && !opened.has(slug)) out.set(slug, n)
|
|
333
|
+
}
|
|
334
|
+
return out
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
export interface OpenTopicRow {
|
|
338
|
+
slug: string
|
|
339
|
+
/** Truly rendered pointers for this slug (deduped/echoed/dropped excluded). */
|
|
340
|
+
injections: number
|
|
341
|
+
/** Opens attributed by slug over the whole open log (wide scope, by design). */
|
|
342
|
+
opens: number
|
|
343
|
+
openRate: number
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
export interface OpenBandRow {
|
|
347
|
+
label: string
|
|
348
|
+
/** Rendered fast pointers whose hit score falls in this band. */
|
|
349
|
+
pointers: number
|
|
350
|
+
/** Pointer-sourced opens whose attributed score falls in this band. */
|
|
351
|
+
opens: number
|
|
352
|
+
openRate: number
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
export interface OpenPanelStats {
|
|
356
|
+
/** Rendered pointer entries across the window: hits − deduped − echoed − dropped, plus slow pointers (same-slug fast+slow pairs counted once). */
|
|
357
|
+
pointerEntries: number
|
|
358
|
+
/** DISTINCT slugs from the window's rendered set with an in-window open
|
|
359
|
+
* (search-sourced and never-rendered slugs excluded; one per slug). */
|
|
360
|
+
opensInWindow: number
|
|
361
|
+
openRate: number
|
|
362
|
+
topics: OpenTopicRow[]
|
|
363
|
+
zeroOpen: { slug: string; injections: number }[]
|
|
364
|
+
zeroOpenTotal: number
|
|
365
|
+
pointerOpens: number
|
|
366
|
+
searchOpens: number
|
|
367
|
+
/** False when no open record carries a source field (pre-enrichment data). */
|
|
368
|
+
hasSourceData: boolean
|
|
369
|
+
bands: OpenBandRow[]
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* Aggregate the injection-vs-open panel from the two JSONL windows. The
|
|
374
|
+
* per-topic/band denominators count only pointers that truly rendered
|
|
375
|
+
* (deduped/echoed/dropped excluded — same exclusion family as
|
|
376
|
+
* aggregateUsage); the header numerator counts only DISTINCT slugs from
|
|
377
|
+
* that rendered set with an open inside the injection window — a search
|
|
378
|
+
* open or an open of a never-rendered slug cannot enter, and repeat opens
|
|
379
|
+
* of the same slug count once, so the rate can no longer be pinned at
|
|
380
|
+
* 100% by repeated or off-scope opens. Opens per slug stay wide-scope
|
|
381
|
+
* (the model may open a topic long after its last injection). The
|
|
382
|
+
* zero-open list shares the decay's rule: rendered injections counted
|
|
383
|
+
* within the rolling 30-day window (anchored at the newest round), opens
|
|
384
|
+
* wide-scope.
|
|
385
|
+
*/
|
|
386
|
+
export function aggregateOpenPanel(
|
|
387
|
+
injections: readonly InjectionRecord[],
|
|
388
|
+
opens: readonly OpenRecord[],
|
|
389
|
+
): OpenPanelStats {
|
|
390
|
+
// Numerator window: the earliest round represented in the denominator;
|
|
391
|
+
// the zero-open anchor is the newest round — the panel describes the log
|
|
392
|
+
// window it was fed, so a stale log still shows the topics that
|
|
393
|
+
// dominated it. The rendered-slug set below shares that window: only
|
|
394
|
+
// opens of slugs this window rendered can enter the numerator.
|
|
395
|
+
let windowStart = Number.POSITIVE_INFINITY
|
|
396
|
+
let windowEnd = Number.NEGATIVE_INFINITY
|
|
397
|
+
for (const r of injections) {
|
|
398
|
+
const atMs = Date.parse(r.at)
|
|
399
|
+
if (!Number.isFinite(atMs)) continue
|
|
400
|
+
if (atMs < windowStart) windowStart = atMs
|
|
401
|
+
if (atMs > windowEnd) windowEnd = atMs
|
|
402
|
+
}
|
|
403
|
+
const topicCounts = new Map<string, number>()
|
|
404
|
+
const bandPointers = new Array<number>(SCORE_BANDS.length).fill(0)
|
|
405
|
+
let pointerEntries = 0
|
|
406
|
+
const renderedSlugs = new Set<string>()
|
|
407
|
+
for (const r of injections) {
|
|
408
|
+
// renderedPointers keeps the entry/per-topic counts honest: excluded
|
|
409
|
+
// slugs never counted, a fast+slow pair for the same slug counted once.
|
|
410
|
+
for (const p of renderedPointers(r)) {
|
|
411
|
+
pointerEntries += 1
|
|
412
|
+
renderedSlugs.add(p.slug)
|
|
413
|
+
topicCounts.set(p.slug, (topicCounts.get(p.slug) ?? 0) + 1)
|
|
414
|
+
const score = p.score
|
|
415
|
+
if (score === undefined) continue
|
|
416
|
+
const bi = SCORE_BANDS.findIndex((b) => score >= b.lo && score < b.hi)
|
|
417
|
+
if (bi >= 0) bandPointers[bi] += 1
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
const opensBySlug = new Map<string, number>()
|
|
421
|
+
const bandOpens = new Array<number>(SCORE_BANDS.length).fill(0)
|
|
422
|
+
let pointerOpens = 0
|
|
423
|
+
let searchOpens = 0
|
|
424
|
+
let hasSourceData = false
|
|
425
|
+
// Numerator: DISTINCT slugs from the window's rendered set with an
|
|
426
|
+
// in-window open — search-sourced opens never count (the model found the
|
|
427
|
+
// topic by asking, not by following a rendered pointer), a never-rendered
|
|
428
|
+
// slug never counts, and repeat opens of the same slug count once.
|
|
429
|
+
const openedSlugsInWindow = new Set<string>()
|
|
430
|
+
for (const o of opens) {
|
|
431
|
+
opensBySlug.set(o.slug, (opensBySlug.get(o.slug) ?? 0) + 1)
|
|
432
|
+
if (o.source === 'pointer' || o.source === 'search') hasSourceData = true
|
|
433
|
+
if (o.source === 'pointer') pointerOpens += 1
|
|
434
|
+
else if (o.source === 'search') searchOpens += 1
|
|
435
|
+
if (o.source === 'pointer' && typeof o.score === 'number') {
|
|
436
|
+
const score = o.score
|
|
437
|
+
const bi = SCORE_BANDS.findIndex((b) => score >= b.lo && score < b.hi)
|
|
438
|
+
if (bi >= 0) bandOpens[bi] += 1
|
|
439
|
+
}
|
|
440
|
+
if (o.source === 'search') continue
|
|
441
|
+
const atMs = Date.parse(o.at)
|
|
442
|
+
// An open joins the windowed numerator unless it is provably older than
|
|
443
|
+
// the window's earliest round (unparsable stamps cannot prove either
|
|
444
|
+
// way) — and only when the window actually rendered its slug.
|
|
445
|
+
if (Number.isFinite(windowStart) && Number.isFinite(atMs) && atMs < windowStart) continue
|
|
446
|
+
if (!renderedSlugs.has(o.slug)) continue
|
|
447
|
+
openedSlugsInWindow.add(o.slug)
|
|
448
|
+
}
|
|
449
|
+
const opensInWindow = openedSlugsInWindow.size
|
|
450
|
+
const rate = (num: number, den: number): number => (den === 0 ? 0 : Math.min(1, num / den))
|
|
451
|
+
const topics: OpenTopicRow[] = [...topicCounts.entries()]
|
|
452
|
+
.map(([slug, injections]) => {
|
|
453
|
+
const opens = opensBySlug.get(slug) ?? 0
|
|
454
|
+
return { slug, injections, opens, openRate: rate(opens, injections) }
|
|
455
|
+
})
|
|
456
|
+
.sort((a, b) => b.injections - a.injections || a.slug.localeCompare(b.slug))
|
|
457
|
+
// Zero-open cut: the shared 30-day rule (aggregateZeroOpen), anchored at
|
|
458
|
+
// the newest round so window semantics match the decay's rolling window
|
|
459
|
+
// without growing a time-bomb dependency on the wall clock.
|
|
460
|
+
const zeroOpenCounts =
|
|
461
|
+
windowEnd === Number.NEGATIVE_INFINITY ? new Map<string, number>() : aggregateZeroOpen(injections, opens, windowEnd)
|
|
462
|
+
const zeroOpenAll = [...zeroOpenCounts.entries()]
|
|
463
|
+
.map(([slug, injections]) => ({ slug, injections }))
|
|
464
|
+
.sort((a, b) => b.injections - a.injections || a.slug.localeCompare(b.slug))
|
|
465
|
+
return {
|
|
466
|
+
pointerEntries,
|
|
467
|
+
opensInWindow,
|
|
468
|
+
openRate: rate(opensInWindow, pointerEntries),
|
|
469
|
+
topics: topics.slice(0, 10),
|
|
470
|
+
zeroOpen: zeroOpenAll.slice(0, 10).map((t) => ({ slug: t.slug, injections: t.injections })),
|
|
471
|
+
zeroOpenTotal: zeroOpenAll.length,
|
|
472
|
+
pointerOpens,
|
|
473
|
+
searchOpens,
|
|
474
|
+
hasSourceData,
|
|
475
|
+
bands: SCORE_BANDS.map((b, i) => ({
|
|
476
|
+
label: b.label,
|
|
477
|
+
pointers: bandPointers[i],
|
|
478
|
+
opens: bandOpens[i],
|
|
479
|
+
openRate: rate(bandOpens[i], bandPointers[i]),
|
|
480
|
+
})),
|
|
481
|
+
}
|
|
482
|
+
}
|
package/src/retrieval.ts
CHANGED
|
@@ -49,12 +49,26 @@ export interface RetrievalConfig {
|
|
|
49
49
|
usageBoost: number
|
|
50
50
|
/** Usage signals per slug (hits/opens in the window); absent map = no boost. */
|
|
51
51
|
usage?: ReadonlyMap<string, { hits: number; opens: number }>
|
|
52
|
+
/**
|
|
53
|
+
* Zero-open decay (v5, default off — config `zeroOpenDecay`): halve the
|
|
54
|
+
* score of slugs on the zero-open list (≥5 rendered injections in the
|
|
55
|
+
* rolling 30 days, zero opens ever) at score-finalization time, BEFORE
|
|
56
|
+
* the threshold cut — a decayed candidate that drops below the threshold
|
|
57
|
+
* lands in nearMisses with the `zero-open-decay` reason, so the log
|
|
58
|
+
* shows why it stopped injecting. Absent/off = zero behavior change.
|
|
59
|
+
*/
|
|
60
|
+
zeroOpenDecay?: boolean
|
|
61
|
+
/** Zero-open slugs for this round (service-built from the ilog tail). */
|
|
62
|
+
zeroOpenSlugs?: ReadonlySet<string>
|
|
52
63
|
}
|
|
53
64
|
|
|
54
65
|
/** Hard cap for the usage bonus — below threshold 0.3 so usage can never
|
|
55
66
|
* carry a candidate across the gate on its own (ADR 0015). */
|
|
56
67
|
export const USAGE_BOOST_CAP = 0.2
|
|
57
68
|
|
|
69
|
+
/** Score multiplier for zero-open slugs when zeroOpenDecay is on (v5). */
|
|
70
|
+
export const ZERO_OPEN_DECAY_FACTOR = 0.5
|
|
71
|
+
|
|
58
72
|
export const DEFAULT_RETRIEVAL: RetrievalConfig = {
|
|
59
73
|
threshold: 0.3,
|
|
60
74
|
topK: 4,
|
|
@@ -224,6 +238,14 @@ export function scoreTopic(
|
|
|
224
238
|
score *= 0.3
|
|
225
239
|
reasons.push('conflicted-demoted')
|
|
226
240
|
}
|
|
241
|
+
// Zero-open decay (v5): same demotion family as conflicts — applied to
|
|
242
|
+
// the content score BEFORE gateScore is taken, so the threshold cut and
|
|
243
|
+
// the near-miss band both see the halved score; recency still rides
|
|
244
|
+
// outside it as the ranking tiebreaker.
|
|
245
|
+
if (cfg.zeroOpenDecay === true && cfg.zeroOpenSlugs?.has(topic.slug) === true) {
|
|
246
|
+
score *= ZERO_OPEN_DECAY_FACTOR
|
|
247
|
+
reasons.push('zero-open-decay')
|
|
248
|
+
}
|
|
227
249
|
const gateScore = score
|
|
228
250
|
score += recencyBonus
|
|
229
251
|
const bodySet = new Set([...description, ...conclusion])
|
package/src/service.ts
CHANGED
|
@@ -10,11 +10,11 @@ import { readFile, stat } from 'node:fs/promises'
|
|
|
10
10
|
import { readdirSync, readFileSync, statSync } from 'node:fs'
|
|
11
11
|
import { join } from 'node:path'
|
|
12
12
|
import * as okf from './okf.ts'
|
|
13
|
-
import { searchTopics, scoreTopic, passesGate, tokenize, type RetrievableTopic, type SearchOutcome } from './retrieval.ts'
|
|
13
|
+
import { searchTopics, scoreTopic, passesGate, tokenize, type RetrievableTopic, type RetrievalConfig, type SearchOutcome } from './retrieval.ts'
|
|
14
14
|
import { assembleInjection, assemblePointer, POINTER_PER_TOPIC, POINTER_TOTAL, type AssembleResult, type DigestInput, type SlowPointerInput } from './digest.ts'
|
|
15
15
|
import type { BundleStore, Observation, SaveResult } from './store.ts'
|
|
16
16
|
import type { TopicsConfigValue } from './config.ts'
|
|
17
|
-
import { aggregateStats, querySample, aggregateUsage, USAGE_WINDOW_DAYS, type AggregateStats, type InjectionRecord, type QueryBuildShape, type ShadowVerdict, type SlowItem, type UsageSignal } from './ilog.ts'
|
|
17
|
+
import { aggregateStats, querySample, aggregateUsage, aggregateZeroOpen, ZERO_OPEN_TAIL, USAGE_WINDOW_DAYS, attributeOpen, OPEN_ATTRIBUTION_TAIL, type AggregateStats, type InjectionRecord, type OpenRecord, type QueryBuildShape, type ShadowVerdict, type SlowItem, type UsageSignal } from './ilog.ts'
|
|
18
18
|
import { fileHistory, fileAtRev } from './git.ts'
|
|
19
19
|
import type { Sync } from './sync.ts'
|
|
20
20
|
|
|
@@ -25,6 +25,37 @@ interface CacheEntry {
|
|
|
25
25
|
slug: string
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
/** Tolerant JSONL line read: undefined = unreadable file. Unlike an empty
|
|
29
|
+
* array, undefined tells the caller to fail open AND skip caching — a
|
|
30
|
+
* successful stat paired with a failed read must not freeze an empty
|
|
31
|
+
* snapshot into the cache until the next append. */
|
|
32
|
+
function readLogLinesSync(file: string): string[] | undefined {
|
|
33
|
+
try {
|
|
34
|
+
return readFileSync(file, 'utf8').split('\n')
|
|
35
|
+
} catch {
|
|
36
|
+
return undefined
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Parse already-split lines back-to-front, keeping the last `tail` parsed
|
|
41
|
+
* records in order: blanks and torn JSON lines are skipped (and never
|
|
42
|
+
* consume tail budget), so a torn tail line cannot crowd out a good one.
|
|
43
|
+
* Shared by the usage/zero-open cache — one tail parse feeds both. */
|
|
44
|
+
function parseJsonlLinesSync<T>(lines: readonly string[], tail = Number.POSITIVE_INFINITY): T[] {
|
|
45
|
+
const out: T[] = []
|
|
46
|
+
for (let i = lines.length - 1; i >= 0 && out.length < tail; i -= 1) {
|
|
47
|
+
const line = lines[i]
|
|
48
|
+
if (line.trim() === '') continue
|
|
49
|
+
try {
|
|
50
|
+
out.push(JSON.parse(line) as T)
|
|
51
|
+
} catch {
|
|
52
|
+
continue // torn tail line
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
out.reverse()
|
|
56
|
+
return out
|
|
57
|
+
}
|
|
58
|
+
|
|
28
59
|
export interface RetrieveOutcome {
|
|
29
60
|
outcome: SearchOutcome
|
|
30
61
|
injection: AssembleResult
|
|
@@ -50,13 +81,22 @@ export class TopicsService {
|
|
|
50
81
|
private cache = new Map<string, CacheEntry>()
|
|
51
82
|
/** mtime-keyed parse of the observations log, grouped by session. */
|
|
52
83
|
private echoCache: { mtimeMs: number; size: number; bySession: Map<string, Set<string>> } | undefined
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
84
|
+
/** Single shared cache over the two feedback JSONL logs (S2): the key is
|
|
85
|
+
* the (mtime,size) quartet of injections.jsonl + opens.jsonl, one read
|
|
86
|
+
* and one tail parse feed BOTH aggregates — usageSignalsSync and
|
|
87
|
+
* zeroOpenSlugsSync used to each keep a full-parse cache of their own.
|
|
88
|
+
* Invalidation stays append-driven: the store only ever appends, so any
|
|
89
|
+
* write moves mtime+size. Never populated when a read failed (N1). */
|
|
90
|
+
private logSignalsCache:
|
|
91
|
+
| {
|
|
92
|
+
injMs: number
|
|
93
|
+
injSize: number
|
|
94
|
+
opensMs: number
|
|
95
|
+
opensSize: number
|
|
96
|
+
signals: Map<string, UsageSignal>
|
|
97
|
+
zeroOpenSlugs: ReadonlySet<string>
|
|
98
|
+
}
|
|
99
|
+
| undefined
|
|
60
100
|
readonly store: BundleStore
|
|
61
101
|
private readonly getConfig: () => TopicsConfigValue
|
|
62
102
|
readonly sync?: Sync
|
|
@@ -131,7 +171,7 @@ export class TopicsService {
|
|
|
131
171
|
invalidate(): void {
|
|
132
172
|
this.cache.clear()
|
|
133
173
|
this.echoCache = undefined
|
|
134
|
-
this.
|
|
174
|
+
this.logSignalsCache = undefined
|
|
135
175
|
}
|
|
136
176
|
|
|
137
177
|
/**
|
|
@@ -186,10 +226,36 @@ export class TopicsService {
|
|
|
186
226
|
* Usage signals (ADR 0015): mtime-cached aggregate over the injection log
|
|
187
227
|
* and the opens log, rolling USAGE_WINDOW_DAYS. Synchronous like
|
|
188
228
|
* echoSlugsSync — the retrieval hot path is sync — and cheap when warm:
|
|
189
|
-
* a fresh cache costs two stat() calls
|
|
229
|
+
* a fresh cache costs two stat() calls, one read of each log, and feeds
|
|
230
|
+
* the zero-open set from the same parse. Unreadable logs fail open to an
|
|
190
231
|
* empty map (no boost), never an error.
|
|
191
232
|
*/
|
|
192
233
|
usageSignalsSync(): Map<string, UsageSignal> {
|
|
234
|
+
return this.logSignalsSync().signals
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Zero-open slug set for the decay (v5): the shared aggregateZeroOpen
|
|
239
|
+
* rule over the injection tail and the whole opens log (compacted, so a
|
|
240
|
+
* full read stays small). Shares one cache and one parse with
|
|
241
|
+
* usageSignalsSync; unreadable logs fail open to an empty set (no
|
|
242
|
+
* decay), never an error. Only called when zeroOpenDecay is on.
|
|
243
|
+
*/
|
|
244
|
+
zeroOpenSlugsSync(): ReadonlySet<string> {
|
|
245
|
+
return this.logSignalsSync().zeroOpenSlugs
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* The one shared feedback-log read behind both signals: stat both files
|
|
250
|
+
* (mtime+size quartet key), parse the injections TAIL only
|
|
251
|
+
* (ZERO_OPEN_TAIL rounds comfortably cover both 30-day windows) and the
|
|
252
|
+
* opens log whole (the zero-open rule is whole-history), then feed the
|
|
253
|
+
* two pure aggregates from that single parse. A failed read still fails
|
|
254
|
+
* open to empty inputs but NEVER populates the cache (N1) — otherwise a
|
|
255
|
+
* stat-success/read-failure race would freeze an empty snapshot until
|
|
256
|
+
* the next append.
|
|
257
|
+
*/
|
|
258
|
+
private logSignalsSync(): { signals: Map<string, UsageSignal>; zeroOpenSlugs: ReadonlySet<string> } {
|
|
193
259
|
const injFile = join(this.store.metaDir(), 'injections.jsonl')
|
|
194
260
|
const opensFile = join(this.store.metaDir(), 'opens.jsonl')
|
|
195
261
|
const statOf = (file: string): { mtimeMs: number; size: number } => {
|
|
@@ -202,7 +268,7 @@ export class TopicsService {
|
|
|
202
268
|
}
|
|
203
269
|
const injSt = statOf(injFile)
|
|
204
270
|
const opensSt = statOf(opensFile)
|
|
205
|
-
const cached = this.
|
|
271
|
+
const cached = this.logSignalsCache
|
|
206
272
|
if (
|
|
207
273
|
cached !== undefined &&
|
|
208
274
|
cached.injMs === injSt.mtimeMs &&
|
|
@@ -210,38 +276,38 @@ export class TopicsService {
|
|
|
210
276
|
cached.opensMs === opensSt.mtimeMs &&
|
|
211
277
|
cached.opensSize === opensSt.size
|
|
212
278
|
) {
|
|
213
|
-
return cached
|
|
279
|
+
return cached
|
|
214
280
|
}
|
|
215
|
-
const
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
281
|
+
const injLines = readLogLinesSync(injFile)
|
|
282
|
+
const opensLines = readLogLinesSync(opensFile)
|
|
283
|
+
const injections = parseJsonlLinesSync<InjectionRecord>(injLines ?? [], ZERO_OPEN_TAIL)
|
|
284
|
+
const opens = parseJsonlLinesSync<OpenRecord>(opensLines ?? [])
|
|
285
|
+
const now = Date.now()
|
|
286
|
+
const signals = aggregateUsage(injections, opens, USAGE_WINDOW_DAYS, now)
|
|
287
|
+
const zeroOpenSlugs: ReadonlySet<string> = new Set(aggregateZeroOpen(injections, opens, now).keys())
|
|
288
|
+
if (injLines !== undefined && opensLines !== undefined) {
|
|
289
|
+
this.logSignalsCache = {
|
|
290
|
+
injMs: injSt.mtimeMs,
|
|
291
|
+
injSize: injSt.size,
|
|
292
|
+
opensMs: opensSt.mtimeMs,
|
|
293
|
+
opensSize: opensSt.size,
|
|
294
|
+
signals,
|
|
295
|
+
zeroOpenSlugs,
|
|
228
296
|
}
|
|
229
|
-
return out
|
|
230
|
-
}
|
|
231
|
-
const signals = aggregateUsage(
|
|
232
|
-
parseLines(injFile),
|
|
233
|
-
parseLines(opensFile),
|
|
234
|
-
USAGE_WINDOW_DAYS,
|
|
235
|
-
Date.now(),
|
|
236
|
-
)
|
|
237
|
-
this.usageCache = {
|
|
238
|
-
injMs: injSt.mtimeMs,
|
|
239
|
-
injSize: injSt.size,
|
|
240
|
-
opensMs: opensSt.mtimeMs,
|
|
241
|
-
opensSize: opensSt.size,
|
|
242
|
-
signals,
|
|
243
297
|
}
|
|
244
|
-
return signals
|
|
298
|
+
return { signals, zeroOpenSlugs }
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* zeroOpenDecay fields for scoring calls: empty when the gate is off —
|
|
303
|
+
* no file reads, zero behavior change — and the shared mtime-cached set
|
|
304
|
+
* when on. Spread into every searchTopics/scoreTopic cfg that already
|
|
305
|
+
* carries the usage signals (injection paths + explicit topic_search;
|
|
306
|
+
* the slow lane's recall band stays untouched, mirroring usageBoost).
|
|
307
|
+
*/
|
|
308
|
+
private zeroOpenCfg(): Pick<RetrievalConfig, 'zeroOpenDecay' | 'zeroOpenSlugs'> {
|
|
309
|
+
if (this.cfg.zeroOpenDecay !== true) return {}
|
|
310
|
+
return { zeroOpenDecay: true, zeroOpenSlugs: this.zeroOpenSlugsSync() }
|
|
245
311
|
}
|
|
246
312
|
|
|
247
313
|
/**
|
|
@@ -269,6 +335,7 @@ export class TopicsService {
|
|
|
269
335
|
conflicts,
|
|
270
336
|
usageBoost: cfg.usageBoost,
|
|
271
337
|
usage: this.usageSignalsSync(),
|
|
338
|
+
...this.zeroOpenCfg(),
|
|
272
339
|
})
|
|
273
340
|
const bySlug = new Map(roster.map((r) => [r.slug, r]))
|
|
274
341
|
const entries: DigestInput[] = []
|
|
@@ -368,6 +435,7 @@ export class TopicsService {
|
|
|
368
435
|
conflicts,
|
|
369
436
|
usageBoost: cfg.usageBoost,
|
|
370
437
|
usage: this.usageSignalsSync(),
|
|
438
|
+
...this.zeroOpenCfg(),
|
|
371
439
|
})
|
|
372
440
|
const exclude = dedup?.exclude
|
|
373
441
|
const echo = dedup?.echo
|
|
@@ -427,6 +495,7 @@ export class TopicsService {
|
|
|
427
495
|
structuralGate: true,
|
|
428
496
|
usageBoost: cfg.usageBoost,
|
|
429
497
|
usage: this.usageSignalsSync(),
|
|
498
|
+
...this.zeroOpenCfg(),
|
|
430
499
|
})
|
|
431
500
|
shadow.push({
|
|
432
501
|
slug: item.slug,
|
|
@@ -532,9 +601,14 @@ export class TopicsService {
|
|
|
532
601
|
const doc = slug === '' ? undefined : await this.store.readTopic(slug).catch(() => undefined)
|
|
533
602
|
if (doc === undefined) return { found: false, slug }
|
|
534
603
|
// Awaited (tool path, not hot): a settled write makes the pointer-open
|
|
535
|
-
// stat reliable; a failed log write must not fail the tool.
|
|
604
|
+
// stat reliable; a failed log write must not fail the tool. Source
|
|
605
|
+
// attribution rides the same write: a cheap injections-tail scan decides
|
|
606
|
+
// pointer vs search and, for pointers, carries the hit score and the
|
|
607
|
+
// open→injection lag (ilog.attributeOpen).
|
|
608
|
+
const at = new Date().toISOString()
|
|
609
|
+
const attribution = await this.attributeOpen(slug, sessionId, Date.parse(at))
|
|
536
610
|
await this.store
|
|
537
|
-
.appendOpenRecord({ slug, at
|
|
611
|
+
.appendOpenRecord({ slug, at, ...(sessionId !== undefined ? { sessionId } : {}), ...attribution })
|
|
538
612
|
.catch(() => undefined)
|
|
539
613
|
// Optional fields are OMITTED, never present-as-undefined: the host's
|
|
540
614
|
// tool-output validation rejects any object carrying an undefined-valued
|
|
@@ -564,6 +638,18 @@ export class TopicsService {
|
|
|
564
638
|
return found
|
|
565
639
|
}
|
|
566
640
|
|
|
641
|
+
/**
|
|
642
|
+
* Resolve the open's source attribution: the injections tail (bounded by
|
|
643
|
+
* OPEN_ATTRIBUTION_TAIL) scanned for a same-session hit within the
|
|
644
|
+
* attribution window. Read failures degrade to 'search' — attribution is
|
|
645
|
+
* telemetry, never a blocker.
|
|
646
|
+
*/
|
|
647
|
+
private async attributeOpen(slug: string, sessionId: string | undefined, nowMs: number): Promise<{ source: 'pointer' | 'search'; score?: number; sinceInjectionMs?: number }> {
|
|
648
|
+
if (sessionId === undefined) return { source: 'search' }
|
|
649
|
+
const tail = await this.store.readInjectionRecords(OPEN_ATTRIBUTION_TAIL).catch(() => [])
|
|
650
|
+
return attributeOpen(tail, slug, sessionId, nowMs)
|
|
651
|
+
}
|
|
652
|
+
|
|
567
653
|
/** Sync roster read (same mtime cache as roster()). */
|
|
568
654
|
rosterSync(): RetrievableTopic[] {
|
|
569
655
|
let files: string[]
|
|
@@ -706,6 +792,7 @@ export class TopicsService {
|
|
|
706
792
|
conflicts: await this.store.getConflicts(),
|
|
707
793
|
usageBoost: cfg.usageBoost,
|
|
708
794
|
usage: this.usageSignalsSync(),
|
|
795
|
+
...this.zeroOpenCfg(),
|
|
709
796
|
})
|
|
710
797
|
}
|
|
711
798
|
|
package/src/store.ts
CHANGED
|
@@ -26,7 +26,7 @@ import { randomUUID } from 'node:crypto'
|
|
|
26
26
|
import * as okf from './okf.ts'
|
|
27
27
|
import * as gitmod from './git.ts'
|
|
28
28
|
import { actorFor } from './paths.ts'
|
|
29
|
-
import type { InjectionRecord } from './ilog.ts'
|
|
29
|
+
import type { InjectionRecord, OpenRecord } from './ilog.ts'
|
|
30
30
|
|
|
31
31
|
export interface TopicMeta {
|
|
32
32
|
slug: string
|
|
@@ -612,14 +612,15 @@ export class BundleStore {
|
|
|
612
612
|
|
|
613
613
|
// ------------------------------------------------------------------
|
|
614
614
|
// Pointer-open log (v4 §4.3) — one line per topic_open call; feeds the
|
|
615
|
-
// 「指针打开率」 stat. Features only (slug/time), never
|
|
615
|
+
// 「指针打开率」 stat. Features only (slug/time + attribution), never
|
|
616
|
+
// conversation text.
|
|
616
617
|
// ------------------------------------------------------------------
|
|
617
618
|
|
|
618
619
|
private opensPath(): string {
|
|
619
620
|
return join(this.metaDir(), 'opens.jsonl')
|
|
620
621
|
}
|
|
621
622
|
|
|
622
|
-
async appendOpenRecord(record:
|
|
623
|
+
async appendOpenRecord(record: OpenRecord): Promise<void> {
|
|
623
624
|
await mkdir(this.metaDir(), { recursive: true })
|
|
624
625
|
await appendFile(this.opensPath(), `${JSON.stringify(record)}\n`, 'utf8')
|
|
625
626
|
await this.compactOpensIfNeeded()
|
|
@@ -638,22 +639,27 @@ export class BundleStore {
|
|
|
638
639
|
await atomicWrite(file, lines.slice(-Math.max(1, Math.floor(lines.length / 4))).map((l) => `${l}\n`).join(''))
|
|
639
640
|
}
|
|
640
641
|
|
|
641
|
-
/** Opens in the recent window; tolerant of a torn tail like the other JSONLs.
|
|
642
|
-
|
|
642
|
+
/** Opens in the recent window; tolerant of a torn tail like the other JSONLs.
|
|
643
|
+
* Enrichment fields (sessionId/source/score/sinceInjectionMs) are optional:
|
|
644
|
+
* pre-enrichment lines carry only slug/at and read back without them. */
|
|
645
|
+
async readOpenRecords(limit = 2000): Promise<OpenRecord[]> {
|
|
643
646
|
let raw: string
|
|
644
647
|
try {
|
|
645
648
|
raw = await readFile(this.opensPath(), 'utf8')
|
|
646
649
|
} catch {
|
|
647
650
|
return []
|
|
648
651
|
}
|
|
649
|
-
const out:
|
|
652
|
+
const out: OpenRecord[] = []
|
|
650
653
|
for (const line of raw.split('\n')) {
|
|
651
654
|
if (line.trim() === '') continue
|
|
652
655
|
try {
|
|
653
|
-
const parsed = JSON.parse(line) as { slug?: unknown; at?: unknown; sessionId?: unknown }
|
|
656
|
+
const parsed = JSON.parse(line) as { slug?: unknown; at?: unknown; sessionId?: unknown; source?: unknown; score?: unknown; sinceInjectionMs?: unknown }
|
|
654
657
|
if (typeof parsed.slug === 'string' && typeof parsed.at === 'string') {
|
|
655
|
-
const rec:
|
|
658
|
+
const rec: OpenRecord = { slug: parsed.slug, at: parsed.at }
|
|
656
659
|
if (typeof parsed.sessionId === 'string') rec.sessionId = parsed.sessionId
|
|
660
|
+
if (parsed.source === 'pointer' || parsed.source === 'search') rec.source = parsed.source
|
|
661
|
+
if (typeof parsed.score === 'number') rec.score = parsed.score
|
|
662
|
+
if (typeof parsed.sinceInjectionMs === 'number') rec.sinceInjectionMs = parsed.sinceInjectionMs
|
|
657
663
|
out.push(rec)
|
|
658
664
|
}
|
|
659
665
|
} catch {
|