@aiwayds/dsh-topics-memory 0.6.0 → 0.7.0

package/src/service.ts

@@ -10,11 +10,11 @@ import { readFile, stat } from 'node:fs/promises'
 import { readdirSync, readFileSync, statSync } from 'node:fs'
 import { join } from 'node:path'
 import * as okf from './okf.ts'
-import { searchTopics, tokenize, type RetrievableTopic, type SearchOutcome } from './retrieval.ts'
-import { assembleInjection, type AssembleResult, type DigestInput } from './digest.ts'
+import { searchTopics, scoreTopic, passesGate, tokenize, type RetrievableTopic, type SearchOutcome } from './retrieval.ts'
+import { assembleInjection, assemblePointer, POINTER_PER_TOPIC, POINTER_TOTAL, type AssembleResult, type DigestInput, type SlowPointerInput } from './digest.ts'
 import type { BundleStore, Observation, SaveResult } from './store.ts'
 import type { TopicsConfigValue } from './config.ts'
-import { aggregateStats, querySample, type AggregateStats, type InjectionRecord } from './ilog.ts'
+import { aggregateStats, querySample, type AggregateStats, type InjectionRecord, type QueryBuildShape, type ShadowVerdict, type SlowItem } from './ilog.ts'
 import { fileHistory, fileAtRev } from './git.ts'
 import type { Sync } from './sync.ts'
 
@@ -32,6 +32,20 @@ export interface RetrieveOutcome {
   recorded: boolean
 }
 
+/**
+ * Slow-lane payload handed to the hot path at consumption time (v4 §4.2).
+ * `items` are the rerank picks (0-2, why lines included); metadata feeds the
+ * ilog lane field family. The hot path resolves titles, filters dedup,
+ * renders, and takes the log-only shadow verdicts.
+ */
+export interface SlowDelivery {
+  items: readonly { slug: string; why: string }[]
+  computedAt: string
+  queryBuild: QueryBuildShape
+  model: string
+  ms: number
+}
+
 export class TopicsService {
   private cache = new Map<string, CacheEntry>()
   readonly store: BundleStore
@@ -89,6 +103,7 @@ export class TopicsService {
         description: doc.fm.description,
         status: doc.fm.status,
         tags: doc.fm.tags,
+        triggers: doc.fm.triggers,
         depends: doc.fm.depends,
         links: okf.bodyLinkSlugs(doc.body),
         generatedAt: doc.fm.generated.at,
@@ -135,10 +150,13 @@ export class TopicsService {
       const doc = await this.readDoc(hit.slug)
       if (doc !== undefined) entries.push({ slug: hit.slug, doc, hit })
     }
-    const injection = assembleInjection(entries, {
-      perTopicBudget: cfg.perTopicBudget,
-      totalBudget: cfg.totalBudget,
-    })
+    const injection =
+      cfg.injectMode !== 'digest'
+        ? assemblePointer(entries, { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) })
+        : assembleInjection(entries, {
+            perTopicBudget: cfg.perTopicBudget,
+            totalBudget: cfg.totalBudget,
+          })
     const record: InjectionRecord = {
       at: new Date().toISOString(),
       queryTokenCount: tokenize(query).length,
@@ -163,18 +181,50 @@ export class TopicsService {
    * would always lose the race and inject nothing. Reads files with sync fs
    * behind the mtime cache; the log record is written fire-and-forget.
    *
-   * `dedup.exclude` (session-level injection dedup) filters hits AFTER
-   * retrieval and BEFORE assembly: excluded slugs are reported back as
-   * `deduped` (and logged as `record.deduped`), never packed. `included`
-   * mirrors what actually entered the context — only these may be marked
-   * as injected by the caller; budget-dropped slugs stay injectable later.
+   * `dedup.exclude` (session-level injection dedup) filters hits AND slow
+   * picks before assembly; excluded slugs are reported back as `deduped`
+   * (and logged as `record.deduped`), never packed. `included` mirrors the
+   * fast pointers that entered the context and `slowIncluded` the slow ones
+   * — only these may be marked as injected by the caller; budget-dropped
+   * slugs stay injectable later.
+   *
+   * `slow` (v4 §4.2) carries the consumed pending when one exists: picks are
+   * packed after the fast pointers under the same budget, take log-only
+   * shadow re-gate verdicts against the CURRENT query (B3: record, never
+   * block), and fill the record's lane field family.
    */
-  retrieveSync(query: string, sessionId?: string, dedup?: { exclude?: ReadonlySet<string> }): { text: string; outcome: SearchOutcome; deduped: string[]; included: string[] } {
+  retrieveSync(query: string, sessionId?: string, dedup?: { exclude?: ReadonlySet<string> }, slow?: SlowDelivery, slowExpired?: 'ttl' | 'turn-lag'): { text: string; outcome: SearchOutcome; deduped: string[]; included: string[]; slowIncluded: string[] } {
     const cfg = this.cfg
     const roster = this.rosterSync()
     const empty: SearchOutcome = { hits: [], nearMisses: [], rosterSize: roster.length }
     if (!cfg.autoInject || query.trim() === '' || roster.length === 0) {
-      return { text: '', outcome: empty, deduped: [], included: [] }
+      // A consumed/expired pending must always leave an ilog trace (ADR 0014:
+      // 过期/消费都写 ilog) — even when the fast path early-returns here
+      // because the roster vanished between production and consumption.
+      if (cfg.autoInject && query.trim() !== '' && (slow !== undefined || slowExpired !== undefined)) {
+        const trace: InjectionRecord = {
+          at: new Date().toISOString(),
+          queryTokenCount: tokenize(query).length,
+          querySample: querySample(query),
+          rosterSize: roster.length,
+          hits: [],
+          nearMisses: [],
+          injected: false,
+          why: slowExpired !== undefined ? `slow-expired-${slowExpired}` : 'slow-no-roster',
+          lane: 'slow',
+        }
+        if (sessionId !== undefined) trace.sessionId = sessionId
+        if (slowExpired !== undefined) trace.slowExpired = slowExpired
+        if (slow !== undefined) {
+          trace.computedAt = slow.computedAt
+          trace.queryBuild = slow.queryBuild
+          trace.slowModel = slow.model
+          trace.slowMs = slow.ms
+          trace.slow = []
+        }
+        void this.store.appendInjectionRecord(trace).catch(() => undefined)
+      }
+      return { text: '', outcome: empty, deduped: [], included: [], slowIncluded: [] }
     }
     const conflicts = this.store.getConflictsSync()
     const outcome = searchTopics(query, roster, {
@@ -187,35 +237,154 @@ export class TopicsService {
     })
     const exclude = dedup?.exclude
     const deduped = exclude === undefined ? [] : outcome.hits.filter((h) => exclude.has(h.slug)).map((h) => h.slug)
+    const bySlug = new Map(roster.map((r) => [r.slug, r]))
     const entries: DigestInput[] = []
+    const unreadable: string[] = []
     for (const hit of outcome.hits) {
       if (exclude?.has(hit.slug)) continue
       const doc = this.readDocSync(hit.slug)
-      if (doc !== undefined) entries.push({ slug: hit.slug, doc, hit })
+      if (doc === undefined) {
+        // The hit has no readable doc — without this accounting it used to
+        // vanish from `included` AND `dropped` (v4 design §2 疑点).
+        unreadable.push(hit.slug)
+        continue
+      }
+      entries.push({ slug: hit.slug, doc, hit })
+    }
+    // ---- Slow-lane merge (log-only shadow gate; consumption bookkeeping) ----
+    const slowPointers: SlowPointerInput[] = []
+    const slowDelivered: SlowItem[] = []
+    const shadow: ShadowVerdict[] = []
+    let slowIncluded: string[] = []
+    const consumedAt = new Date().toISOString()
+    if (slow !== undefined) {
+      const queryTokens = new Set(tokenize(query))
+      for (const item of slow.items) {
+        if (exclude?.has(item.slug)) {
+          if (!deduped.includes(item.slug)) deduped.push(item.slug)
+          continue
+        }
+        const meta = bySlug.get(item.slug)
+        if (meta === undefined) continue // vanished between production and consumption
+        const verdict = scoreTopic(queryTokens, meta, {
+          threshold: cfg.matchThreshold,
+          topK: cfg.topK,
+          tagBoost: cfg.tagBoost,
+          graphDepth: cfg.graphDepth,
+          recencyWindowDays: cfg.recencyWindowDays,
+          conflicts,
+          structuralGate: true,
+        })
+        shadow.push({
+          slug: item.slug,
+          pass: passesGate(verdict),
+          why: verdict.strong ? `strong:${verdict.reasons[0] ?? 'field'}` : `body-hits:${verdict.bodyHits}`,
+        })
+        slowPointers.push({
+          slug: item.slug,
+          title: meta.title,
+          status: meta.status,
+          description: meta.description,
+          why: item.why,
+        })
+      }
+    }
+    const pointerMode = cfg.injectMode !== 'digest'
+    // Pointer budget: per-entry fixed at the design's 80, total clamped to the
+    // fast-lane upper bound of 600 (I4) — totalBudget stays tunable downward.
+    const injection = pointerMode
+      ? assemblePointer(entries, { perTopicBudget: POINTER_PER_TOPIC, totalBudget: Math.min(cfg.totalBudget, POINTER_TOTAL) }, slowPointers)
+      : assembleInjection(entries, { perTopicBudget: cfg.perTopicBudget, totalBudget: cfg.totalBudget }, slowPointers)
+    slowIncluded = injection.slowIncluded ?? []
+    for (const pointer of slowPointers) {
+      if (slowIncluded.includes(pointer.slug)) slowDelivered.push({ slug: pointer.slug, why: pointer.why })
     }
-    const injection = assembleInjection(entries, {
-      perTopicBudget: cfg.perTopicBudget,
-      totalBudget: cfg.totalBudget,
-    })
     const record: InjectionRecord = {
       at: new Date().toISOString(),
       queryTokenCount: tokenize(query).length,
       querySample: querySample(query),
       rosterSize: outcome.rosterSize,
-      hits: outcome.hits.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons, viaGraph: h.viaGraph })),
-      nearMisses: outcome.nearMisses.map((h) => ({ slug: h.slug, score: h.score })),
+      hits: outcome.hits.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons, viaGraph: h.viaGraph, strong: h.strong, bodyHits: h.bodyHits })),
+      // Reasons ride along: the gate-blocked marker is the replay evidence
+      // the P3 threshold decision needs (ADR 0014).
+      nearMisses: outcome.nearMisses.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons })),
       injected: injection.text !== '',
       usedTokens: injection.usedTokens,
     }
     if (sessionId !== undefined) record.sessionId = sessionId
     if (deduped.length > 0) record.deduped = deduped
-    if (injection.text === '' && outcome.hits.length > 0) {
+    if (unreadable.length > 0) {
+      record.dropped = [...(record.dropped ?? []), ...unreadable.map((slug) => ({ slug, reason: 'doc-unreadable' }))]
+    }
+    if (slow !== undefined) {
+      // Any round whose pending was consumed is a slow-participating round —
+      // lane never stays undefined when the lane family is present (a
+      // delivered pick with fast pointers is 'mixed'; a consumed-but-empty
+      // round beside fast pointers stays 'fast'; everything else 'slow').
+      record.lane = slowDelivered.length > 0 ? (injection.included.length > 0 ? 'mixed' : 'slow') : injection.included.length > 0 ? 'fast' : 'slow'
+      record.computedAt = slow.computedAt
+      record.consumedAt = consumedAt
+      if (shadow.length > 0) record.shadowVerdict = shadow
+      record.queryBuild = slow.queryBuild
+      record.slowModel = slow.model
+      record.slowMs = slow.ms
+      record.slow = slowDelivered
+    }
+    if (slowExpired !== undefined) {
+      // Expiry is recorded INDEPENDENT of whether the fast lane injected —
+      // the 赶上率 denominator must not lose rounds to a coincidental hit.
+      record.slowExpired = slowExpired
+      if (record.lane === undefined) record.lane = injection.text !== '' ? 'fast' : 'slow'
+    }
+    if (injection.text === '' && (outcome.hits.length > 0 || slowPointers.length > 0)) {
       // All hits deduped is a different story than a budget squeeze — say so.
-      record.why = deduped.length === outcome.hits.length ? 'dedup' : 'below-budget-or-dropped'
+      record.why = deduped.length === outcome.hits.length && outcome.hits.length > 0 ? 'dedup' : 'below-budget-or-dropped'
+    } else if (injection.text === '' && slowExpired !== undefined) {
+      // The pending died at its hard bounds (TTL / turn-lag) without ever
+      // being delivered — the denominator the 赶上率 needs (v4 B4).
+      record.why = `slow-expired-${slowExpired}`
     }
-    if (injection.dropped.length > 0) record.dropped = injection.dropped
+    if (slowExpired !== undefined && injection.text === '') record.lane = 'slow'
+    if (injection.dropped.length > 0) record.dropped = [...(record.dropped ?? []), ...injection.dropped]
     void this.store.appendInjectionRecord(record).catch(() => undefined)
-    return { text: injection.text, outcome, deduped, included: injection.included }
+    return { text: injection.text, outcome, deduped, included: injection.included, slowIncluded }
+  }
+
+  /**
+   * topic_open tool path (v4 §4.1): full conclusion / open questions /
+   * recommendations plus the staleness notice timestamp (fm.generated.at —
+   * the store stamps it on every write, so it IS the last-updated instant).
+   * Logs an open record for the pointer-open-rate stat.
+   */
+  async openTopic(slug: string, sessionId?: string): Promise<{
+    found: boolean
+    slug: string
+    title?: string
+    status?: okf.TopicStatus
+    updatedAt?: string
+    description?: string
+    conclusion?: string
+    openQuestions?: string[]
+    recommendations?: string
+  }> {
+    const doc = await this.store.readTopic(slug).catch(() => undefined)
+    if (doc === undefined) return { found: false, slug }
+    // Awaited (tool path, not hot): a settled write makes the pointer-open
+    // stat reliable; a failed log write must not fail the tool.
+    await this.store
+      .appendOpenRecord({ slug, at: new Date().toISOString(), ...(sessionId !== undefined ? { sessionId } : {}) })
+      .catch(() => undefined)
+    return {
+      found: true,
+      slug,
+      title: doc.fm.title,
+      status: doc.fm.status,
+      updatedAt: doc.fm.generated.at,
+      description: doc.fm.description,
+      conclusion: okf.sectionOf(doc.body, okf.CONCLUSION_HEADING) ?? '',
+      openQuestions: doc.fm.open_questions,
+      recommendations: okf.sectionOf(doc.body, okf.RECOMMENDATIONS_HEADING) ?? '',
+    }
   }
 
   /** Sync roster read (same mtime cache as roster()). */
@@ -256,6 +425,7 @@ export class TopicsService {
         description: doc.fm.description,
         status: doc.fm.status,
         tags: doc.fm.tags,
+        triggers: doc.fm.triggers,
         depends: doc.fm.depends,
         links: okf.bodyLinkSlugs(doc.body),
         generatedAt: doc.fm.generated.at,
@@ -286,6 +456,7 @@ export class TopicsService {
     title: string
     description?: string
     tags?: string[]
+    triggers?: string[]
     depends?: string[]
     openQuestions?: string[]
     impact?: string[]
@@ -320,6 +491,10 @@ export class TopicsService {
       generated: existing?.fm.generated ?? { by: 'pending', at: now },
     }
     if (input.description !== undefined && input.description !== '') fm.description = input.description
+    // Triggers pass through on create; on update the caller's value wins, else
+    // the stored list survives (same preserve-on-update semantics as tags).
+    const triggers = sanitizeTriggers(input.triggers ?? existing?.fm.triggers ?? [])
+    if (triggers.length > 0) fm.triggers = triggers
     if (existing !== undefined && existing.fm.verified !== undefined) fm.verified = existing.fm.verified
     const result = await this.store.saveTopic(
       { slug, doc: { fm, body } },
@@ -342,6 +517,9 @@ export class TopicsService {
       tagBoost: cfg.tagBoost,
       graphDepth: cfg.graphDepth,
       recencyWindowDays: cfg.recencyWindowDays,
+      // Explicit tool: the model asked for recall, so the injection-only
+      // structural gate must not hide results.
+      structuralGate: false,
       conflicts: await this.store.getConflicts(),
     })
   }
@@ -398,3 +576,14 @@ function dedupeLower(tags: string[]): string[] {
   }
   return out
 }
+
+function sanitizeTriggers(triggers: readonly string[]): string[] {
+  const out: string[] = []
+  for (const t of triggers) {
+    if (typeof t !== 'string') continue
+    const key = t.trim()
+    if (key === '') continue
+    out.push(key)
+  }
+  return out
+}

package/src/store.ts

@@ -493,6 +493,59 @@ export class BundleStore {
     return out.slice(-limit)
   }
 
+  // ------------------------------------------------------------------
+  // Pointer-open log (v4 §4.3) — one line per topic_open call; feeds the
+  // 「指针打开率」 stat. Features only (slug/time), never conversation text.
+  // ------------------------------------------------------------------
+
+  private opensPath(): string {
+    return join(this.metaDir(), 'opens.jsonl')
+  }
+
+  async appendOpenRecord(record: { slug: string; at: string; sessionId?: string }): Promise<void> {
+    await mkdir(this.metaDir(), { recursive: true })
+    await appendFile(this.opensPath(), `${JSON.stringify(record)}\n`, 'utf8')
+    await this.compactOpensIfNeeded()
+  }
+
+  private async compactOpensIfNeeded(): Promise<void> {
+    const file = this.opensPath()
+    let size = 0
+    try {
+      size = (await readFile(file, 'utf8')).length
+    } catch {
+      return
+    }
+    if (size <= 512 * 1024) return
+    const lines = (await readFile(file, 'utf8')).split('\n').filter((l) => l.trim() !== '')
+    await atomicWrite(file, lines.slice(-Math.max(1, Math.floor(lines.length / 4))).map((l) => `${l}\n`).join(''))
+  }
+
+  /** Opens in the recent window; tolerant of a torn tail like the other JSONLs. */
+  async readOpenRecords(limit = 2000): Promise<{ slug: string; at: string; sessionId?: string }[]> {
+    let raw: string
+    try {
+      raw = await readFile(this.opensPath(), 'utf8')
+    } catch {
+      return []
+    }
+    const out: { slug: string; at: string; sessionId?: string }[] = []
+    for (const line of raw.split('\n')) {
+      if (line.trim() === '') continue
+      try {
+        const parsed = JSON.parse(line) as { slug?: unknown; at?: unknown; sessionId?: unknown }
+        if (typeof parsed.slug === 'string' && typeof parsed.at === 'string') {
+          const rec: { slug: string; at: string; sessionId?: string } = { slug: parsed.slug, at: parsed.at }
+          if (typeof parsed.sessionId === 'string') rec.sessionId = parsed.sessionId
+          out.push(rec)
+        }
+      } catch {
+        // Torn tail line tolerated.
+      }
+    }
+    return out.slice(-limit)
+  }
+
   // ------------------------------------------------------------------
   // Conflicted topics (ADR 0003) — retrieval demotes these
   // ------------------------------------------------------------------

package/src/tools.ts

@@ -28,6 +28,7 @@ export function buildTopicTools(service: TopicsService) {
       conclusion: { type: 'string', required: true, description: 'The current best conclusion, self-contained markdown prose' },
       description: { type: 'string', description: 'One-line summary for indexes and snippets' },
       tags: { ...STRING_ARRAY, description: 'Cross-cutting tags (project, domain, component); lowercased on save' },
+      triggers: { ...STRING_ARRAY, description: 'Short recall phrases that should bring this topic back (specific nouns/terms, no generic words)' },
       depends: { ...STRING_ARRAY, description: 'Slugs of prerequisite topics (topics this one builds on)' },
       open_questions: { ...STRING_ARRAY, description: 'Unresolved questions this topic still carries' },
       impact: { ...STRING_ARRAY, description: 'What this conclusion affects: topics, projects, decisions' },
@@ -59,6 +60,7 @@ export function buildTopicTools(service: TopicsService) {
         conclusion: args.conclusion,
         description: args.description,
         tags: args.tags,
+        triggers: args.triggers,
         depends: args.depends,
         openQuestions: args.open_questions,
         impact: args.impact,
@@ -71,6 +73,54 @@ export function buildTopicTools(service: TopicsService) {
     },
   })
 
+  const topicOpen = defineTool({
+    name: 'topic_open',
+    description:
+      'Open one topic from the long-term memory in full: conclusion, open questions, recommendations, ' +
+      'plus a staleness notice. Use when a <topic-memory> pointer (or a search hit) is actually relevant — ' +
+      'pointers are one-line hints; this fetches the real content.',
+    parameters: {
+      slug: { type: 'string', required: true, description: 'Topic slug (from a pointer or topic_search)' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          found: { type: 'boolean', required: true },
+          slug: { type: 'string', required: true },
+          title: { type: 'string' },
+          status: { type: 'string' },
+          updatedAt: { type: 'string' },
+          description: { type: 'string' },
+          conclusion: { type: 'string' },
+          openQuestions: { type: 'array', items: { type: 'string' } },
+          recommendations: { type: 'string' },
+        },
+      },
+      render: (_args, value) => [
+        {
+          type: 'text' as const,
+          text: !value.found
+            ? `Topic \`${value.slug}\` 不存在（可能已合并或改名；用 topic_search 找）。`
+            : [
+                `快照于 ${value.updatedAt}，代码事实以源码为准。`,
+                `### ${value.title} [${value.status}] (topics:${value.slug})`,
+                value.description ?? '',
+                value.conclusion !== '' ? `# Conclusion\n\n${value.conclusion}` : '',
+                value.openQuestions !== undefined && value.openQuestions.length > 0 ? `待决: ${value.openQuestions.join('；')}` : '',
+                value.recommendations !== '' ? `# Recommendations\n\n${value.recommendations}` : '',
+              ]
+                .filter((l) => l !== '')
+                .join('\n\n'),
+        },
+      ],
+    },
+    async execute(args) {
+      return await service.openTopic(args.slug)
+    },
+  })
+
   const topicSearch = defineTool({
     name: 'topic_search',
     description:
@@ -215,7 +265,7 @@ export function buildTopicTools(service: TopicsService) {
     },
   })
 
-  return [topicSave, topicSearch, topicObserve, topicHistory]
+  return [topicSave, topicOpen, topicSearch, topicObserve, topicHistory]
 }
 
 function extractConclusion(body: string): string {