@aiwayds/dsh-topics-memory 0.21.0 → 0.23.0

package/src/embedding.ts

@@ -0,0 +1,294 @@
+/**
+ * Local embedding client (ollama) — the semantic layer behind the nearFloor
+ * reranker (A4 决议 2, spec §3.1).
+ *
+ * Semantics (mirror of jev/client's outcome mode):
+ *  - every failure returns as a structured {ok:false, outcome} result, NEVER
+ *    a throw — the caller degrades explicitly to the pure-lexical path;
+ *  - one AbortSignal.timeout per request, NO retry (query 2s / batch 30s);
+ *  - localhost only in v1 (ADR 0017 local exception). Remote providers do
+ *    not exist here by design: a remote embedding may only ever enter a
+ *    slow-lane/offline path, never the fast lane (red line ①).
+ *
+ * @module embedding
+ */
+
+import { execFileSync } from 'node:child_process'
+
+/** Default ollama bind (the plugin's own default; OLLAMA_HOST overrides). */
+export const DEFAULT_EMBED_HOST = 'http://127.0.0.1:11434'
+
+/** Provider default model when embeddingModel is the '' sentinel
+ *  (spec §7: ollama → nomic-embed-text, 768 dims, A1 §5 measured). */
+export const DEFAULT_EMBED_MODEL = 'nomic-embed-text'
+
+/** Query-side timeout — the reranker sits in the spliced hot path; a slow
+ *  ollama must lose the race, not the injection. */
+export const EMBED_QUERY_TIMEOUT_MS = 2000
+/** Batch-side timeout (index rebuild: A1 measured ~9.5s for 137 topics). */
+export const EMBED_BATCH_TIMEOUT_MS = 30_000
+
+/** Per-text byte cap before anything leaves the process (query端 p50=153 tok
+ *  ~ well under this; the cap only bites on paste pollution). */
+export const MAX_TEXT_BYTES = 2048
+
+/** The only provider v1 speaks. Anything else is an explicit
+ *  unsupported_provider outcome, not a silent attempt. */
+export const SUPPORTED_PROVIDER = 'ollama'
+
+/** The config slice the embed functions need (pass straight from
+ *  TopicsConfigValue). '' sentinels: provider off / provider default model. */
+export interface EmbeddingConfig {
+  provider?: string
+  model?: string
+  /** Explicit host; absent → OLLAMA_HOST env → DEFAULT_EMBED_HOST. */
+  host?: string
+}
+
+/** Resolve the ollama base URL: explicit arg wins, then OLLAMA_HOST, then
+ *  the localhost default. Bare `host:port` values get the http:// prefix
+ *  (ollama's own OLLAMA_HOST convention); trailing slashes are trimmed. */
+export function resolveEmbedHost(explicit?: string): string {
+  const raw = (explicit ?? process.env.OLLAMA_HOST ?? '').trim()
+  const withScheme = raw === '' ? DEFAULT_EMBED_HOST : /^https?:\/\//.test(raw) ? raw : `http://${raw.replace(/^tcp:\/\//, '')}`
+  return withScheme.replace(/\/+$/, '')
+}
+
+/** '' sentinel → provider default model. */
+export function resolveEmbedModel(model?: string): string {
+  const m = (model ?? '').trim()
+  return m === '' ? DEFAULT_EMBED_MODEL : m
+}
+
+/** Truncate a string to ≤ maxBytes UTF-8 bytes WITHOUT splitting a multibyte
+ *  character: cut at the byte cap, then back off over any continuation bytes
+ *  (0b10xxxxxx) so the decoded tail is never a replacement char. */
+export function truncateUtf8(text: string, maxBytes: number = MAX_TEXT_BYTES): string {
+  if (maxBytes <= 0) return ''
+  const buf = Buffer.from(text, 'utf8')
+  if (buf.length <= maxBytes) return text
+  let end = maxBytes
+  while (end > 0 && (buf[end]! & 0xc0) === 0x80) end -= 1
+  return buf.subarray(0, end).toString('utf8')
+}
+
+// ---------------------------------------------------------------------------
+// Outcomes — the explicit-degradation vocabulary. Callers branch on `ok`;
+// `outcome` is the human/queryable why (降级状态可查, spec §3.1).
+// ---------------------------------------------------------------------------
+
+export type EmbeddingFailureOutcome =
+  | 'provider_off' // config provider is the '' sentinel — semantic layer off
+  | 'unsupported_provider' // configured, but not a v1 provider (no remote!)
+  | 'empty_input' // nothing to embed — refused before any network activity
+  | 'network' // connect refused / DNS / socket — ollama not serving
+  | 'timeout' // the one AbortSignal fired
+  | 'model_missing' // ollama answered 404: model not pulled
+  | 'http_4xx'
+  | 'http_5xx'
+  | 'bad_json'
+  | 'bad_shape' // 200 but the embeddings array is malformed / dims disagree
+
+export interface EmbedBatchOk {
+  ok: true
+  /** Vectors aligned 1:1 with the (truncated) input texts. */
+  vectors: number[][]
+  model: string
+  dim: number
+  latencyMs: number
+}
+
+export interface EmbedQueryOk {
+  ok: true
+  vector: number[]
+  model: string
+  dim: number
+  latencyMs: number
+}
+
+export interface EmbedFailure {
+  ok: false
+  outcome: EmbeddingFailureOutcome
+  message: string
+  host: string
+  model: string
+  latencyMs: number
+}
+
+export type EmbedResult = EmbedBatchOk | EmbedFailure
+export type EmbedQueryResult = EmbedQueryOk | EmbedFailure
+export type ProbeResult = { ok: true; host: string; model: string; dim: number; latencyMs: number } | EmbedFailure
+
+function isPlainObject(v: unknown): v is Record<string, unknown> {
+  return typeof v === 'object' && v !== null && !Array.isArray(v)
+}
+
+/** Shared request path: POST {host}/api/embed, outcome-mode failures only. */
+async function embedRequest(
+  texts: readonly string[],
+  config: EmbeddingConfig,
+  timeoutMs: number,
+): Promise<EmbedResult> {
+  const provider = (config.provider ?? '').trim()
+  if (provider === '') {
+    return { ok: false, outcome: 'provider_off', message: "embeddingProvider is '' — the semantic layer is off (pure lexical).", host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
+  }
+  if (provider !== SUPPORTED_PROVIDER) {
+    return { ok: false, outcome: 'unsupported_provider', message: `embeddingProvider '${provider}' is not supported in v1 (only '${SUPPORTED_PROVIDER}'); remote providers may not enter the fast lane.`, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
+  }
+  if (texts.length === 0) {
+    return { ok: false, outcome: 'empty_input', message: 'no texts to embed.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
+  }
+  const host = resolveEmbedHost(config.host)
+  const model = resolveEmbedModel(config.model)
+  const started = Date.now()
+  let res: Response
+  try {
+    res = await fetch(`${host}/api/embed`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ model, input: texts.map((t) => truncateUtf8(t)) }),
+      signal: AbortSignal.timeout(timeoutMs),
+    })
+  } catch (err) {
+    const outcome: EmbeddingFailureOutcome =
+      err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError') ? 'timeout' : 'network'
+    return { ok: false, outcome, message: `${outcome}: ${err instanceof Error ? err.message : String(err)}`, host, model, latencyMs: Date.now() - started }
+  }
+  if (!res.ok) {
+    const text = (await res.text().catch(() => '')).slice(0, 300)
+    // ollama answers 404 {"error":"model \"…\" not found, try pulling it first"}
+    const outcome: EmbeddingFailureOutcome = res.status === 404 ? 'model_missing' : res.status >= 500 ? 'http_5xx' : 'http_4xx'
+    return { ok: false, outcome, message: `ollama responded ${res.status}: ${text}`, host, model, latencyMs: Date.now() - started }
+  }
+  let json: unknown
+  try {
+    json = await res.json()
+  } catch (err) {
+    return { ok: false, outcome: 'bad_json', message: `non-JSON body: ${err instanceof Error ? err.message : String(err)}`, host, model, latencyMs: Date.now() - started }
+  }
+  const vectors = isPlainObject(json) ? json.embeddings : undefined
+  if (
+    !Array.isArray(vectors) ||
+    vectors.length !== texts.length ||
+    !vectors.every((v) => Array.isArray(v) && v.length > 0 && v.every((n) => typeof n === 'number' && Number.isFinite(n))) ||
+    new Set(vectors.map((v) => (v as number[]).length)).size !== 1
+  ) {
+    return { ok: false, outcome: 'bad_shape', message: 'embeddings must be a rectangular [texts.length][dim] array of finite numbers.', host, model, latencyMs: Date.now() - started }
+  }
+  return { ok: true, vectors: vectors as number[][], model, dim: (vectors[0] as number[]).length, latencyMs: Date.now() - started }
+}
+
+/** Batch-embed texts (index build/rebuild path). Vectors align 1:1 with the
+ *  input order; each text is truncated to MAX_TEXT_BYTES first. */
+export async function embedTexts(
+  texts: readonly string[],
+  config: EmbeddingConfig = {},
+  opts: { timeoutMs?: number } = {},
+): Promise<EmbedResult> {
+  const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_BATCH_TIMEOUT_MS
+  return embedRequest(texts, config, timeoutMs)
+}
+
+/** Embed one query (reranker hot path, 2s budget). Empty/whitespace queries
+ *  fail locally with 'empty_input' — nothing leaves the process. */
+export async function embedQuery(
+  query: string,
+  config: EmbeddingConfig = {},
+  opts: { timeoutMs?: number } = {},
+): Promise<EmbedQueryResult> {
+  if ((query ?? '').trim() === '') {
+    return { ok: false, outcome: 'empty_input', message: 'empty query.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
+  }
+  const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_QUERY_TIMEOUT_MS
+  const r = await embedRequest([query], config, timeoutMs)
+  if (!r.ok) return r
+  return { ok: true, vector: r.vectors[0]!, model: r.model, dim: r.dim, latencyMs: r.latencyMs }
+}
+
+/**
+ * Probe: is the server there, is the model pulled? One micro embed call —
+ * unlike a /api/tags listing it validates the FULL path (server + model +
+ * response shape) and yields the vector dim for index validation, at the
+ * same 15-30ms cost (A1 §5). Structured result; never throws.
+ */
+export async function probeEmbedding(config: EmbeddingConfig = {}, opts: { timeoutMs?: number } = {}): Promise<ProbeResult> {
+  const timeoutMs = Number(opts.timeoutMs) > 0 ? Number(opts.timeoutMs) : EMBED_QUERY_TIMEOUT_MS
+  const r = await embedRequest(['dsh-topics-memory semantic layer probe'], config, timeoutMs)
+  if (!r.ok) return r
+  return { ok: true, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), dim: r.dim, latencyMs: r.latencyMs }
+}
+
+/**
+ * Build an embedder handle for retrieval's `ensureIndexFresh` — a callable
+ * embedTexts bound to this config, carrying the RESOLVED model identity the
+ * index must match. ensureIndexFresh reads `.model` only; the actual
+ * rebuild stays with the caller.
+ */
+export function embedderWith(
+  config: EmbeddingConfig,
+): ((texts: readonly string[]) => Promise<EmbedResult>) & { model: string } {
+  const fn = (texts: readonly string[]) => embedTexts(texts, config)
+  return Object.assign(fn, { model: resolveEmbedModel(config.model) })
+}
+
+// --- Sync query embed (Phase-2 wiring) --------------------------------------
+// The spliced hot path is synchronous (the chancelu lesson): retrieveSync
+// cannot await. A local ollama query is 15-30ms steady state (A1 §5), so a
+// bounded SYNC subprocess call keeps the reranker on the hot lane inside the
+// ADR 0017 local exception — hard-capped so a wedged ollama loses the race,
+// never the injection. curl ships with the host OS; node has no sync HTTP.
+
+/** Hard sync budget for the hot-lane query embed. Generous over the measured
+ *  15-30ms, merciless for a wedged server. */
+export const EMBED_QUERY_SYNC_TIMEOUT_MS = 250
+
+export function embedQuerySync(query: string, config: EmbeddingConfig = {}): EmbedQueryResult {
+  const provider = (config.provider ?? '').trim()
+  if (provider === '') {
+    return { ok: false, outcome: 'provider_off', message: "embeddingProvider is '' — the semantic layer is off.", host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
+  }
+  if (provider !== SUPPORTED_PROVIDER) {
+    return { ok: false, outcome: 'unsupported_provider', message: `embeddingProvider '${provider}' is not supported in v1.`, host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
+  }
+  if ((query ?? '').trim() === '') {
+    return { ok: false, outcome: 'empty_input', message: 'empty query.', host: resolveEmbedHost(config.host), model: resolveEmbedModel(config.model), latencyMs: 0 }
+  }
+  const host = resolveEmbedHost(config.host)
+  const model = resolveEmbedModel(config.model)
+  const started = Date.now()
+  let raw: string
+  try {
+    raw = execFileSync(
+      'curl',
+      [
+        '-sS',
+        '--max-time', String(Math.ceil(EMBED_QUERY_SYNC_TIMEOUT_MS / 1000)),
+        '-X', 'POST', `${host}/api/embed`,
+        '-H', 'content-type: application/json',
+        '-d', JSON.stringify({ model, input: [truncateUtf8(query)] }),
+      ],
+      { timeout: EMBED_QUERY_SYNC_TIMEOUT_MS, encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] },
+    )
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err)
+    const outcome: EmbeddingFailureOutcome = /timed out|ETIMEDOUT|maxTime/i.test(message) ? 'timeout' : 'network'
+    return { ok: false, outcome, message: `${outcome}: ${message}`, host, model, latencyMs: Date.now() - started }
+  }
+  let json: unknown
+  try {
+    json = JSON.parse(raw)
+  } catch {
+    return { ok: false, outcome: 'bad_json', message: 'ollama returned non-JSON (sync path).', host, model, latencyMs: Date.now() - started }
+  }
+  const embeddings = isPlainObject(json) ? (json as Record<string, unknown>).embeddings : undefined
+  const first = Array.isArray(embeddings) && embeddings.length > 0 ? embeddings[0] : undefined
+  if (!Array.isArray(first)) {
+    return { ok: false, outcome: 'bad_shape', message: 'ollama response has no embeddings[0] array (sync path).', host, model, latencyMs: Date.now() - started }
+  }
+  const vector = (first as unknown[]).filter((n): n is number => typeof n === 'number')
+  if (vector.length === 0 || vector.length !== (first as unknown[]).length) {
+    return { ok: false, outcome: 'bad_shape', message: 'embeddings[0] malformed (sync path).', host, model, latencyMs: Date.now() - started }
+  }
+  return { ok: true, vector, model, dim: vector.length, latencyMs: Date.now() - started }
+}

package/src/ilog.ts

@@ -27,6 +27,14 @@ export interface SlowItem {
   why: string
 }
 
+/**
+ * Slow-lane full-lifecycle outcome (A4 决议 3, spec §3.2): where a slow-lane
+ * run (or round with a slow-lane stake) ended. Precedent = decisions.jsonl's
+ * outcome enum — a closed vocabulary so 「没跑」(no-run/no-claim) and 「跑了
+ * 全败」(llm-error/timeout) are finally distinguishable in the log.
+ */
+export type SlowOutcome = 'no-run' | 'no-claim' | 'no-candidates' | 'llm-error' | 'timeout' | 'delivered' | 'expired'
+
 export interface InjectionRecord {
   at: string
   sessionId?: string
@@ -60,6 +68,33 @@ export interface InjectionRecord {
   slowMs?: number
   /** Slow-lane pointers this round, in delivery order. */
   slow?: SlowItem[]
+  /**
+   * Slow-lane disposition for this round (A4 决议 3, spec §8.2): `delivered`
+   * when a pending was consumed and packed, `expired` at its hard bounds, and
+   * the producing run's failure point (no-run/no-claim/no-candidates/
+   * llm-error/timeout) when a spliced arrived with no pending. Optional —
+   * older lines predate the field and pure fast-lane rounds may omit it.
+   * Written once with the record, never backfilled (ilog append-only).
+   */
+  slowOutcome?: SlowOutcome
+}
+
+/**
+ * One topic_open event (pointer-open log, v4 §4.3). Everything beyond
+ * `slug`/`at` is optional and late-added: older lines carry only the
+ * original pair, and every reader must tolerate the rest being absent.
+ */
+export interface OpenRecord {
+  slug: string
+  at: string
+  /** Session that issued the topic_open, when the host surface exposed one. */
+  sessionId?: string
+  /** Attribution: opened off an injected pointer, or off a topic_search hit. */
+  source?: 'pointer' | 'search'
+  /** Retrieval score of the matched injection hit (source='pointer' only). */
+  score?: number
+  /** open time − the matched injection record's at, in ms (source='pointer' only). */
+  sinceInjectionMs?: number
 }
 
 export interface AggregateStats {
@@ -191,3 +226,274 @@ export function aggregateUsage(
   }
   return out
 }
+
+// ---------------------------------------------------------------------
+// Injection-vs-open metrics (v5) — open enrichment attribution + the
+// /topics stats panel aggregate. All pure: file I/O stays in store/service.
+// ---------------------------------------------------------------------
+
+/** How many tail injection records the cheap open attribution scans. */
+export const OPEN_ATTRIBUTION_TAIL = 50
+/** An injected pointer stays attributable for 48h (a work session's span). */
+export const OPEN_ATTRIBUTION_WINDOW_MS = 48 * 3_600_000
+
+/**
+ * Cheap source attribution for one open event: scan the injection tail
+ * (newest first) for a same-session record within the attribution window
+ * where the slug was truly delivered to the model's eyes. That is hits
+ * minus echoed/dropped (echo suppression and budget drops never rendered),
+ * PLUS slow-lane deliveries, PLUS deduped hits — dedup only blocks
+ * re-packing THIS round; the pointer rendered earlier in the same session
+ * and the model saw it (deliberately wider than renderedPointers: the panel
+ * denominator asks "did it render this round", attribution asks "did the
+ * model ever see it"). A fast-hit match carries that hit's score; a
+ * slow-only match carries none (slow pointers have no lexical score).
+ * Anything else (no session id, other session, expired, never rendered)
+ * is `search`.
+ */
+export function attributeOpen(
+  injections: readonly InjectionRecord[],
+  slug: string,
+  sessionId: string | undefined,
+  nowMs: number,
+): { source: 'pointer' | 'search'; score?: number; sinceInjectionMs?: number } {
+  if (sessionId === undefined) return { source: 'search' }
+  for (let i = injections.length - 1; i >= 0; i -= 1) {
+    const record = injections[i]
+    if (record.sessionId !== sessionId) continue
+    const atMs = Date.parse(record.at)
+    if (!Number.isFinite(atMs) || nowMs - atMs > OPEN_ATTRIBUTION_WINDOW_MS) continue
+    const sinceInjectionMs = Math.max(0, nowMs - atMs)
+    const suppressed = new Set<string>([...(record.echoed ?? []), ...(record.dropped ?? []).map((d) => d.slug)])
+    const hit = record.hits.find((h) => h.slug === slug && !suppressed.has(h.slug))
+    if (hit !== undefined) return { source: 'pointer', score: hit.score, sinceInjectionMs }
+    if ((record.slow ?? []).some((s) => s.slug === slug)) return { source: 'pointer', sinceInjectionMs }
+  }
+  return { source: 'search' }
+}
+
+/** Score bands for the open-rate-by-score cut of the stats panel. */
+export const SCORE_BANDS: readonly { label: string; lo: number; hi: number }[] = [
+  { label: '[0.3,0.5)', lo: 0.3, hi: 0.5 },
+  { label: '[0.5,1)', lo: 0.5, hi: 1 },
+  { label: '[1,2)', lo: 1, hi: 2 },
+  { label: '[2,5)', lo: 2, hi: 5 },
+  { label: '[5,∞)', lo: 5, hi: Number.POSITIVE_INFINITY },
+]
+
+/** Zero-open listing threshold: injections at or above this with zero opens. */
+export const ZERO_OPEN_MIN_INJECTIONS = 5
+/** Rolling window of the zero-open rule: only rendered injections inside it count. */
+export const ZERO_OPEN_WINDOW_DAYS = 30
+/** Injection-tail rows the decay read scans — a bound comfortably covering
+ *  the 30-day window at observed round volumes. */
+export const ZERO_OPEN_TAIL = 500
+
+/**
+ * Slugs that truly rendered in one round, in delivery order: raw hits minus
+ * the round's exclusions (deduped/echoed/dropped were never assembled), plus
+ * delivered slow pointers; a slug the fast lane already delivered counts
+ * once even when the slow lane also reports it delivered ("not packed
+ * twice"). Scores ride along from the fast hit only — slow pointers carry
+ * no lexical score. Shared by the stats panel and the zero-open rule.
+ */
+export function renderedPointers(record: InjectionRecord): { slug: string; score?: number }[] {
+  const excluded = new Set<string>([
+    ...(record.deduped ?? []),
+    ...(record.echoed ?? []),
+    ...(record.dropped ?? []).map((d) => d.slug),
+  ])
+  const seen = new Set<string>()
+  const out: { slug: string; score?: number }[] = []
+  for (const h of record.hits) {
+    if (excluded.has(h.slug) || seen.has(h.slug)) continue
+    seen.add(h.slug)
+    out.push({ slug: h.slug, score: h.score })
+  }
+  for (const s of record.slow ?? []) {
+    if (seen.has(s.slug)) continue
+    seen.add(s.slug)
+    out.push({ slug: s.slug })
+  }
+  return out
+}
+
+/**
+ * Zero-open slug set — the single shared 零打开高频 rule (v5): slugs whose
+ * truly-rendered pointer count within the rolling ZERO_OPEN_WINDOW_DAYS
+ * reaches ZERO_OPEN_MIN_INJECTIONS while the WHOLE open log has never
+ * recorded an open (wide scope — an open long after the last injection
+ * still disqualifies). Returns slug → windowed rendered-pointer count for
+ * the qualifying slugs; `/topics stats` (the panel's zero-open list) and
+ * the retrieval decay (`zeroOpenDecay`) both consume this one
+ * implementation. Records with unparsable `at` cannot prove window
+ * membership and are skipped (same tolerance as aggregateUsage).
+ */
+export function aggregateZeroOpen(
+  injections: readonly InjectionRecord[],
+  opens: readonly { slug: string }[],
+  nowMs: number,
+): Map<string, number> {
+  const cutoff = nowMs - ZERO_OPEN_WINDOW_DAYS * 86_400_000
+  const counts = new Map<string, number>()
+  for (const record of injections) {
+    const at = Date.parse(record.at)
+    if (!Number.isFinite(at) || at < cutoff) continue
+    for (const p of renderedPointers(record)) {
+      counts.set(p.slug, (counts.get(p.slug) ?? 0) + 1)
+    }
+  }
+  const opened = new Set(opens.map((o) => o.slug))
+  const out = new Map<string, number>()
+  for (const [slug, n] of counts) {
+    if (n >= ZERO_OPEN_MIN_INJECTIONS && !opened.has(slug)) out.set(slug, n)
+  }
+  return out
+}
+
+export interface OpenTopicRow {
+  slug: string
+  /** Truly rendered pointers for this slug (deduped/echoed/dropped excluded). */
+  injections: number
+  /** Opens attributed by slug over the whole open log (wide scope, by design). */
+  opens: number
+  openRate: number
+}
+
+export interface OpenBandRow {
+  label: string
+  /** Rendered fast pointers whose hit score falls in this band. */
+  pointers: number
+  /** Pointer-sourced opens whose attributed score falls in this band. */
+  opens: number
+  openRate: number
+}
+
+export interface OpenPanelStats {
+  /** Rendered pointer entries across the window: hits − deduped − echoed − dropped, plus slow pointers (same-slug fast+slow pairs counted once). */
+  pointerEntries: number
+  /** DISTINCT slugs from the window's rendered set with an in-window open
+   *  (search-sourced and never-rendered slugs excluded; one per slug). */
+  opensInWindow: number
+  openRate: number
+  topics: OpenTopicRow[]
+  zeroOpen: { slug: string; injections: number }[]
+  zeroOpenTotal: number
+  pointerOpens: number
+  searchOpens: number
+  /** False when no open record carries a source field (pre-enrichment data). */
+  hasSourceData: boolean
+  bands: OpenBandRow[]
+}
+
+/**
+ * Aggregate the injection-vs-open panel from the two JSONL windows. The
+ * per-topic/band denominators count only pointers that truly rendered
+ * (deduped/echoed/dropped excluded — same exclusion family as
+ * aggregateUsage); the header numerator counts only DISTINCT slugs from
+ * that rendered set with an open inside the injection window — a search
+ * open or an open of a never-rendered slug cannot enter, and repeat opens
+ * of the same slug count once, so the rate can no longer be pinned at
+ * 100% by repeated or off-scope opens. Opens per slug stay wide-scope
+ * (the model may open a topic long after its last injection). The
+ * zero-open list shares the decay's rule: rendered injections counted
+ * within the rolling 30-day window (anchored at the newest round), opens
+ * wide-scope.
+ */
+export function aggregateOpenPanel(
+  injections: readonly InjectionRecord[],
+  opens: readonly OpenRecord[],
+): OpenPanelStats {
+  // Numerator window: the earliest round represented in the denominator;
+  // the zero-open anchor is the newest round — the panel describes the log
+  // window it was fed, so a stale log still shows the topics that
+  // dominated it. The rendered-slug set below shares that window: only
+  // opens of slugs this window rendered can enter the numerator.
+  let windowStart = Number.POSITIVE_INFINITY
+  let windowEnd = Number.NEGATIVE_INFINITY
+  for (const r of injections) {
+    const atMs = Date.parse(r.at)
+    if (!Number.isFinite(atMs)) continue
+    if (atMs < windowStart) windowStart = atMs
+    if (atMs > windowEnd) windowEnd = atMs
+  }
+  const topicCounts = new Map<string, number>()
+  const bandPointers = new Array<number>(SCORE_BANDS.length).fill(0)
+  let pointerEntries = 0
+  const renderedSlugs = new Set<string>()
+  for (const r of injections) {
+    // renderedPointers keeps the entry/per-topic counts honest: excluded
+    // slugs never counted, a fast+slow pair for the same slug counted once.
+    for (const p of renderedPointers(r)) {
+      pointerEntries += 1
+      renderedSlugs.add(p.slug)
+      topicCounts.set(p.slug, (topicCounts.get(p.slug) ?? 0) + 1)
+      const score = p.score
+      if (score === undefined) continue
+      const bi = SCORE_BANDS.findIndex((b) => score >= b.lo && score < b.hi)
+      if (bi >= 0) bandPointers[bi] += 1
+    }
+  }
+  const opensBySlug = new Map<string, number>()
+  const bandOpens = new Array<number>(SCORE_BANDS.length).fill(0)
+  let pointerOpens = 0
+  let searchOpens = 0
+  let hasSourceData = false
+  // Numerator: DISTINCT slugs from the window's rendered set with an
+  // in-window open — search-sourced opens never count (the model found the
+  // topic by asking, not by following a rendered pointer), a never-rendered
+  // slug never counts, and repeat opens of the same slug count once.
+  const openedSlugsInWindow = new Set<string>()
+  for (const o of opens) {
+    opensBySlug.set(o.slug, (opensBySlug.get(o.slug) ?? 0) + 1)
+    if (o.source === 'pointer' || o.source === 'search') hasSourceData = true
+    if (o.source === 'pointer') pointerOpens += 1
+    else if (o.source === 'search') searchOpens += 1
+    if (o.source === 'pointer' && typeof o.score === 'number') {
+      const score = o.score
+      const bi = SCORE_BANDS.findIndex((b) => score >= b.lo && score < b.hi)
+      if (bi >= 0) bandOpens[bi] += 1
+    }
+    if (o.source === 'search') continue
+    const atMs = Date.parse(o.at)
+    // An open joins the windowed numerator unless it is provably older than
+    // the window's earliest round (unparsable stamps cannot prove either
+    // way) — and only when the window actually rendered its slug.
+    if (Number.isFinite(windowStart) && Number.isFinite(atMs) && atMs < windowStart) continue
+    if (!renderedSlugs.has(o.slug)) continue
+    openedSlugsInWindow.add(o.slug)
+  }
+  const opensInWindow = openedSlugsInWindow.size
+  const rate = (num: number, den: number): number => (den === 0 ? 0 : Math.min(1, num / den))
+  const topics: OpenTopicRow[] = [...topicCounts.entries()]
+    .map(([slug, injections]) => {
+      const opens = opensBySlug.get(slug) ?? 0
+      return { slug, injections, opens, openRate: rate(opens, injections) }
+    })
+    .sort((a, b) => b.injections - a.injections || a.slug.localeCompare(b.slug))
+  // Zero-open cut: the shared 30-day rule (aggregateZeroOpen), anchored at
+  // the newest round so window semantics match the decay's rolling window
+  // without growing a time-bomb dependency on the wall clock.
+  const zeroOpenCounts =
+    windowEnd === Number.NEGATIVE_INFINITY ? new Map<string, number>() : aggregateZeroOpen(injections, opens, windowEnd)
+  const zeroOpenAll = [...zeroOpenCounts.entries()]
+    .map(([slug, injections]) => ({ slug, injections }))
+    .sort((a, b) => b.injections - a.injections || a.slug.localeCompare(b.slug))
+  return {
+    pointerEntries,
+    opensInWindow,
+    openRate: rate(opensInWindow, pointerEntries),
+    topics: topics.slice(0, 10),
+    zeroOpen: zeroOpenAll.slice(0, 10).map((t) => ({ slug: t.slug, injections: t.injections })),
+    zeroOpenTotal: zeroOpenAll.length,
+    pointerOpens,
+    searchOpens,
+    hasSourceData,
+    bands: SCORE_BANDS.map((b, i) => ({
+      label: b.label,
+      pointers: bandPointers[i],
+      opens: bandOpens[i],
+      openRate: rate(bandOpens[i], bandPointers[i]),
+    })),
+  }
+}

package/src/index.ts

@@ -501,8 +501,9 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
       if (cfg.autoInject) {
         const consumed = slowLane.consume(sessionId, observer.turnCountOf(sessionId))
         if (consumed !== undefined) {
-          if ('pending' in consumed) slow = consumed.pending
-          else slowExpired = consumed.expired
+          if ('pending' in consumed) {
+            slow = { ...consumed.pending, outcome: consumed.pending.outcome }
+          } else slowExpired = consumed.expired
         }
       }
       const r = service.retrieveSync(query, sessionId, seen === undefined && echo === undefined ? undefined : { exclude: seen, echo }, slow, slowExpired)
@@ -708,7 +709,10 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
     // and its own settle hook does the release.
     const run = distiller.request(sessionId, reason)
     if (run !== undefined) {
-      void run.finally(() => releaseSessionLlm(sessionId)).catch(() => undefined)
+      void run
+        .finally(() => releaseSessionLlm(sessionId))
+        .then(() => service.refreshEmbeddingIndex())
+        .catch(() => undefined)
     }
   })
 
@@ -916,7 +920,11 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
                 // no-adapter detail instead
               }
               const run = distiller.request(sessionId, 'boot-replay')
-              if (run !== undefined) void run.catch(() => undefined)
+              if (run !== undefined) {
+                void run
+                  .then(() => service.refreshEmbeddingIndex())
+                  .catch(() => undefined)
+              }
             })
             .catch(() => undefined)
           // Consolidation cadence check — after the pull (freshest pool),
@@ -951,6 +959,9 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
               })
               .catch(() => undefined)
           }
+          // Embedding index maintenance (A4/A6, spec §3): boot-time freshness
+          // check + incremental rebuild — provider '' / packageMode off = no-op.
+          void service.refreshEmbeddingIndex().catch(() => undefined)
         })
         .catch((error: unknown) => {
           warn(`dsh-topics-memory 启动链失败（不影响会话启动）：${error instanceof Error ? error.message : String(error)}`)
@@ -1075,7 +1086,11 @@ export async function apply(ctx: Context, config: TopicsRuntimeConfig = {}): Pro
           ? { ok: false, reason: 'in-flight', created: [], updated: [], marked: 0 }
           : { ok: false, reason: 'no-model', created: [], updated: [], marked: 0 }
       }
-      return run.finally(() => releaseSessionLlm(sessionId))
+      void run
+        .finally(() => releaseSessionLlm(sessionId))
+        .then(() => service.refreshEmbeddingIndex())
+        .catch(() => undefined)
+      return run
     }
     // Manual /topics consolidate trigger: same lane as the cadence path, same
     // trigger-time llm capture and in-flight guard (the command runs inside a
@@ -1146,6 +1161,9 @@ const DEFAULTS: TopicsConfigValue = {
   distillMaxModelCalls: 8,
   consolidateCadence: 'daily',
   deprecatedTtlDays: 15,
+  embeddingProvider: '',
+  embeddingModel: '',
+  packageMode: 'off',
   usageBoost: 0.15,
   pushDebounceSeconds: 45,
 }