@aiwayds/dsh-topics-memory 0.22.0 → 0.23.0

package/lib/quality.js

@@ -15,8 +15,11 @@
  * Picks rest in a per-session pending slot and inject at the NEXT spliced
  * (steer message), then the slot is gone (消费即清). Hard bounds: per-call
  * 20s, pipeline 45s, pending TTL 10min, turn-lag ≤2. Every failure is
- * contained and silent — I3 (可丢弃性): async products may be missing, never
- * blocking. The lexical re-gate at consumption is SHADOW-ONLY (v4 B3): it
+ * contained — async products may be missing, never blocking (I3 可丢弃性) —
+ * but no longer SILENT (A4 决议 3): each run lands a `slow-lane-outcome`
+ * disposition (SlowOutcome) on the session and in the pipeline-events
+ * ledger, so 「没跑」and 「跑了全败」are distinguishable in the log. The
+ * lexical re-gate at consumption is SHADOW-ONLY (v4 B3): it
  * records verdicts into ilog and never blocks a pick.
  *
  * @module quality
@@ -25,6 +28,7 @@ import { searchTopics } from "./retrieval.js";
 import { jevAskDual } from "./jev/dual.js";
 import { band as jevBand } from "./jev/thresholds.js";
 import { digestFor, logVerdicts } from "./jev/log.js";
+import { recordPipelineEvent } from "./pipeline-events.js";
 export const PENDING_TTL_MS = 10 * 60_000;
 /** Hard drift bound: a pending older than this many turn-ends expires. */
 export const TURN_LAG_LIMIT = 2;
@@ -184,11 +188,33 @@ export function parseJsonObject(raw) {
     }
     throw new Error('unbalanced JSON in output');
 }
+/** The closed outcome for an error that escaped the produce body: our own
+ *  withTimeout rejections and the pipeline-deadline abort are `timeout`,
+ *  everything else (model error, bad transport, unexpected throw) is
+ *  `llm-error`. Raw error text never enters the ledger (jev/log redaction
+ *  envelope — the outcome class is the observable, not the message). */
+function classifySlowError(e, controller) {
+    if (e !== null && typeof e === 'object') {
+        const msg = e.message;
+        if (typeof msg === 'string' && msg.startsWith('slow-lane call timeout'))
+            return 'timeout';
+    }
+    if (controller.signal.aborted)
+        return 'timeout';
+    return 'llm-error';
+}
 export class SlowLane {
     pending = new Map();
     inFlight = new Map();
     /** Same-session shadow serialization — a second trigger never overlaps. */
     shadowInFlight = new Map();
+    /**
+     * The session's most recent slow-lane disposition (A4 决议 3): what a
+     * spliced with NO pending reports as its `slowOutcome` — without it a
+     * consumed-undefined round cannot say whether the lane never ran or ran
+     * and failed. One slot per session, overwritten by every trigger.
+     */
+    outcomeBySession = new Map();
     service;
     caller;
     /**
@@ -212,10 +238,39 @@ export class SlowLane {
     hasInFlight(sessionId) {
         return this.inFlight.has(sessionId);
     }
+    /**
+     * The session's most recent slow-lane disposition (A4 §3.2): a round whose
+     * consume() came up empty writes THIS as its ilog `slowOutcome` (spec §8.2
+     * — 「spliced 收到但未 produce 也留痕」). Undefined when the lane never
+     * triggered for the session.
+     */
+    lastOutcomeOf(sessionId) {
+        return this.outcomeBySession.get(sessionId);
+    }
+    /**
+     * One disposition, two sinks (A4 决议 3): the per-session slot above and a
+     * `slow-lane-outcome` row in the pipeline-events ledger. Metadata only
+     * (outcome/reason/session/turn/timing — never ring or query text). Never
+     * throws, never rejects: traceability must not become a failure source.
+     */
+    traceOutcome(sessionId, turnId, outcome, reason, extra = {}) {
+        try {
+            this.outcomeBySession.set(sessionId, outcome);
+            void recordPipelineEvent(this.service.store.root, {
+                at: new Date().toISOString(),
+                kind: 'slow-lane-outcome',
+                detail: { outcome, reason, sessionId, turnId, ...extra },
+            });
+        }
+        catch {
+            // contained — the ledger is best-effort by design
+        }
+    }
     /** Drop the pending slot (session teardown, restore boundary). */
     clear(sessionId) {
         this.pending.delete(sessionId);
         this.shadowInFlight.delete(sessionId);
+        this.outcomeBySession.delete(sessionId);
     }
     /**
      * Fastgate shadow (design §2 row 3): fire-and-forget on the SAME sampled
@@ -318,21 +373,36 @@ export class SlowLane {
     }
     /**
      * Produce a pending slot for the session's next steer message. Fire-and-
-     * forget; every guard failure is silent. Policy gates live here (config,
-     * sampling, in-flight); host gates (delegation, distill yield) live with
-     * the caller, which owns those services.
+     * forget; every guard failure is contained (never breaks the event
+     * handler) and QUALIFIED (A4 决议 3): a skipped trigger lands a `no-run`
+     * outcome naming the guard, so a later spliced can report why there was
+     * nothing to consume. Policy gates live here (config, sampling, in-flight);
+     * host gates (delegation, distill yield) live with the caller, which owns
+     * those services.
      */
     dispatch(sessionId, input) {
         try {
             const cfg = this.service.cfg;
-            if (cfg.qualityLane !== 'sampled' && cfg.qualityLane !== 'always')
+            if (cfg.qualityLane !== 'sampled' && cfg.qualityLane !== 'always') {
+                this.traceOutcome(sessionId, input.turnId, 'no-run', 'lane-off');
                 return;
-            if (cfg.qualityLane === 'sampled' && input.turnId % SAMPLED_EVERY !== 0)
+            }
+            if (cfg.qualityLane === 'sampled' && input.turnId % SAMPLED_EVERY !== 0) {
+                this.traceOutcome(sessionId, input.turnId, 'no-run', 'cadence');
+                return;
+            }
+            if (this.caller === undefined) {
+                this.traceOutcome(sessionId, input.turnId, 'no-run', 'no-caller');
                 return;
-            if (this.caller === undefined || this.inFlight.has(sessionId))
+            }
+            if (this.inFlight.has(sessionId)) {
+                this.traceOutcome(sessionId, input.turnId, 'no-run', 'in-flight');
                 return;
-            if (input.ring.length === 0)
+            }
+            if (input.ring.length === 0) {
+                this.traceOutcome(sessionId, input.turnId, 'no-run', 'empty-ring');
                 return;
+            }
             const run = this.produce(sessionId, input).finally(() => {
                 this.inFlight.delete(sessionId);
                 this.onSettle(sessionId);
@@ -347,11 +417,15 @@ export class SlowLane {
     async produce(sessionId, input) {
         const caller = this.caller;
         if (caller === undefined)
-            return;
+            return; // unreachable — dispatch gated it, already traced
         const started = Date.now();
         const controller = new AbortController();
         const deadline = setTimeout(() => controller.abort(), PIPELINE_TIMEOUT_MS);
         deadline.unref?.();
+        // A4 决议 3: every exit qualifies — outcome on the session + the ledger.
+        const trace = (outcome, reason) => {
+            this.traceOutcome(sessionId, input.turnId, outcome, reason, { ms: Date.now() - started });
+        };
         try {
             const cfg = this.service.cfg;
             const ringText = input.ring
@@ -373,18 +447,23 @@ export class SlowLane {
                 built = parseJsonObject(buildRaw);
             }
             catch {
-                return; // unparseable build → no pending, nothing logged, next turn retries
+                trace('llm-error', 'build-bad-json'); // unparseable build → no pending, next turn retries
+                return;
             }
-            if (built.needs !== true || typeof built.query !== 'string' || built.query.trim() === '')
+            if (built.needs !== true || typeof built.query !== 'string' || built.query.trim() === '') {
+                trace('no-claim', 'build-no-need'); // the query builder claimed no background need
                 return;
+            }
             const query = built.query.trim();
             const stripped = Array.isArray(built.ignore)
                 ? built.ignore.filter((s) => typeof s === 'string' && s.trim() !== '').slice(0, 5)
                 : [];
             const queryBuild = { rawChars: ringText.length, keptChars: query.length, stripped };
             const roster = await this.service.roster().catch(() => []);
-            if (roster.length === 0)
+            if (roster.length === 0) {
+                trace('no-candidates', 'roster-empty');
                 return;
+            }
             // Recall-oriented band: gate OFF (the rerank IS this lane's gate);
             // candidates = threshold passers + the near-miss band beneath them.
             const band = searchTopics(query, roster, {
@@ -399,8 +478,10 @@ export class SlowLane {
             const candidates = [...band.hits, ...band.nearMisses]
                 .filter((c) => !exclude?.has(c.slug))
                 .slice(0, CANDIDATE_LIMIT);
-            if (candidates.length === 0)
+            if (candidates.length === 0) {
+                trace('no-candidates', 'band-empty');
                 return;
+            }
             const bySlug = new Map(roster.map((r) => [r.slug, r]));
             const payload = candidates.flatMap((c) => {
                 const meta = bySlug.get(c.slug);
@@ -417,8 +498,10 @@ export class SlowLane {
                     },
                 ];
             });
-            if (payload.length === 0)
+            if (payload.length === 0) {
+                trace('no-candidates', 'payload-empty');
                 return;
+            }
             // ---- jev batch rerank (design §2 row 1) — replaces the LLM rerank ----
             // One batched noul request over the whole band; the calibrated bands
             // decide the picks. A settled jev round is FINAL here (picks or none) —
@@ -427,10 +510,16 @@ export class SlowLane {
             if (this.service.cfg.jevEnabled === true) {
                 const jevItems = await this.jevRerank(query, payload);
                 if (jevItems !== undefined) {
-                    if (jevItems.length > 0)
+                    if (jevItems.length > 0) {
                         this.settlePending(sessionId, input, jevItems, queryBuild, started);
+                        trace('delivered', 'jev-picks');
+                    }
+                    else {
+                        trace('no-candidates', 'jev-zero-picks');
+                    }
                     return;
                 }
+                // jev failure → the legacy rerank below owns the final trace.
             }
             const rerankRaw = await withTimeout(caller({
                 system: RERANK_PROMPT,
@@ -450,6 +539,7 @@ export class SlowLane {
                 reranked = parseJsonObject(rerankRaw);
             }
             catch {
+                trace('llm-error', 'rerank-bad-json');
                 return;
             }
             const legal = new Set(payload.map((p) => p.slug));
@@ -472,18 +562,29 @@ export class SlowLane {
                 if (items.length >= 2)
                     break;
             }
-            if (items.length === 0)
+            if (items.length === 0) {
+                trace('no-candidates', 'rerank-zero-picks'); // ran fine, gate picked nothing worth injecting
                 return;
+            }
             this.settlePending(sessionId, input, items, queryBuild, started);
+            trace('delivered', 'rerank-picks');
         }
-        catch {
-            // timeouts, aborts, model errors — I3: the async product is simply absent
+        catch (e) {
+            // RED LINE, A4 决议 3 (revising the old silent catch): the fast lane's
+            // fail-silently principle only covers the SYNCHRONOUS seam — this is an
+            // async lane, so the failure must be observable while still never
+            // rethrowing at the caller (I3 可丢弃性: the async product is absent,
+            // but its absence is now qualified, not mute).
+            trace(classifySlowError(e, controller), 'call-failed');
         }
         finally {
             clearTimeout(deadline);
         }
     }
-    /** One pending slot per settled pipeline (the 0-2 picks + lane metadata). */
+    /** One pending slot per settled pipeline (the 0-2 picks + lane metadata).
+     *  The pending carries its producing run's disposition (A4 §3.2) so the
+     *  ilog writer can copy it verbatim — a pending only exists when the run
+     *  delivered, hence the constant. */
     settlePending(sessionId, input, items, queryBuild, started) {
         const route = this.service.cfg;
         this.pending.set(sessionId, {
@@ -493,6 +594,7 @@ export class SlowLane {
             queryBuild,
             model: `${route.distillProvider}/${route.distillModel}`,
             ms: Date.now() - started,
+            outcome: 'delivered',
         });
     }
     /**
@@ -587,10 +689,12 @@ export class SlowLane {
         const produced = Date.parse(pending.computedAt);
         if (!Number.isNaN(produced) && Date.now() - produced > PENDING_TTL_MS) {
             this.pending.delete(sessionId);
+            this.traceOutcome(sessionId, turnId, 'expired', 'ttl', { ageMs: Date.now() - produced });
             return { expired: 'ttl' };
         }
         if (turnId - pending.turnId > TURN_LAG_LIMIT) {
             this.pending.delete(sessionId);
+            this.traceOutcome(sessionId, turnId, 'expired', 'turn-lag', { producedTurnId: pending.turnId });
             return { expired: 'turn-lag' };
         }
         this.pending.delete(sessionId);

package/lib/retrieval.d.ts

@@ -12,6 +12,7 @@
  * @module retrieval
  */
 import type { TopicMeta } from './store.ts';
+import type { EmbedResult } from './embedding.ts';
 export interface RetrievableTopic extends TopicMeta {
     /** `# Conclusion` section text (may be empty). */
     conclusion: string;
@@ -120,3 +121,108 @@ export declare function passesGate(s: {
  * evidence shows what the gate is costing.
  */
 export declare function searchTopics(query: string, roster: readonly RetrievableTopic[], cfg?: Partial<RetrievalConfig>): SearchOutcome;
+/**
+ * Token cap in tokenize() units — the SAME unit injections.jsonl logs as
+ * queryTokenCount (p90=462 baseline; real-query p50=153 stays untouched).
+ * Raised or lowered via SanitizeOptions.maxTokens.
+ */
+export declare const QUERY_TOKEN_CAP = 96;
+/**
+ * Lines longer than this are folded to their head (log-paste guard). Sized
+ * so the 96-token cap is the binding constraint, not the fold: 96 latin
+ * tokens ≈ 650+ chars, so a legitimate long single-line query passes the
+ * fold intact and only the token cap trims it. (实现期定 constant.)
+ */
+export declare const MAX_LINE_CHARS = 720;
+export interface SanitizeOptions {
+    maxTokens?: number;
+    maxLineChars?: number;
+}
+/**
+ * Sanitize a raw user query for retrieval (spec §3.3): strip fenced code
+ * blocks and URLs, fold line-leading shell prompts, fold over-long lines to
+ * their head, cap the distinct-token count. Returns '' when nothing lexical
+ * survives — the caller treats that as a zero-token query (natural skip).
+ * Pure function; the hot lane only reaches it behind packageMode.
+ */
+export declare function sanitizeQuery(raw: string, opts?: SanitizeOptions): string;
+export interface EmbeddingIndexEntry {
+    model: string;
+    dim: number;
+    vec: number[];
+}
+/** slug -> entry, exactly as persisted in meta/embeddings.json. */
+export type EmbeddingIndex = Record<string, EmbeddingIndexEntry>;
+export declare function embeddingIndexPath(bundleDir: string): string;
+/** Load meta/embeddings.json from a bundle dir. Missing file, corrupt JSON,
+ *  or malformed entries are skipped/empty — never throws (a broken index
+ *  must degrade to the lexical path, not break retrieval). */
+export declare function loadEmbeddingIndex(bundleDir: string): EmbeddingIndex;
+/** Persist the index in the canonical shape (the rebuild's write side —
+ *  embedding/rebuilding itself stays with the caller). Offline rebuild
+ *  path, not the hot lane. */
+export declare function saveEmbeddingIndex(bundleDir: string, index: EmbeddingIndex, mkdir?: boolean): void;
+/**
+ * An embed function bound to a config, carrying the RESOLVED model identity
+ * the index must match (build once via embedding.ts's `embedderWith`).
+ * ensureIndexFresh reads `.model` only — it never calls the function; the
+ * rebuild is the caller's job.
+ */
+export type EmbedderFn = ((texts: readonly string[]) => Promise<EmbedResult>) & {
+    model: string;
+};
+/**
+ * Freshness audit: which slugs need (re)embedding — missing from the index,
+ * built by a different model, or shape-broken (dim/vec disagree). Returns
+ * them in roster order; an EMPTY list means the index covers the roster.
+ * A '' embedder model means the semantic layer is off → nothing to maintain.
+ *
+ * NOTE the freshness boundary: content-change invalidation needs a stored
+ * content hash, which the pinned {model,dim,vec} entry shape does not carry
+ * — the rebuild trigger policy is spec-实现期定 and stays with the caller.
+ */
+export declare function ensureIndexFresh(index: EmbeddingIndex, topics: readonly {
+    slug: string;
+}[], embedFn: EmbedderFn): string[];
+export declare const RERANK_FLOOR = 0.2;
+export declare const RERANK_CEILING = 0.3;
+export declare const SEMANTIC_RERANK_REASON = "semantic-rerank";
+export interface RerankedHit extends SearchHit {
+    /** Cosine(query, slug-vector), rounded 4dp. Present only on reranked
+     *  hits; absent = the candidate rode through unchanged. */
+    similarity?: number;
+}
+export type RerankPassthroughReason = 'no-candidates' | 'no-query-vector' | 'empty-index' | 'no-rerankable-candidates';
+export interface RerankOptions {
+    /** Expected index model; entries built by another model count as
+     *  vector-less (they ride through at the band tail). */
+    expectedModel?: string;
+    floor?: number;
+    ceiling?: number;
+}
+export interface RerankResult {
+    /** Candidates in final order. On passthrough: the input array, same
+     *  order, no tags — byte-identical ordering, explicit why. */
+    hits: RerankedHit[];
+    /** true = the semantic layer engaged (order may or may not have moved). */
+    reranked: boolean;
+    passthroughReason?: RerankPassthroughReason;
+}
+/** Cosine similarity; 0 for dim mismatch or zero vectors (never throws). */
+export declare function cosineSimilarity(a: readonly number[], b: readonly number[]): number;
+/**
+ * Reorder the nearFloor band (0.20–0.30 by gateScore) by cosine similarity
+ * against the query vector. Band members with a usable index vector sort by
+ * similarity (stable — ties keep input order) and gain the
+ * `semantic-rerank` reason plus their similarity; everyone else — out of
+ * band, unindexed, model-mismatched — passes through UNCHANGED at the tail
+ * in input order. Scores are never touched.
+ *
+ * Degradation is explicit and total: provider/model unconfigured, probe
+ * failure, missing query vector or missing index all return the input as-is
+ * with `reranked:false` and a passthroughReason (原序透传，不许静默 — the
+ * caller logs it into the degraded status). `queryText` rides the signature
+ * for provenance/debug parity with the caller's logging; the v1 math is
+ * vector-only.
+ */
+export declare function rerankNearFloor(queryText: string, queryVec: readonly number[] | null | undefined, nearFloorCandidates: readonly SearchHit[], index: EmbeddingIndex, opts?: RerankOptions): RerankResult;