@aiwayds/dsh-topics-memory 0.22.0 → 0.24.0

package/lib/retrieval.js

@@ -11,6 +11,8 @@
  *
  * @module retrieval
  */
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
 import { dependsSlugs } from "./okf.js";
 /** Hard cap for the usage bonus — below threshold 0.3 so usage can never
  *  carry a candidate across the gate on its own (ADR 0015). */
@@ -256,3 +258,281 @@ export function searchTopics(query, roster, cfg = {}) {
     }
     return { hits, nearMisses, rosterSize: roster.length };
 }
+// ---------------------------------------------------------------------------
+// Injection-quality line (wave/injection-quality, spec §2.1/§3.1/§3.3) —
+// CAPABILITY ONLY. None of this is wired into searchTopics or any service
+// path; the caller gates it on packageMode ≠ 'off' (Phase 2 wiring). With
+// the config keys at their defaults every code path below is unreachable
+// from the hot lane, so default-off behavior stays byte-for-byte.
+// ---------------------------------------------------------------------------
+// --- ① Query sanitization (A4 决议 4, spec §3.3) ---------------------------
+// Pure, LLM-free, millisecond-scale. Treats the queryTokenCount p90=462
+// paste-pollution tail: fenced code blocks, URLs, line-leading shell
+// prompts, over-long lines, and a hard token cap.
+/**
+ * Token cap in tokenize() units — the SAME unit injections.jsonl logs as
+ * queryTokenCount (p90=462 baseline; real-query p50=153 stays untouched).
+ * Raised or lowered via SanitizeOptions.maxTokens.
+ */
+export const QUERY_TOKEN_CAP = 96;
+/**
+ * Lines longer than this are folded to their head (log-paste guard). Sized
+ * so the 96-token cap is the binding constraint, not the fold: 96 latin
+ * tokens ≈ 650+ chars, so a legitimate long single-line query passes the
+ * fold intact and only the token cap trims it. (实现期定 constant.)
+ */
+export const MAX_LINE_CHARS = 720;
+const FENCE_OPEN = /^[ \t]*(```|~~~)/;
+// Line-leading shell prompt: `user@host:~/repo$ `, `C:\> `, or a bare
+// `$ `/`# `/`% `/`> `/`❯ ` marker. Requires trailing whitespace so words
+// like `$PATH` or `#tag` survive untouched.
+const LINE_PROMPT = /^[ \t]*(?:[\w.@:/\\~-]*[$#%>❯➜])[ \t]+/;
+const URL_RE = /\b(?:https?:\/\/|www\.)[^\s<>"'`)\]]+/gi;
+// Chunks the cap walks: one latin/digit word (or hyphenated compound) per
+// match, or one CJK run (sliced when a run alone busts the cap).
+const CHUNK_RUN = new RegExp(`${WORD_RUN.source}|${CJK_RUN.source}`, 'gu');
+/** Remove fenced code blocks wholesale (markers AND body): fenced code is
+ *  the paste-pollution payload; the prose around it is the actual query. */
+function stripFencedBlocks(text) {
+    const out = [];
+    let open = null;
+    for (const line of text.split('\n')) {
+        const m = line.match(FENCE_OPEN);
+        if (open === null) {
+            if (m !== null)
+                open = m[1];
+            else
+                out.push(line);
+        }
+        else if (m !== null && m[1] === open) {
+            open = null; // closing fence — block fully dropped
+        }
+        // inside an open fence: drop the line
+    }
+    return out.join('\n');
+}
+/** Cap the sanitized text to ≤ cap distinct retrieval tokens. Fast path:
+ *  at or under the cap the text passes through VERBATIM (no re-chunking —
+ *  hyphens, punctuation and spacing survive byte-exact; retrieval scores
+ *  tokens, not bytes, but the string may be echoed into logs/injection).
+ *  Only an over-cap text rebuilds from chunks: head-first (a query's intent
+ *  leads; pollution trails), with CJK runs sliced at the character level so
+ *  a single giant run truncates instead of being dropped whole. */
+function capTokens(text, cap) {
+    const total = tokenize(text).length;
+    if (total === 0)
+        return ''; // nothing lexical survives — caller sees a zero-token query
+    if (total <= cap)
+        return text;
+    const seen = new Set();
+    const kept = [];
+    for (const chunk of text.match(CHUNK_RUN) ?? []) {
+        const fresh = tokenize(chunk).filter((t) => !seen.has(t));
+        if (fresh.length === 0)
+            continue; // already-covered chunk adds nothing
+        if (seen.size + fresh.length <= cap) {
+            for (const t of fresh)
+                seen.add(t);
+            kept.push(chunk);
+            continue;
+        }
+        const room = cap - seen.size;
+        if (room <= 0)
+            break;
+        // Over budget: CJK runs truncate exactly (slice room+1 chars → ≤ room
+        // fresh bigrams, so the cap invariant holds). Latin chunks are single
+        // tokens by construction — they never reach here with room > 0, but if
+        // one ever does, dropping it beats busting the cap.
+        if (/^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(chunk)) {
+            kept.push(chunk.slice(0, room + 1));
+        }
+        break;
+    }
+    return kept.join(' ');
+}
+/**
+ * Sanitize a raw user query for retrieval (spec §3.3): strip fenced code
+ * blocks and URLs, fold line-leading shell prompts, fold over-long lines to
+ * their head, cap the distinct-token count. Returns '' when nothing lexical
+ * survives — the caller treats that as a zero-token query (natural skip).
+ * Pure function; the hot lane only reaches it behind packageMode.
+ */
+export function sanitizeQuery(raw, opts = {}) {
+    const cap = opts.maxTokens ?? QUERY_TOKEN_CAP;
+    const maxLineChars = opts.maxLineChars ?? MAX_LINE_CHARS;
+    if (raw === '')
+        return '';
+    let text = stripFencedBlocks(raw);
+    text = text.replace(URL_RE, ' ');
+    text = text
+        .split('\n')
+        .map((line) => {
+        let l = line.replace(LINE_PROMPT, '');
+        if (l.length > maxLineChars)
+            l = l.slice(0, maxLineChars);
+        return l;
+    })
+        .join('\n');
+    text = text.replace(/\s+/g, ' ').trim();
+    if (text === '')
+        return '';
+    return capTokens(text, cap);
+}
+export function embeddingIndexPath(bundleDir) {
+    return join(bundleDir, 'meta', 'embeddings.json');
+}
+function parseEmbeddingIndexEntry(value) {
+    if (typeof value !== 'object' || value === null)
+        return null;
+    const e = value;
+    if (typeof e.model !== 'string' || e.model === '')
+        return null;
+    if (typeof e.dim !== 'number' || !Number.isInteger(e.dim) || e.dim <= 0)
+        return null;
+    if (!Array.isArray(e.vec) || e.vec.length !== e.dim)
+        return null;
+    if (!e.vec.every((n) => typeof n === 'number' && Number.isFinite(n)))
+        return null;
+    return { model: e.model, dim: e.dim, vec: e.vec };
+}
+/** Load meta/embeddings.json from a bundle dir. Missing file, corrupt JSON,
+ *  or malformed entries are skipped/empty — never throws (a broken index
+ *  must degrade to the lexical path, not break retrieval). */
+export function loadEmbeddingIndex(bundleDir) {
+    let raw;
+    try {
+        raw = readFileSync(embeddingIndexPath(bundleDir), 'utf8');
+    }
+    catch {
+        return {};
+    }
+    try {
+        const json = JSON.parse(raw);
+        if (typeof json !== 'object' || json === null || Array.isArray(json))
+            return {};
+        const out = {};
+        for (const [slug, entry] of Object.entries(json)) {
+            if (slug === '')
+                continue;
+            const parsed = parseEmbeddingIndexEntry(entry);
+            if (parsed !== null)
+                out[slug] = parsed;
+        }
+        return out;
+    }
+    catch {
+        return {};
+    }
+}
+/** Persist the index in the canonical shape (the rebuild's write side —
+ *  embedding/rebuilding itself stays with the caller). Offline rebuild
+ *  path, not the hot lane. */
+export function saveEmbeddingIndex(bundleDir, index, mkdir = true) {
+    if (mkdir)
+        mkdirSync(join(bundleDir, 'meta'), { recursive: true });
+    writeFileSync(embeddingIndexPath(bundleDir), JSON.stringify(index));
+}
+/**
+ * Freshness audit: which slugs need (re)embedding — missing from the index,
+ * built by a different model, or shape-broken (dim/vec disagree). Returns
+ * them in roster order; an EMPTY list means the index covers the roster.
+ * A '' embedder model means the semantic layer is off → nothing to maintain.
+ *
+ * NOTE the freshness boundary: content-change invalidation needs a stored
+ * content hash, which the pinned {model,dim,vec} entry shape does not carry
+ * — the rebuild trigger policy is spec-实现期定 and stays with the caller.
+ */
+export function ensureIndexFresh(index, topics, embedFn) {
+    if ((embedFn.model ?? '') === '')
+        return [];
+    const stale = [];
+    for (const t of topics) {
+        const e = index[t.slug];
+        if (e === undefined || e.model !== embedFn.model || e.dim !== e.vec.length || !(e.dim > 0))
+            stale.push(t.slug);
+    }
+    return stale;
+}
+// --- ③ nearFloor semantic rerank (A4 决议 2, spec §3.1) ---------------------
+// Cosine reordering of the nearFloor band ONLY (0.20–0.30 — the 93%
+// near-miss mass). Thresholds, gateScore and the structural gate are
+// untouched: the reranker reorders and tags, never re-scores.
+export const RERANK_FLOOR = 0.2;
+export const RERANK_CEILING = 0.3;
+export const SEMANTIC_RERANK_REASON = 'semantic-rerank';
+/** Cosine similarity; 0 for dim mismatch or zero vectors (never throws). */
+export function cosineSimilarity(a, b) {
+    if (a.length === 0 || a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < a.length; i += 1) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    if (na === 0 || nb === 0)
+        return 0;
+    return dot / Math.sqrt(na * nb);
+}
+/**
+ * Reorder the nearFloor band (0.20–0.30 by gateScore) by cosine similarity
+ * against the query vector. Band members with a usable index vector sort by
+ * similarity (stable — ties keep input order) and gain the
+ * `semantic-rerank` reason plus their similarity; everyone else — out of
+ * band, unindexed, model-mismatched — passes through UNCHANGED at the tail
+ * in input order. Scores are never touched.
+ *
+ * Degradation is explicit and total: provider/model unconfigured, probe
+ * failure, missing query vector or missing index all return the input as-is
+ * with `reranked:false` and a passthroughReason (原序透传，不许静默 — the
+ * caller logs it into the degraded status). `queryText` rides the signature
+ * for provenance/debug parity with the caller's logging; the v1 math is
+ * vector-only.
+ */
+export function rerankNearFloor(queryText, queryVec, nearFloorCandidates, index, opts = {}) {
+    void queryText; // v1 math is vector-only — see doc above
+    if (nearFloorCandidates.length === 0) {
+        return { hits: [], reranked: false, passthroughReason: 'no-candidates' };
+    }
+    if (!Array.isArray(queryVec) || queryVec.length === 0) {
+        return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'no-query-vector' };
+    }
+    if (Object.keys(index).length === 0) {
+        return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'empty-index' };
+    }
+    const floor = opts.floor ?? RERANK_FLOOR;
+    const ceiling = opts.ceiling ?? RERANK_CEILING;
+    const simOf = new Map();
+    for (const hit of nearFloorCandidates) {
+        const entry = index[hit.slug];
+        const usable = entry !== undefined &&
+            entry.dim === entry.vec.length &&
+            entry.dim > 0 &&
+            entry.dim === queryVec.length &&
+            (opts.expectedModel === undefined || entry.model === opts.expectedModel);
+        simOf.set(hit, usable ? cosineSimilarity(queryVec, entry.vec) : null);
+    }
+    const band = nearFloorCandidates.filter((h) => {
+        const sim = simOf.get(h);
+        return sim !== null && sim !== undefined && h.gateScore !== undefined && h.gateScore >= floor && h.gateScore < ceiling;
+    });
+    if (band.length === 0) {
+        return { hits: [...nearFloorCandidates], reranked: false, passthroughReason: 'no-rerankable-candidates' };
+    }
+    const rerankedSet = new Set(band);
+    const tail = nearFloorCandidates.filter((h) => !rerankedSet.has(h));
+    // Stable sort: V8's Array#sort keeps input order for equal similarities.
+    const ordered = [...band].sort((x, y) => simOf.get(y) - simOf.get(x));
+    return {
+        hits: [
+            ...ordered.map((h) => {
+                const sim = simOf.get(h);
+                return { ...h, reasons: [...h.reasons, SEMANTIC_RERANK_REASON], similarity: Math.round(sim * 10_000) / 10_000 };
+            }),
+            ...tail.map((h) => ({ ...h })),
+        ],
+        reranked: true,
+    };
+}

package/lib/service.d.ts

@@ -10,7 +10,7 @@ import { type RetrievableTopic, type SearchOutcome } from './retrieval.ts';
 import { type AssembleResult } from './digest.ts';
 import type { BundleStore, Observation, SaveResult } from './store.ts';
 import type { TopicsConfigValue } from './config.ts';
-import { type AggregateStats, type QueryBuildShape, type UsageSignal } from './ilog.ts';
+import { type AggregateStats, type QueryBuildShape, type SlowOutcome, type UsageSignal } from './ilog.ts';
 import type { Sync } from './sync.ts';
 export interface RetrieveOutcome {
     outcome: SearchOutcome;
@@ -33,6 +33,42 @@ export interface SlowDelivery {
     queryBuild: QueryBuildShape;
     model: string;
     ms: number;
+    /** Producing run's disposition (A4 全留痕) — copied into ilog.slowOutcome. */
+    outcome?: SlowOutcome;
+}
+/**
+ * One structured observation from the new repo's signal layer (B1 §2.1
+ * whitelist): FEATURES ONLY, never conversation text (红线⑤). Unknown caller
+ * fields cannot pass — the whitelist copy in feedObservations is the gate.
+ */
+export interface StructuredObservation {
+    /** Signal type from the B1 closed list (`tool-error`, `turn-end-negative`,
+     *  `unused-injection`, `repeat-call`, `interruption-pair`, `jev-calibration`). */
+    type: string;
+    /** ISO instant; defaults to now. */
+    at?: string;
+    sessionId?: string;
+    /** Bounded scalar feature fields (toolName/errorName/count/band/...). */
+    detail?: Record<string, string | number | boolean>;
+}
+export interface FeedResult {
+    accepted: number;
+    duplicates: number;
+    rejected: number;
+    ids: string[];
+    /** Present when the call itself was refused (missing/blank actor). */
+    error?: 'unauthorized';
+}
+export interface StoreResult {
+    ok: boolean;
+    slug: string;
+    /** Refusal reason; absent on success. */
+    reason?: 'unauthorized' | 'not-found' | 'not-a-lesson' | 'invalid-state';
+    /** confidence before the transition (when the lesson was read). */
+    from?: okf.LessonConfidence;
+    /** removed only: bundle-relative archive path (git-revertable). */
+    archivedTo?: string;
+    committed?: boolean;
 }
 export declare class TopicsService {
     private cache;
@@ -101,6 +137,22 @@ export declare class TopicsService {
      * the slow lane's recall band stays untouched, mirroring usageBoost).
      */
     private zeroOpenCfg;
+    /** mtime-cached view of meta/embeddings.json (sync read — hot-lane caller).
+     *  Missing/corrupt index = {} → the reranker degrades to lexical order. */
+    private embedIndexCache;
+    private embeddingIndexCacheGet;
+    /** Shadow bypass ledger (spec §8.3): one line per shadow round — feature
+     *  fields only, never query text (jev-log discipline). Silent on failure;
+     *  512KB truncation keeps the newest three quarters. Shared by the package
+     *  shadow (meta/package-shadow.jsonl) and the lesson shadow (B2,
+     *  meta/lesson-shadow.jsonl). */
+    private appendShadowLedger;
+    private packageShadowLedgerPath;
+    private lessonShadowLedgerPath;
+    /** Boot-time embedding index maintenance (spec §3: boot 校验+增量重建).
+     *  Provider '' or packageMode off → no-op (default-off byte-for-byte).
+     *  Full-roster embed on rebuild (A1 §5: ~9.5s for 137 topics). */
+    refreshEmbeddingIndex(): Promise<void>;
     /**
      * Full retrieval round: search → assemble → log (ADR 0006/0007). The
      * injection record is written even for zero-hit rounds — the hit-rate
@@ -140,6 +192,7 @@ export declare class TopicsService {
         echoed: string[];
         included: string[];
         slowIncluded: string[];
+        lessonIncluded: string[];
     };
     /**
      * topic_open tool path (v4 §4.1): full conclusion / open questions /
@@ -167,9 +220,25 @@ export declare class TopicsService {
     private attributeOpen;
     /** Sync roster read (same mtime cache as roster()). */
     rosterSync(): RetrievableTopic[];
+    /** Established, scope-passing lesson entries (B2, spec 卷一 §6.3) — sync,
+     *  sharing the roster's mtime cache. Injection reads ONLY established
+     *  lessons here (MJ-3: hint 仅供审批队列展示，永不注入); scope filter =
+     *  workspace lessons must carry the current workspace id (MJ-2); scope=global
+     *  and legacy entries without a workspaceId pass (fail-open — the filter's
+     *  default is permissive, the confidence gate is the hard one). */
+    lessonsSync(): (RetrievableTopic & {
+        confidence: okf.LessonConfidence;
+        source: okf.LessonSource;
+        scope: okf.LessonScope;
+        workspaceId?: string;
+    })[];
     private readDocSync;
     private readDoc;
-    /** topic_save tool path: create or update with actor stamping. */
+    /** topic_save tool path: create or update with actor stamping. `kind:
+     *  'lesson'` (B2 data plane) switches the OKF type to Lesson and stamps the
+     *  lesson frontmatter (scope/confidence/source/workspaceId) with the
+     *  confidence↔status derivation (hint↔draft, established↔stable —
+     *  confidence is the fact source). */
     saveTopic(input: {
         title: string;
         description?: string;
@@ -178,11 +247,19 @@ export declare class TopicsService {
         depends?: string[];
         openQuestions?: string[];
         impact?: string[];
+        /** A2 semantic-link targets; store.saveTopic lands them as body wikilinks. */
+        links?: string[];
         status?: okf.TopicStatus;
         conclusion?: string;
         recommendations?: string;
         source?: 'model' | 'distill';
         slug?: string;
+        /** B2 lesson data plane: entry kind (default topic). */
+        kind?: okf.TopicKind;
+        scope?: okf.LessonScope;
+        confidence?: okf.LessonConfidence;
+        /** Lesson provenance (input.source above is the tool-path tag). */
+        lessonSource?: okf.LessonSource;
     }): Promise<SaveResult & {
         slug: string;
     }>;
@@ -192,6 +269,30 @@ export declare class TopicsService {
         sessionId?: string;
         source?: 'model' | 'auto';
     }): Promise<Observation>;
+    /**
+     * ① 观察投喂: structured observations (B1 §2.1 whitelist, FEATURES ONLY —
+     * 对话原文禁入, 红线⑤) appended to the observations stream with the `fed`
+     * provenance marker and `kind: 'signal'`; each line carries `by: actor`.
+     * The whitelist copy drops unknown caller fields, caps detail entries and
+     * string sizes, and renders a sorted feature line as the observation text.
+     * Duplicates (same type+session+detail in the recent tail) are counted, not
+     * appended (重复观察由原仓去重). Fail-soft: one malformed entry is counted
+     * as rejected, never sinks the batch.
+     */
+    feedObservations(obs: readonly StructuredObservation[], opts: {
+        actor: string;
+    }): Promise<FeedResult>;
+    /**
+     * ② lesson 状态变更 (B2/卷二 §6.2): hint→established / →deprecated /
+     * →removed. established flips confidence (the FACT source) and derives
+     * status (established↔stable); deprecated flips status only (the TTL sweep
+     * owns the rest); removed = archive FIRST (byte-exact copy to
+     * `archive/lessons/` inside the bundle, git-revertable — 红线④) then out of
+     * the roster, one commit. Each call lands a `seam-lesson-status` audit row.
+     */
+    setLessonStatus(slug: string, to: 'established' | 'deprecated' | 'removed', opts: {
+        actor: string;
+    }): Promise<StoreResult>;
     search(query: string, topK?: number): Promise<SearchOutcome>;
     history(slug: string, limit?: number): Promise<{
         entries: {