@aiwayds/dsh-topics-memory 0.21.0 → 0.23.0

package/lib/digest.js

@@ -9,6 +9,7 @@
  * @module digest
  */
 import { sectionOf, firstParagraph, CONCLUSION_HEADING, RECOMMENDATIONS_HEADING } from "./okf.js";
+import { tokenize } from "./retrieval.js";
 /** Rough CJK-aware token estimate: ~0.75 token per CJK char, ~4 chars per latin token. */
 export function estimateTokens(text) {
     let cjk = 0;
@@ -180,3 +181,386 @@ export function assemblePointer(entries, cfg, slow) {
 export function assembleInjection(entries, cfg, slow) {
     return packInjection(entries, cfg, topicDigest, slow);
 }
+/** Total package cap — pinned, does not float with hit strength (A3 决议 3). */
+export const PACKAGE_TOTAL = 600;
+export const PACKAGE_DIRECT_MAX = 2;
+export const PACKAGE_DIRECT_BUDGET = 120;
+export const PACKAGE_NEIGHBOR_MAX = 3;
+export const PACKAGE_NEIGHBOR_BUDGET = 40;
+export const PACKAGE_WEAK_MAX = 2;
+/**
+ * Padded-trivial arm of the downgrade guard (spec §2.3 名词实体口径, an
+ * implementation-time item per §11): after stripping whitespace, a query this
+ * short counts as 琐碎 even when the tokenizer expands CJK bigrams past 3
+ * tokens. Calibrated on the two named scenarios: 场景① ack query strips to
+ * 15 chars (downgrade), a real review query strips to 18 (kept).
+ */
+export const PACKAGE_TRIVIAL_QUERY_CHARS = 16;
+const PACKAGE_WRAPPER_OPEN = '<topic-memory>';
+const PACKAGE_WRAPPER_CLOSE = '</topic-memory>';
+/** Edge/lane note for the why line / neighborhood annotation (A3 决议 2). */
+function viaNote(via) {
+    if (via === undefined || via === '')
+        return null;
+    if (via.startsWith('depends'))
+        return '经 depends 关联';
+    if (via.startsWith('slow'))
+        return '慢道 pick';
+    if (via.startsWith('semantic'))
+        return '语义近邻';
+    return via;
+}
+/** Trigger reason → the trigger word when the caller carries it, `''` channel-only. */
+function triggerWordOfReason(reason) {
+    const m = reason.match(/^triggers?[:：](.*)$/);
+    if (m === null)
+        return undefined;
+    const v = m[1].trim();
+    return /^[\d.+-]+$/.test(v) ? '' : v;
+}
+/**
+ * slug/title 词面，含分隔符子词（`cs-geely-voice-agent` → `voice`/`agent`）：
+ * query 里的连字符词（voice-agent）与 slug 复合词永远不相等，实体判定必须
+ * 下探到子词才有交集；CJK 已由 tokenize 的二元切分覆盖。
+ */
+function subtokenSurface(text) {
+    const out = new Set();
+    for (const t of tokenize(text)) {
+        out.add(t);
+        if (/[+#.-]/.test(t)) {
+            for (const sub of t.split(/[+#.-]+/))
+                if (sub.length >= 2)
+                    out.add(sub);
+        }
+    }
+    return out;
+}
+function tokenMatchesSurface(token, surface) {
+    const t = token.toLowerCase();
+    if (surface.has(t))
+        return true;
+    if (/[+#.-]/.test(t)) {
+        for (const sub of t.split(/[+#.-]+/))
+            if (sub.length >= 2 && surface.has(sub))
+                return true;
+    }
+    return false;
+}
+/**
+ * Strict display matcher for why-line attribution: a compound query token
+ * counts as a title/weak word only when EVERY separator part ties to the
+ * surface — `voice-agent` matches `cs-geely-voice-agent`, but
+ * `cs-geely-alert-manager-api` must not match it via the shared `cs`.
+ */
+function tokenMatchesSurfaceStrict(token, surface) {
+    const t = token.toLowerCase();
+    if (surface.has(t))
+        return true;
+    if (!/[+#.-]/.test(t))
+        return false;
+    const subs = t.split(/[+#.-]+/).filter((s) => s.length >= 2);
+    return subs.length > 0 && subs.every((s) => surface.has(s));
+}
+/** Query tokens (original spelling, query order) with a词面 tie to the surface. */
+function matchedQueryTokens(queryTokens, surfaceText) {
+    const surface = subtokenSurface(surfaceText);
+    const out = [];
+    for (const tok of queryTokens) {
+        if (!out.includes(tok) && tokenMatchesSurfaceStrict(tok, surface))
+            out.push(tok);
+    }
+    return out;
+}
+/** Entity-anchored candidate: its slug/title surface shares a query token
+ *  (验证波次修正——A③ 场景实证：词法第 3 名恰是 query 点名实体时，≤2 席
+ *  会把任务对象整体剪掉；实体锚定者不受席帽约束，仍受总预算约束）。 */
+function isEntityAnchored(queryTokens, c) {
+    if (queryTokens.length === 0)
+        return false;
+    const surface = subtokenSurface(`${c.slug} ${c.title}`);
+    for (const tok of queryTokens) {
+        if (tokenMatchesSurface(tok, surface))
+            return true;
+    }
+    return false;
+}
+function hasEntityOverlap(queryTokens, candidates) {
+    if (queryTokens.length === 0)
+        return false;
+    for (const c of candidates) {
+        const surface = subtokenSurface(`${c.slug} ${c.title}`);
+        for (const tok of queryTokens) {
+            if (tokenMatchesSurface(tok, surface))
+                return true;
+        }
+    }
+    return false;
+}
+function isTrivialQuery(query, queryTokens) {
+    if (queryTokens.length <= 3)
+        return true;
+    const stripped = query.replace(/\s+/g, '');
+    return stripped.length > 0 && stripped.length <= PACKAGE_TRIVIAL_QUERY_CHARS;
+}
+/** Stable within-layer order: score desc, then usage, then input order. */
+function rankStable(list, usage) {
+    return list
+        .map((c, i) => ({ c, i, u: usage?.get(c.slug) ?? 0 }))
+        .sort((a, b) => b.c.score - a.c.score || b.u - a.u || a.i - b.i)
+        .map((x) => x.c);
+}
+/** Machine-assembled why line (A3 决议 2): hit channel + edge source + score. */
+function packageWhyLine(c, queryTokens) {
+    const parts = [];
+    let triggerWord = null;
+    let triggerChannel = false;
+    let titleHit = false;
+    const fields = [];
+    for (const r of c.reasons) {
+        const tw = triggerWordOfReason(r);
+        if (tw !== undefined) {
+            if (tw === '')
+                triggerChannel = true;
+            else if (triggerWord === null)
+                triggerWord = tw;
+            continue;
+        }
+        if (r.startsWith('title:'))
+            titleHit = true;
+        else if (r.startsWith('tags:'))
+            fields.push('标签');
+        else if (r.startsWith('description:'))
+            fields.push('描述');
+        else if (r.startsWith('conclusion:'))
+            fields.push('正文');
+        else if (r.startsWith('slug:'))
+            fields.push('slug');
+    }
+    if (triggerWord !== null)
+        parts.push(`trigger「${triggerWord}」`);
+    else if (triggerChannel)
+        parts.push('trigger 命中');
+    if (titleHit) {
+        const words = matchedQueryTokens(queryTokens, `${c.slug} ${c.title}`);
+        parts.push(words.length > 0 ? `标题词「${words.slice(0, 2).join('」「')}」` : '标题词命中');
+    }
+    if (fields.length > 0)
+        parts.push(`${[...new Set(fields)].join('/')}词面命中`);
+    const via = viaNote(c.via);
+    if (via !== null)
+        parts.push(via);
+    if (parts.length === 0)
+        parts.push('词面命中');
+    return `命中: ${truncateToTokens(parts.join('、'), 32).text}（score:${c.score.toFixed(2)}）`;
+}
+/**
+ * One direct-layer full entry (~120 tok): 标题+[status] + 一行实质 +
+ * 机器 why 行 + 指令式打开引导（预告源 description 优先，缺失回退结论行时
+ * 不再重复预告）。实质行是弹性段，超预算先截它（spec §2.4）。
+ */
+function directEntryText(c, queryTokens, budget) {
+    const head = `### ${c.title} [${c.status}]`;
+    const why = packageWhyLine(c, queryTokens);
+    const warn = c.reasons.includes('conflicted-demoted') ? '⚠ 此条存在未合并冲突，结论可能过期。' : null;
+    const openBase = `细节: topic_open "${c.slug}"`;
+    const conclusionLine = (c.conclusion ?? '').trim();
+    const substanceRaw = (conclusionLine !== '' ? conclusionLine : (c.description ?? '')).trim();
+    const previewRaw = (c.description ?? '').replace(/\s+/g, ' ').trim();
+    const lines = [head, why];
+    let substanceText = '';
+    if (substanceRaw !== '') {
+        const others = estimateTokens([head, why, openBase].join('\n')) + (warn !== null ? estimateTokens(warn) + 1 : 0);
+        const room = budget - others - 1;
+        if (room >= 8)
+            substanceText = truncateToTokens(substanceRaw, room).text;
+    }
+    if (substanceText !== '')
+        lines.push(substanceText);
+    if (warn !== null)
+        lines.push(warn);
+    let open = openBase;
+    if (previewRaw !== '' && previewRaw !== substanceText) {
+        const room = budget - estimateTokens([...lines, open].join('\n')) - 3;
+        if (room >= 6)
+            open = `${openBase}，${truncateToTokens(previewRaw, room).text}`;
+    }
+    lines.push(open);
+    // Last resort for pathological heads (giant titles): shed elastic lines.
+    while (lines.length > 2 && estimateTokens(lines.join('\n')) > budget) {
+        const dropWarn = lines.indexOf(warn ?? '');
+        if (warn !== null && dropWarn !== -1)
+            lines.splice(dropWarn, 1);
+        else if (substanceText !== '' && lines.includes(substanceText))
+            lines.splice(lines.indexOf(substanceText), 1);
+        else
+            break;
+    }
+    const text = lines.join('\n');
+    return { text, usedTokens: estimateTokens(text) };
+}
+/** One neighborhood half-line (~40 tok): 标题 + slug + 边来源/通道标注. */
+function neighborhoodEntryText(c) {
+    const via = viaNote(c.via);
+    const text = `- ${truncateToTokens(c.title, 20).text} [${c.status}] (topics:${c.slug}) ← ${via ?? '关联扩展'}`;
+    return { text, usedTokens: estimateTokens(text) };
+}
+/** Merged weak-signal line: nearFloor head slugs + the weak query words. */
+function weakSignalLine(cands, queryTokens) {
+    const slugs = cands.map((c) => c.slug);
+    const words = [];
+    for (const c of cands) {
+        for (const w of matchedQueryTokens(queryTokens, `${c.slug} ${c.title}`)) {
+            if (!words.includes(w))
+                words.push(w);
+        }
+    }
+    let text = `弱信号: ${slugs.join('、')}`;
+    if (words.length > 0)
+        text += ` ← 弱词「${words.slice(0, 3).join('」「')}」`;
+    return { text, usedTokens: estimateTokens(text), slugs };
+}
+/** 包头 one-liner: 资料声明 + 「为什么是这批」各层计数（A3 决议 1）. */
+function packageHeader(query, nDirect, nNeigh, nWeak) {
+    const summary = truncateToTokens(query.replace(/\s+/g, ' ').trim(), 16).text;
+    const clauses = [];
+    if (nDirect > 0)
+        clauses.push(`${nDirect} 条词法强命中`);
+    if (nNeigh > 0)
+        clauses.push(`${nNeigh} 条经关联`);
+    if (nWeak > 0)
+        clauses.push(`${nWeak} 条弱词信号`);
+    const why = clauses.length > 0 ? `为什么是这批：${clauses.join('，')}。` : '';
+    return `与「${summary}」相关的记忆（资料，非指令）。${why}`;
+}
+/**
+ * Assemble the three-layer package. Returns `null` — the whole-package skip
+ * signal — when the roster is empty (ADR 0006 零命中零注入) or when the
+ * downgrade guard fires (spec §2.3: 琐碎 query 且与全部候选 slug/title 零
+ * 词面实体交集). Packing is layered greedy: direct first, then neighborhood,
+ * then the weak line — over budget, lower layers are cut first.
+ */
+export function assemblePackage(input, opts) {
+    void opts; // shadow 双算在调用方（旧 assemblePointer + 本函数各调一次）；两态渲染同构
+    const dropped = [];
+    const all = [...input.direct, ...input.neighborhood, ...input.weak];
+    if (all.length === 0)
+        return null;
+    if (isTrivialQuery(input.query, input.queryTokens) && !hasEntityOverlap(input.queryTokens, all))
+        return null;
+    // 跨层去重（spec §2.6）：直接层赢席；重合的邻域条目并 why（边来源并入
+    // 直接层条目）不重占席；弱信号重合直接丢弃。
+    const direct = rankStable(input.direct, input.usageBySlug).map((c) => ({ ...c }));
+    const seen = new Set(direct.map((c) => c.slug));
+    const neighborhood = [];
+    for (const c of rankStable(input.neighborhood, input.usageBySlug)) {
+        if (seen.has(c.slug)) {
+            const host = direct.find((d) => d.slug === c.slug);
+            if (host !== undefined && host.via === undefined && c.via !== undefined)
+                host.via = c.via;
+            dropped.push({ slug: c.slug, reason: 'duplicate' });
+            continue;
+        }
+        seen.add(c.slug);
+        neighborhood.push(c);
+    }
+    const weak = [];
+    for (const c of rankStable(input.weak, input.usageBySlug)) {
+        if (seen.has(c.slug)) {
+            dropped.push({ slug: c.slug, reason: 'duplicate' });
+            continue;
+        }
+        seen.add(c.slug);
+        weak.push(c);
+    }
+    // Reserve with an upper-bound header (pre-cap counts); the final header is
+    // never longer, so the 600 cap holds after the swap.
+    let used = estimateTokens(`${PACKAGE_WRAPPER_OPEN}\n${packageHeader(input.query, direct.length, neighborhood.length, weak.length)}\n\n${PACKAGE_WRAPPER_CLOSE}`);
+    const layers = { direct: [], neighborhood: [], weak: [] };
+    const directBlocks = [];
+    for (const c of direct) {
+        if (directBlocks.length >= PACKAGE_DIRECT_MAX && !isEntityAnchored(input.queryTokens, c)) {
+            dropped.push({ slug: c.slug, reason: 'direct-cap' });
+            continue;
+        }
+        const budget = Math.min(PACKAGE_DIRECT_BUDGET, PACKAGE_TOTAL - used);
+        if (budget < 24) {
+            dropped.push({ slug: c.slug, reason: 'total-budget' });
+            continue;
+        }
+        const block = directEntryText(c, input.queryTokens, budget);
+        if (used + block.usedTokens > PACKAGE_TOTAL) {
+            dropped.push({ slug: c.slug, reason: 'total-budget' });
+            continue;
+        }
+        directBlocks.push(block.text);
+        layers.direct.push(c.slug);
+        used += block.usedTokens;
+    }
+    const neighBlocks = [];
+    for (const c of neighborhood) {
+        if (neighBlocks.length >= PACKAGE_NEIGHBOR_MAX) {
+            dropped.push({ slug: c.slug, reason: 'neighborhood-cap' });
+            continue;
+        }
+        const budget = Math.min(PACKAGE_NEIGHBOR_BUDGET, PACKAGE_TOTAL - used);
+        if (budget < 16) {
+            dropped.push({ slug: c.slug, reason: 'total-budget' });
+            continue;
+        }
+        const block = neighborhoodEntryText(c);
+        if (used + block.usedTokens > PACKAGE_TOTAL) {
+            dropped.push({ slug: c.slug, reason: 'total-budget' });
+            continue;
+        }
+        neighBlocks.push(block.text);
+        layers.neighborhood.push(c.slug);
+        used += block.usedTokens;
+    }
+    const weakTake = weak.slice(0, PACKAGE_WEAK_MAX);
+    for (const c of weak.slice(PACKAGE_WEAK_MAX))
+        dropped.push({ slug: c.slug, reason: 'weak-cap' });
+    let weakText = null;
+    if (weakTake.length > 0) {
+        const full = weakSignalLine(weakTake, input.queryTokens);
+        if (used + full.usedTokens <= PACKAGE_TOTAL) {
+            weakText = full.text;
+            layers.weak = full.slugs;
+            used += full.usedTokens;
+        }
+        else if (weakTake.length > 1) {
+            const single = weakSignalLine(weakTake.slice(0, 1), input.queryTokens);
+            if (used + single.usedTokens <= PACKAGE_TOTAL) {
+                weakText = single.text;
+                layers.weak = single.slugs;
+                used += single.usedTokens;
+                for (const slug of weakTake.slice(1).map((c) => c.slug))
+                    dropped.push({ slug, reason: 'total-budget' });
+            }
+            else {
+                for (const slug of full.slugs)
+                    dropped.push({ slug, reason: 'total-budget' });
+            }
+        }
+        else {
+            for (const slug of full.slugs)
+                dropped.push({ slug, reason: 'total-budget' });
+        }
+    }
+    const parts = [packageHeader(input.query, layers.direct.length, layers.neighborhood.length, layers.weak.length)];
+    if (directBlocks.length > 0)
+        parts.push(['### 直接相关', ...directBlocks].join('\n\n'));
+    if (neighBlocks.length > 0)
+        parts.push(['### 邻域', ...neighBlocks].join('\n'));
+    if (weakText !== null)
+        parts.push(weakText);
+    if (parts.length === 1)
+        return null; // everything budget-cut → 零命中零注入
+    const text = `${PACKAGE_WRAPPER_OPEN}\n${parts.join('\n\n')}\n${PACKAGE_WRAPPER_CLOSE}`;
+    return {
+        text,
+        usedTokens: estimateTokens(text),
+        included: [...layers.direct, ...layers.neighborhood, ...layers.weak],
+        dropped,
+        layers,
+    };
+}

package/lib/distill.d.ts

@@ -37,6 +37,8 @@ export interface DistillOp {
     depends?: string[];
     open_questions?: string[];
     impact?: string[];
+    /** Semantic-association targets (A2): existing slugs or same-batch op slugs. */
+    links?: string[];
     conclusion?: string;
     recommendations?: string;
     status?: string;
@@ -103,6 +105,27 @@ export declare class Distiller {
      * escape hatch belongs to the main pass).
      */
     private correctiveRetry;
+    /**
+     * The A6 §5.1 schema gate, applied to the parsed batch BEFORE any write:
+     * create ops must carry description + triggers 3-8 条 (the prompt's standing
+     * requirement, now enforced). Deficient creates get exactly ONE 补问 retry
+     * that re-asks for just those ops completed — budget-guarded exactly like
+     * the corrective ids retry (`retrySpent` reports the extra call). Every
+     * still-deficient op is dropped and comes back as a SchemaDrop carrying the
+     * ledger kind: `distill-op-invalid` after a used-but-unsuccessful retry,
+     * `distill-op-retry-exhausted` when no retry could run or the retry call
+     * itself failed. Dropped ops' observations stay unconsumed — the existing
+     * GC/corrective path accounts for them, never a silent loss. Update ops
+     * keep their narrower contract (slug only).
+     */
+    private enforceSchema;
+    /**
+     * The one schema 补问: re-ask for exactly the deficient creates, completed.
+     * Returns key→op for what the model re-emitted (matched by slugified title),
+     * or undefined when the call itself failed / output was unparseable — the
+     * caller then ledgeres `distill-op-retry-exhausted` for every deficient op.
+     */
+    private schemaRetryCall;
     /**
      * Execute a batch's ops with observed_ids sanitized upfront. A scalar
      * observed_ids string (the commonest model shape drift) is normalized to a
@@ -114,8 +137,24 @@ export declare class Distiller {
      * failures (missing fields, unknown update slug) keep the per-op try/catch
      * isolation — one bad op never sinks the batch, and a stall they cause is
      * NOT attributed to observed_ids (no corrective retry for those).
+     *
+     * A2 links: every link target is whitelist-checked against `knownSlugs`
+     * (existing slugs ∪ same-batch op slugs); rejected refs are stripped and
+     * ledgered (`link-ref-rejected`), survivors ride the op's topic write into
+     * the store transaction (先落 topic 再建边 — one write, no separate edge
+     * pass). The set is locally seeded with this call's predicted op slugs and
+     * grows as creates land, so same-batch references resolve in op order.
      */
     private applyOps;
+    /**
+     * A2 links whitelist: a link survives only when its normalized target is a
+     * slug that already exists or belongs to this batch (`knownSlugs`), and not
+     * the op's own topic. Everything else is stripped and ledgered
+     * (`link-ref-rejected` — a dead edge must at least leave a trace). Returns
+     * the surviving targets (undefined when none — no Links section is written)
+     * plus the rejection count for the run detail.
+     */
+    private acceptLinks;
 }
 /** Extract the first balanced JSON object from model output (tolerates fences). */
 export declare function parseOps(raw: string): DistillOp[];