@theseus-whale/dsh-continuity 0.0.0-stage → 4.12.2

package/bin/summary-index.mjs

@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+/**
+ * Build the middle-layer summary index from one raw session.
+ *
+ * Phase 0/1:
+ *   - imports existing compaction/summary records (native checkpoint + gist)
+ *   - plans missing windows by seq/token gaps
+ *   - dry-run by default; --apply writes summaries/index.json
+ *
+ * Usage:
+ *   node bin/summary-index.mjs --raw <session.v4.jsonl.zstd> [--out <index.json>] [--json]
+ *   node bin/summary-index.mjs --raw ... --apply [--session-dir <dir>]
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import zlib from 'node:zlib';
+import { buildSummaryIndex, summaryIndexPath } from '../src/memory/summary-index.mjs';
+import { buildWindowSummary } from '../src/memory/window-summary.mjs';
+
+function arg(name, fallback = null) {
+  const i = process.argv.indexOf(`--${name}`);
+  return i >= 0 ? process.argv[i + 1] ?? fallback : fallback;
+}
+
+function readRawLines(file) {
+  if (!/\.zstd$/i.test(file)) {
+    return fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, '').split(/\r?\n/).filter(Boolean);
+  }
+  const buf = fs.readFileSync(file);
+  const magic = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]);
+  const starts = [];
+  for (let at = 0; ;) {
+    const found = buf.indexOf(magic, at);
+    if (found < 0) break;
+    starts.push(found);
+    at = found + magic.length;
+  }
+  const lines = [];
+  for (const start of starts) {
+    let out;
+    try { out = zlib.zstdDecompressSync(buf.subarray(start)); } catch { continue; }
+    lines.push(...out.toString('utf8').split(/\r?\n/).filter(Boolean));
+  }
+  return lines;
+}
+
+const raw = arg('raw');
+if (!raw) {
+  console.error('用法: node bin/summary-index.mjs --raw <session.v4.jsonl[.zstd]> [--out <index.json>] [--apply] [--json]');
+  process.exit(2);
+}
+if (!fs.existsSync(raw)) {
+  console.error(`raw not found: ${raw}`);
+  process.exit(2);
+}
+
+const sessionDirArg = arg('session-dir');
+const sessionDir = sessionDirArg ?? path.dirname(raw);
+const sessionId = arg('session-id', path.basename(sessionDir));
+const windowTokens = Math.max(1000, Number(arg('window-tokens', 40000)) || 40000);
+const outArg = arg('out');
+const out = outArg ?? (sessionDirArg ? summaryIndexPath(sessionDirArg) : null);
+const apply = process.argv.includes('--apply');
+const extractPending = process.argv.includes('--extract-pending');
+const printJson = process.argv.includes('--json');
+if (apply && !out) {
+  console.error('--apply requires explicit --session-dir (Continuity session dir) or --out');
+  process.exit(2);
+}
+
+const events = [];
+const seen = new Set();
+for (const line of readRawLines(raw)) {
+  let event;
+  try { event = JSON.parse(line.replace(/^\uFEFF/, '')); } catch { continue; }
+  const seq = Number(event?.seq);
+  if (Number.isInteger(seq)) {
+    if (seen.has(seq)) continue;
+    seen.add(seq);
+  }
+  events.push(event);
+}
+events.sort((a, b) => Number(a.seq) - Number(b.seq));
+
+const bootstrapFile = path.join(sessionDir, 'summaries', 'bootstrap-ranges.json');
+let bootstrapRanges = [];
+try {
+  if (fs.existsSync(bootstrapFile)) {
+    const parsed = JSON.parse(fs.readFileSync(bootstrapFile, 'utf8'));
+    if (Array.isArray(parsed?.ranges)) bootstrapRanges = parsed.ranges;
+  }
+} catch { /* malformed sidecar is ignored; index still builds */ }
+
+const baseIndex = buildSummaryIndex({ sessionId, events, windowTokens, bootstrapRanges });
+const windowSummaries = extractPending
+  ? baseIndex.pendingWindows.map((pending) => buildWindowSummary({
+    events,
+    fromSeq: pending.fromSeq,
+    toSeq: pending.toSeq,
+    maxTokens: 800,
+  }))
+  : [];
+const windowRanges = windowSummaries.map((summary) => ({
+  fromSeq: summary.fromSeq,
+  toSeq: summary.toSeq,
+  tokens: summary.approxTokens,
+  hash: summary.hash,
+  at: new Date().toISOString(),
+}));
+const index = windowSummaries.length > 0
+  ? buildSummaryIndex({ sessionId, events, windowTokens, bootstrapRanges, windowRanges, windowRecords: windowSummaries })
+  : baseIndex;
+if (apply && extractPending) {
+  const windowDir = path.join(path.dirname(out), 'windows');
+  fs.mkdirSync(windowDir, { recursive: true });
+  for (const summary of windowSummaries) {
+    fs.writeFileSync(
+      path.join(windowDir, `${summary.fromSeq}-${summary.toSeq}.json`),
+      JSON.stringify(summary, null, 2),
+      'utf8',
+    );
+  }
+}
+if (apply) {
+  fs.mkdirSync(path.dirname(out), { recursive: true });
+  const tmp = `${out}.tmp-${process.pid}`;
+  fs.writeFileSync(tmp, JSON.stringify(index, null, 2), 'utf8');
+  fs.renameSync(tmp, out);
+}
+
+if (printJson) {
+  console.log(JSON.stringify(index, null, 2));
+} else {
+  console.log(JSON.stringify({
+    sessionId: index.sessionId,
+    raw: path.resolve(raw),
+    out: apply ? path.resolve(out) : null,
+    apply,
+    windowTokens: index.windowTokens,
+    maxSummarySeq: index.maxSummarySeq,
+    coveredThroughSeq: index.coveredThroughSeq,
+    latestSummaryAt: index.latestSummaryAt,
+    records: index.records.length,
+    checkpoints: index.records.filter((record) => record.kind === 'checkpoint').length,
+    gists: index.records.filter((record) => record.kind === 'gist').length,
+    pendingWindows: index.pendingWindows.length,
+    pending: index.pendingWindows.slice(0, 8),
+    extractedWindows: windowSummaries.length,
+    coverage: index.coverage,
+  }, null, 2));
+}

package/bin/summary-query.mjs

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+/**
+ * Read-only query over the Continuity summary index.
+ * This is a first step for recall's summaryHits; no model, no writes.
+ *
+ * Usage:
+ *   node bin/summary-query.mjs --index <index.json> --query "..." [--limit 5] [--json]
+ *   node bin/summary-query.mjs --session-dir <dir> --query "..."
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { summaryIndexPath } from '../src/memory/summary-index.mjs';
+
+function arg(name, fallback = null) {
+  const i = process.argv.indexOf(`--${name}`);
+  return i >= 0 ? process.argv[i + 1] ?? fallback : fallback;
+}
+
+const sessionDir = arg('session-dir');
+const indexFile = arg('index', sessionDir ? summaryIndexPath(sessionDir) : null);
+const query = String(arg('query', '')).trim();
+const limit = Math.min(Math.max(Number(arg('limit', 5)) || 5, 1), 50);
+const json = process.argv.includes('--json');
+if (!indexFile || !fs.existsSync(indexFile)) {
+  console.error('summary index not found; pass --index or --session-dir');
+  process.exit(2);
+}
+if (!query) {
+  console.error('query required');
+  process.exit(2);
+}
+
+const index = JSON.parse(fs.readFileSync(indexFile, 'utf8').replace(/^\uFEFF/, ''));
+const q = query.normalize('NFKC').toLowerCase();
+const qChars = [...q];
+const cjkQuery = /[\u3400-\u9fff]/.test(q);
+const records = Array.isArray(index.records) ? index.records : [];
+const scored = [];
+for (const record of records) {
+  const text = String(record.text ?? '').normalize('NFKC').toLowerCase();
+  if (!text) continue;
+  let score = 0;
+  const exact = text.includes(q);
+  if (exact) score += 100;
+  if (cjkQuery && qChars.length >= 2) {
+    const covered = qChars.filter((ch) => text.includes(ch)).length;
+    if (covered === qChars.length) score += 5;
+  }
+  if (score > 0) scored.push({ record, score, exact });
+}
+scored.sort((a, b) => b.score - a.score || b.record.toSeq - a.record.toSeq);
+const hits = scored.slice(0, limit).map(({ record, score, exact }) => ({
+  id: record.id,
+  kind: record.kind,
+  fromSeq: record.fromSeq,
+  toSeq: record.toSeq,
+  fromTurn: record.fromTurn,
+  toTurn: record.toTurn,
+  threads: record.threads,
+  approxTokens: record.approxTokens,
+  score: Number(score.toFixed(2)),
+  exact,
+  excerpt: String(record.text ?? '').slice(0, 500),
+}));
+const result = {
+  index: path.resolve(indexFile),
+  query,
+  total: scored.length,
+  shown: hits.length,
+  hits,
+};
+if (json) console.log(JSON.stringify(result, null, 2));
+else {
+  console.log(`query=${query} total=${result.total} shown=${result.shown}`);
+  for (const hit of hits) {
+    console.log(`\n[${hit.kind}] ${hit.id} · turn ${hit.fromTurn ?? '?'}-${hit.toTurn ?? '?'} · seq ${hit.fromSeq}-${hit.toSeq} · score ${hit.score}`);
+    console.log(hit.excerpt.replace(/\s+/g, ' ').slice(0, 300));
+  }
+}

package/bin/vec-check.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+/**
+ * Check whether the local embedding runtime produces a usable vector.
+ *
+ * Run from the installed package:
+ *   node bin/vec-check.mjs
+ */
+import { embedTexts } from '../src/facets/embedding.mjs';
+
+try {
+  const [vector] = await embedTexts(['连续性记忆审计自检']);
+  const dim = Array.isArray(vector) ? vector.length : 0;
+  const finite = Array.isArray(vector) && vector.every((value) => Number.isFinite(value));
+  console.log(JSON.stringify({
+    ok: dim > 0 && finite,
+    dimensions: dim,
+    first3: Array.isArray(vector) ? vector.slice(0, 3) : [],
+  }, null, 2));
+  if (dim === 0 || !finite) process.exit(1);
+} catch (error) {
+  console.error(JSON.stringify({
+    ok: false,
+    error: String(error?.message ?? error),
+    hint: 'run pnpm approve-builds, then pnpm rebuild onnxruntime-node sharp in the DSH profile',
+  }, null, 2));
+  process.exit(1);
+}

package/bin/zh_ner_error_audit.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+import json
+import sys
+from collections import Counter
+
+src, out = sys.argv[1], sys.argv[2]
+data = json.load(open(src, encoding="utf-8"))
+rows = data["rows"]
+gold_count = Counter()
+pred_count = Counter()
+fp = []
+fn = []
+for row in rows:
+    gold = {(g["entity"], g["label"]) for g in row["gold"]}
+    pred = {(p["text"].replace(" ", ""), p["label"]) for p in row["pred"]}
+    for item in gold:
+        gold_count[item[1]] += 1
+    for item in pred:
+        pred_count[item[1]] += 1
+    for item in pred - gold:
+        fp.append({"text": item[0], "label": item[1], "sentence": row["text"]})
+    for item in gold - pred:
+        fn.append({"text": item[0], "label": item[1], "sentence": row["text"]})
+
+summary = {
+    "n": data["n"],
+    "score": data["score"],
+    "gold_by_label": dict(gold_count),
+    "pred_by_label": dict(pred_count),
+    "fp_by_label": dict(Counter(x["label"] for x in fp)),
+    "fn_by_label": dict(Counter(x["label"] for x in fn)),
+    "fp_samples": fp[:20],
+    "fn_samples": fn[:20],
+}
+json.dump(summary, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
+print(json.dumps({k: v for k, v in summary.items() if not k.endswith("samples")}, ensure_ascii=False, indent=2))

package/bin/zh_ner_slot_eval.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+import json
+import sys
+
+MAP = {
+    "name": "PERSON",
+    "address": "PLACE",
+    "scene": "PLACE",
+    "company": "ORG",
+    "organization": "ORG",
+    "government": "ORG",
+    "book": "WORK",
+    "movie": "WORK",
+    "game": "WORK",
+    "position": "ROLE",
+}
+
+
+def coarse(label):
+    return MAP.get(label, label)
+
+
+data = json.load(open(sys.argv[1], encoding="utf-8"))
+tp = fp = fn = 0
+for row in data["rows"]:
+    gold = {(g["entity"], coarse(g["label"])) for g in row["gold"]}
+    pred = {(p["text"].replace(" ", ""), coarse(p["label"])) for p in row["pred"]}
+    tp += len(gold & pred)
+    fp += len(pred - gold)
+    fn += len(gold - pred)
+precision = 0 if tp + fp == 0 else tp / (tp + fp)
+recall = 0 if tp + fn == 0 else tp / (tp + fn)
+f1 = 0 if precision + recall == 0 else 2 * precision * recall / (precision + recall)
+print(json.dumps({
+    "n": data["n"],
+    "exact_label_f1": data["score"]["f1"],
+    "coarse_slot": {"tp": tp, "fp": fp, "fn": fn, "precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4)},
+}, ensure_ascii=False, indent=2))