@theseus-whale/dsh-continuity 0.0.0-stage → 4.12.2

package/src/memory/extraction-provider.mjs

@@ -0,0 +1,300 @@
+/**
+ * Extraction provider: default off, cache-backed first.
+ * Model-backed command mode is added in the next stage.
+ */
+import fs from 'node:fs';
+import crypto from 'node:crypto';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import readline from 'node:readline';
+import { attachStructuredExtraction } from './extraction-adapter.mjs';
+import { validateEventSlots } from './event-model.mjs';
+import { createCompositeProvider } from './extraction-composite.mjs';
+
+export function resolveExtractionMode(env = process.env) {
+  const raw = String(env.DSH_EXTRACTION_MODE ?? 'off').toLowerCase();
+  if (raw === 'cache' || raw === 'model') return raw;
+  return 'off';
+}
+
+export function splitArgs(value) {
+  const text = String(value ?? '').trim();
+  if (!text) return [];
+  if (text.startsWith('[')) {
+    try {
+      const parsed = JSON.parse(text);
+      return Array.isArray(parsed) ? parsed.map(String) : [];
+    } catch {
+      return [];
+    }
+  }
+  const out = [];
+  let current = '';
+  let quote = null;
+  for (const char of text) {
+    if (quote) {
+      if (char === quote) quote = null;
+      else current += char;
+    } else if (char === '"' || char === "'") {
+      quote = char;
+    } else if (/\s/.test(char)) {
+      if (current) {
+        out.push(current);
+        current = '';
+      }
+    } else {
+      current += char;
+    }
+  }
+  if (current) out.push(current);
+  return out;
+}
+
+function keyFor(text) {
+  return crypto.createHash('sha256').update(String(text ?? ''), 'utf8').digest('hex');
+}
+
+function readCache(cachePath) {
+  if (!cachePath || !fs.existsSync(cachePath)) return {};
+  try {
+    const parsed = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
+    return parsed && typeof parsed === 'object' ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+export function createCacheProvider({ cachePath = null, mode = 'cache' } = {}) {
+  const cache = mode === 'cache' ? readCache(cachePath) : {};
+  return {
+    mode,
+    async extract({ text } = {}) {
+      const started = Date.now();
+      if (mode === 'off') return { ok: false, source: 'off', reason: 'disabled', latencyMs: 0 };
+      const key = keyFor(text);
+      const hit = cache[key];
+      if (!hit) return { ok: false, source: 'cache', reason: 'cache-miss', latencyMs: Date.now() - started };
+      if (hit.slots) {
+        const checked = validateEventSlots(hit.slots);
+        if (!checked.ok) return { ok: false, source: 'cache', reason: 'invalid-slots', errors: checked.errors, latencyMs: Date.now() - started };
+        return { ok: true, source: 'cache', data: { ...hit, slots: checked.slots }, latencyMs: Date.now() - started };
+      }
+      return { ok: true, source: 'cache', data: hit, latencyMs: Date.now() - started };
+    },
+  };
+}
+
+function runCommand({ command, args = [], payload, timeoutMs = 8000, env = process.env }) {
+  return new Promise((resolve) => {
+    const child = spawn(command, args, { env, stdio: ['pipe', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    let settled = false;
+    const finish = (value) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(value);
+    };
+    const timer = setTimeout(() => {
+      child.kill();
+      finish({ ok: false, reason: 'timeout' });
+    }, timeoutMs);
+    child.stdout.on('data', (chunk) => { stdout += chunk; });
+    child.stderr.on('data', (chunk) => { stderr += chunk; });
+    child.on('error', (error) => finish({ ok: false, reason: 'spawn-error', error: error.message }));
+    child.on('close', (code) => {
+      if (code !== 0) return finish({ ok: false, reason: 'exit-code', code, stderr: stderr.slice(-500) });
+      const lines = stdout.trim().split(/\r?\n/).filter(Boolean);
+      try {
+        return finish({ ok: true, value: JSON.parse(lines.at(-1)) });
+      } catch {
+        return finish({ ok: false, reason: 'invalid-json', stderr: stderr.slice(-500) });
+      }
+    });
+    child.stdin.end(JSON.stringify(payload));
+  });
+}
+
+export function createCommandProvider({ command, args = [], timeoutMs = 8000, env = process.env } = {}) {
+  const invoke = async (payload) => {
+    const started = Date.now();
+    if (!command) return { ok: false, source: 'command', reason: 'missing-command', latencyMs: 0 };
+    const run = await runCommand({ command, args, payload, timeoutMs, env });
+    if (!run.ok) return { ok: false, source: 'command', ...run, latencyMs: Date.now() - started };
+    const data = run.value ?? {};
+    if (data.ok === false) return { ok: false, source: 'command', reason: data.reason ?? 'service-error', latencyMs: Date.now() - started };
+    if (data.slots) {
+      const checked = validateEventSlots(data.slots);
+      if (!checked.ok) return { ok: false, source: 'command', reason: 'invalid-slots', errors: checked.errors, latencyMs: Date.now() - started };
+      return { ok: true, source: 'command', data: { ...data, slots: checked.slots }, latencyMs: Date.now() - started };
+    }
+    return { ok: true, source: 'command', data, latencyMs: Date.now() - started };
+  };
+  return {
+    mode: 'model',
+    extract: ({ text, ...rest } = {}) => invoke({ text, ...rest }),
+    healthCheck: () => invoke({ health: true }),
+  };
+}
+
+export function createPersistentCommandProvider({ command, args = [], timeoutMs = 8000, env = process.env } = {}) {
+  let child = null;
+  let reader = null;
+  let started = null;
+  let seq = 0;
+  const pending = new Map();
+
+  const settle = (id, value) => {
+    const entry = pending.get(id);
+    if (!entry) return;
+    pending.delete(id);
+    clearTimeout(entry.timer);
+    entry.resolve(value);
+  };
+
+  const rejectAll = (reason) => {
+    for (const [id, entry] of pending) {
+      clearTimeout(entry.timer);
+      entry.resolve({ ok: false, reason });
+      pending.delete(id);
+    }
+  };
+
+  const start = () => {
+    if (started) return started;
+    started = new Promise((resolve, reject) => {
+      if (!command) return reject(new Error('missing-command'));
+      child = spawn(command, args, { env, stdio: ['pipe', 'pipe', 'pipe'] });
+      child.once('error', (error) => {
+        reject(error);
+        rejectAll('spawn-error');
+      });
+      child.once('close', () => {
+        child = null;
+        reader?.close();
+        reader = null;
+        started = null;
+        rejectAll('process-exit');
+      });
+      reader = readline.createInterface({ input: child.stdout });
+      reader.on('line', (line) => {
+        try {
+          const data = JSON.parse(line);
+          if (data && data.id !== undefined) settle(data.id, { ok: true, data });
+        } catch {
+          /* ignore malformed lines; a timeout/crash will surface */
+        }
+      });
+      child.stderr.on('data', () => {});
+      resolve();
+    });
+    return started;
+  };
+
+  const request = async (payload) => {
+    if (!command) return { ok: false, reason: 'missing-command' };
+    try {
+      await start();
+    } catch (error) {
+      return { ok: false, reason: 'spawn-error', error: error.message };
+    }
+    if (!child) return { ok: false, reason: 'not-running' };
+    const id = ++seq;
+    const result = await new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        pending.delete(id);
+        resolve({ ok: false, reason: 'timeout' });
+      }, timeoutMs);
+      pending.set(id, { resolve, timer });
+      child.stdin.write(`${JSON.stringify({ id, ...payload })}\n`);
+    });
+    return result;
+  };
+
+  return {
+    mode: 'model',
+    persistent: true,
+    async extract({ text, ...rest } = {}) {
+      const startedAt = Date.now();
+      const response = await request({ text, ...rest });
+      if (!response.ok) return { ok: false, source: 'persistent', reason: response.reason, latencyMs: Date.now() - startedAt };
+      const data = response.data ?? {};
+      if (data.ok === false) return { ok: false, source: 'persistent', reason: data.reason ?? 'service-error', latencyMs: Date.now() - startedAt };
+      if (data.slots) {
+        const checked = validateEventSlots(data.slots);
+        if (!checked.ok) return { ok: false, source: 'persistent', reason: 'invalid-slots', errors: checked.errors, latencyMs: Date.now() - startedAt };
+        return { ok: true, source: 'persistent', data: { ...data, slots: checked.slots }, latencyMs: Date.now() - startedAt };
+      }
+      return { ok: true, source: 'persistent', data, latencyMs: Date.now() - startedAt };
+    },
+    healthCheck: () => request({ health: true }),
+    close() {
+      if (child) child.kill();
+      child = null;
+      reader?.close();
+      reader = null;
+      started = null;
+      rejectAll('closed');
+    },
+  };
+}
+
+export function createExtractionProviderFromEnv(env = process.env) {
+  const mode = resolveExtractionMode(env);
+  if (mode === 'off') return createCacheProvider({ mode: 'off' });
+  if (mode === 'cache') return createCacheProvider({ cachePath: env.DSH_EXTRACTION_CACHE ?? null, mode: 'cache' });
+  if (env.DSH_EXTRACTION_CONFIG) {
+    const configPath = path.resolve(env.DSH_EXTRACTION_CONFIG);
+    const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    const persistent = config.persistent !== false && env.DSH_EXTRACTION_PERSISTENT !== '0';
+    const factory = persistent ? createPersistentCommandProvider : createCommandProvider;
+    const service = (spec = {}) => factory({
+      command: spec.command ?? null,
+      args: Array.isArray(spec.args) ? spec.args.map(String) : [],
+      timeoutMs: Number(spec.timeoutMs ?? env.DSH_EXTRACTION_TIMEOUT_MS ?? 8000),
+      env: { ...env, ...(spec.env ?? {}) },
+    });
+    const ner = service(config.services?.ner);
+    return createCompositeProvider({
+      ner,
+      coref: config.services?.coref ? service(config.services.coref) : null,
+      srl: config.services?.srl ? service(config.services.srl) : null,
+      el: config.services?.el ? service(config.services.el) : null,
+    });
+  }
+  const factory = env.DSH_EXTRACTION_PERSISTENT === '1' ? createPersistentCommandProvider : createCommandProvider;
+  return factory({
+    command: env.DSH_EXTRACTION_COMMAND ?? null,
+    args: splitArgs(env.DSH_EXTRACTION_ARGS),
+    timeoutMs: Number(env.DSH_EXTRACTION_TIMEOUT_MS ?? 8000),
+    env,
+  });
+}
+
+export async function stepWithProvider(controller, provider, {
+  text,
+  role = 'user',
+  facets = {},
+  confidence = 1,
+  nowTurn = 0,
+} = {}) {
+  const extraction = await provider.extract({ text });
+  if (!extraction.ok) {
+    const gate = controller.step({ text, role, facets, confidence, eventSlots: null, nowTurn });
+    return { ok: false, extraction, gate };
+  }
+  const data = extraction.data ?? {};
+  const result = attachStructuredExtraction(controller, {
+    text,
+    role,
+    facets,
+    confidence,
+    nowTurn,
+    slots: data.slots ?? null,
+    srl: data.srl ?? null,
+    corefClusters: data.corefClusters ?? [],
+    entityLinks: data.entityLinks ?? [],
+  });
+  return { ok: result.ok, extraction, ...result };
+}

package/src/memory/extraction-score.mjs

@@ -0,0 +1,36 @@
+/**
+ * Entity/slot scoring for extraction benchmarks.
+ * Exact span text + label match; no fuzzy matching.
+ */
+export function scoreEntities(predicted = [], expected = []) {
+  const key = (entity) => `${entity.label.toUpperCase()}\u0000${entity.text}`;
+  const predictedKeys = new Set(predicted.map(key));
+  const expectedKeys = new Set(expected.map(key));
+  let tp = 0;
+  for (const item of predictedKeys) if (expectedKeys.has(item)) tp += 1;
+  const fp = predictedKeys.size - tp;
+  const fn = expectedKeys.size - tp;
+  const precision = predictedKeys.size === 0 ? 0 : tp / predictedKeys.size;
+  const recall = expectedKeys.size === 0 ? 0 : tp / expectedKeys.size;
+  const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
+  return {
+    tp,
+    fp,
+    fn,
+    precision: +precision.toFixed(4),
+    recall: +recall.toFixed(4),
+    f1: +f1.toFixed(4),
+  };
+}
+
+export function aggregateScores(scores) {
+  const total = scores.reduce((acc, score) => ({
+    tp: acc.tp + score.tp,
+    fp: acc.fp + score.fp,
+    fn: acc.fn + score.fn,
+  }), { tp: 0, fp: 0, fn: 0 });
+  const precision = total.tp + total.fp === 0 ? 0 : total.tp / (total.tp + total.fp);
+  const recall = total.tp + total.fn === 0 ? 0 : total.tp / (total.tp + total.fn);
+  const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
+  return { ...total, precision: +precision.toFixed(4), recall: +recall.toFixed(4), f1: +f1.toFixed(4) };
+}

package/src/memory/facet-stabilizer.mjs

@@ -0,0 +1,59 @@
+/**
+ * Stabilize a noisy local facet head with lexical aliases and retrieval votes.
+ * This is a deterministic system layer, not an LLM decision.
+ */
+import { DEFAULT_ALIASES } from '../facets/schema.mjs';
+
+const START_RUN = /开始(?:跑团|一局|团|游戏|新的)|开个新团|新团|开新团|来一局|开团|开一局|进入.*(?:跑团|灯室|游戏)|跑团吧|TRPG/iu;
+
+export function lexicalFacet(text, { startOnly = false } = {}) {
+  const keys = Object.keys(DEFAULT_ALIASES).sort((a, b) => b.length - a.length);
+  const alias = keys.find((key) => text.includes(key));
+  const start = START_RUN.test(text);
+  if (!start && startOnly) return null;
+  if (start) return { activity: '跑团', source: 'start-marker' };
+  if (alias !== undefined) return { ...DEFAULT_ALIASES[alias], source: `alias:${alias}` };
+  return null;
+}
+
+export function voteFacets(records, hits) {
+  const activity = {};
+  const topic = {};
+  for (const hit of hits) {
+    const record = records.find((item) => item.turn === hit.turn);
+    if (record === undefined) continue;
+    activity[record.facets.activity] = (activity[record.facets.activity] ?? 0) + 1;
+    topic[record.facets.topic] = (topic[record.facets.topic] ?? 0) + 1;
+  }
+  const top = (counts) => Object.entries(counts).sort((a, b) => b[1] - a[1])[0] ?? null;
+  return { activity: top(activity), topic: top(topic) };
+}
+
+export function stabilizeFacets({ raw, text, records, keyword = [], semantic = [], summary = [] }) {
+  const facets = { ...raw.facets };
+  const confidence = { ...raw.confidence };
+  const evidence = voteFacets(records, [...keyword, ...semantic]);
+  const summaryEvidence = voteFacets(records, summary);
+  const summaryAgrees = evidence.activity !== null
+    && summaryEvidence.activity !== null
+    && evidence.activity[0] === summaryEvidence.activity[0];
+  if (summaryAgrees) evidence.activity[1] += 1;
+  const lexical = lexicalFacet(text);
+
+  if (lexical !== null) {
+    for (const [slot, value] of Object.entries(lexical)) {
+      facets[slot] = value;
+      confidence[slot] = 0.9;
+    }
+    return { facets, confidence, evidence, summaryEvidence, summaryAgrees, reason: lexical.source };
+  }
+
+  if ((confidence.activity ?? 0) < 0.7 && evidence.activity !== null && evidence.activity[1] >= 3) {
+    facets.activity = evidence.activity[0];
+    confidence.activity = 0.65;
+  }
+  if ((confidence.topic ?? 0) < 0.6 && evidence.topic !== null && evidence.topic[1] >= 3) {
+    facets.topic = evidence.topic[0];
+  }
+  return { facets, confidence, evidence, summaryEvidence, summaryAgrees, reason: 'retrieval' };
+}

package/src/memory/history-map.mjs

@@ -0,0 +1,139 @@
+/**
+ * Active history map: the middle-layer summary that must live in context.
+ *
+ * This renderer is deliberately deterministic: it uses the archive summary
+ * index plus the newest raw summary events, and caps by characters/tokens.
+ * A later LLM summarizer can replace the body; the carrier contract stays.
+ */
+import { createHash } from 'node:crypto';
+
+const DEFAULT_MAX_TOKENS = 4000;
+const CHARS_PER_TOKEN = 4;
+const DETAIL_CHARS = 6000;
+
+function textOfSummary(event) {
+  const summary = event?.data?.summary;
+  if (Array.isArray(summary)) {
+    return summary.map((block) => (block && typeof block === 'object' && typeof block.text === 'string' ? block.text : '')).join('');
+  }
+  return typeof summary === 'string' ? summary : '';
+}
+
+function seqOf(event) {
+  const seq = Number(event?.seq);
+  return Number.isInteger(seq) ? seq : null;
+}
+
+function rangeOf(event) {
+  const start = Number(event?.data?.shadowedRange?.start);
+  const end = Number(event?.data?.shadowedRange?.end);
+  if (!Number.isInteger(start) || !Number.isInteger(end)) return null;
+  return { from: Math.min(start, end), to: Math.max(start, end) };
+}
+
+function mergeRanges(records) {
+  const ranges = records
+    .map((record) => ({ from: Number(record.fromSeq), to: Number(record.toSeq) }))
+    .filter((range) => Number.isInteger(range.from) && Number.isInteger(range.to))
+    .sort((a, b) => a.from - b.from || a.to - b.to);
+  const merged = [];
+  for (const range of ranges) {
+    const last = merged.at(-1);
+    if (last && range.from <= last.to + 1) last.to = Math.max(last.to, range.to);
+    else merged.push({ ...range });
+  }
+  return merged;
+}
+
+function excerpt(text, maxChars = DETAIL_CHARS) {
+  const clean = String(text ?? '').replace(/\r/g, '').trim().replace(/^## /gm, '### ');
+  if (clean.length <= maxChars) return clean;
+  return clean.slice(0, maxChars).replace(/\s+\S*$/, '') + '…';
+}
+
+/**
+ * @param {{index?: object, events?: object[], maxTokens?: number, generatedAt?: string}} input
+ */
+export function buildHistoryMap({ index = null, events = [], maxTokens = DEFAULT_MAX_TOKENS, generatedAt = null } = {}) {
+  const capChars = Math.max(1000, Math.floor(maxTokens * CHARS_PER_TOKEN));
+  const records = Array.isArray(index?.records) ? index.records : [];
+  const ranges = mergeRanges(records);
+  const rawSummaries = (Array.isArray(events) ? events : [])
+    .filter((event) => event?.type === 'compaction/summary')
+    .map((event) => ({ seq: seqOf(event), range: rangeOf(event), text: textOfSummary(event), event }))
+    .filter((item) => item.range && item.text.trim().length > 0)
+    .sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0));
+  const indexedMax = Number(index?.maxSummarySeq);
+  const latestRaw = rawSummaries.at(-1) ?? null;
+  const extra = rawSummaries.filter((item) => !Number.isInteger(indexedMax) || (item.seq ?? -1) > indexedMax);
+
+  const sections = [];
+  const push = (heading, body) => sections.push({ heading, body: String(body ?? '').trim() });
+  push(
+    '位置',
+    [
+      `session=${index?.sessionId ?? ''}`,
+      `coverage=${index?.coverage?.minSeq ?? '?'}..${index?.coverage?.maxSeq ?? '?'}`,
+      `index.maxSummarySeq=${indexedMax || '?'}`,
+      `latestRawSummary=${latestRaw?.seq ?? '?'}`,
+      `depth=1`,
+      `generatedAt=${generatedAt ?? index?.generatedAt ?? 'unknown'}`,
+    ].join(' · '),
+  );
+  const recentRanges = ranges.slice(-3).reverse();
+  for (const range of recentRanges) {
+    const candidates = records
+      .filter((record) => Number(record.toSeq) >= range.from && Number(record.fromSeq) <= range.to)
+      .sort((a, b) => Number(b.toSeq) - Number(a.toSeq)
+        || String(b.text ?? '').length - String(a.text ?? '').length);
+    const picked = candidates.slice(0, 2);
+    if (picked.length > 0) {
+      const perRecord = Math.max(1500, Math.floor(DETAIL_CHARS / picked.length));
+      push(
+        `最近区间 ${range.from}-${range.to}`,
+        picked.map((record) => excerpt(record.text, perRecord)).join('\n\n---\n\n'),
+      );
+    }
+  }
+  if (extra.length > 0) {
+    push(
+      '上次索引之后的 fold',
+      extra.slice(-2).map((item) => `- seq ${item.range.from}-${item.range.to}: ${excerpt(item.text, 1200)}`).join('\n'),
+    );
+  }
+  push(
+    '长期结构',
+    ranges.length === 0
+      ? '(no archive ranges)'
+      : ranges.map((range) => `- seq ${range.from}-${range.to}`).join('\n'),
+  );
+  push('取原文', '细节以 raw session_log 为准；本图只指路，不是事实本身。');
+
+  // Cap by sections, dropping detail from the middle first; keep position/source.
+  let text = '';
+  const rendered = [];
+  for (const section of sections) {
+    const block = `## ${section.heading}\n${section.body}`;
+    if (text.length + block.length + 2 > capChars && rendered.length > 0) {
+      if (section.heading === '取原文') {
+        text += `\n\n## ${section.heading}\n${section.body}`;
+      }
+      continue;
+    }
+    rendered.push(block);
+    text += (text ? '\n\n' : '') + block;
+  }
+  if (text.length > capChars) text = text.slice(0, capChars).replace(/\s+\S*$/, '') + '…';
+  const hash = createHash('sha256').update(text, 'utf8').digest('hex').slice(0, 16);
+  return {
+    text: `[continuity:history-map] ${text}`,
+    hash,
+    maxTokens,
+    chars: text.length,
+    approxTokens: Math.ceil(text.length / CHARS_PER_TOKEN),
+    maxSummarySeq: Number.isInteger(indexedMax) ? indexedMax : null,
+    coveredThroughSeq: Number.isInteger(Number(index?.coveredThroughSeq)) ? Number(index.coveredThroughSeq) : null,
+    latestRawSummarySeq: latestRaw?.seq ?? null,
+    extraFolds: extra.length,
+  };
+}

package/src/memory/language-router.mjs

@@ -0,0 +1,30 @@
+/**
+ * Language router for zh / en / mixed messages.
+ * Not a language detector replacement; deterministic and explainable.
+ */
+export function routeLanguage(text) {
+  const source = String(text ?? '');
+  const cjk = (source.match(/[\u4e00-\u9fff]/g) ?? []).length;
+  const latin = (source.match(/[A-Za-z]/g) ?? []).length;
+  const total = cjk + latin;
+  if (total === 0) return { language: 'unknown', confidence: 0, cjk, latin };
+  const zhRatio = cjk / total;
+  const enRatio = latin / total;
+  let language = 'mixed';
+  if (cjk > 0 && latin > 0 && Math.min(zhRatio, enRatio) >= 0.1) language = 'mixed';
+  else if (zhRatio >= 0.85) language = 'zh';
+  else if (enRatio >= 0.85) language = 'en';
+  return {
+    language,
+    confidence: +Math.max(zhRatio, enRatio).toFixed(2),
+    zhRatio: +zhRatio.toFixed(2),
+    enRatio: +enRatio.toFixed(2),
+    cjk,
+    latin,
+  };
+}
+
+export function languageTag(text) {
+  const route = routeLanguage(text);
+  return `[${route.language}] ${text}`;
+}