@theseus-whale/dsh-continuity 0.0.0-stage → 4.12.2

package/src/memory/bm25.mjs

@@ -0,0 +1,112 @@
+/**
+ * BM25 index over the immutable event log.
+ */
+import { tokenize } from './tokenize.mjs';
+
+export class BM25Index {
+  constructor({ k1 = 1.5, b = 0.75 } = {}) {
+    this.k1 = k1;
+    this.b = b;
+    this.docs = [];
+    this.postings = new Map();
+    this.docLengths = [];
+    this.avgLength = 0;
+  }
+
+  build(records) {
+    this.docs = records.map((record) => ({
+      turn: record.turn,
+      seqStart: record.seqStart,
+      seqEnd: record.seqEnd,
+    }));
+    this.postings = new Map();
+    this.docLengths = [];
+    let total = 0;
+    records.forEach((record, index) => {
+      const tokens = tokenize(record.text);
+      this.docLengths[index] = tokens.length;
+      total += tokens.length;
+      const tf = new Map();
+      for (const token of tokens) tf.set(token, (tf.get(token) ?? 0) + 1);
+      for (const [token, count] of tf) {
+        if (!this.postings.has(token)) this.postings.set(token, new Map());
+        this.postings.get(token).set(index, count);
+      }
+    });
+    this.avgLength = records.length === 0 ? 0 : total / records.length;
+    return this;
+  }
+
+  search(query, { topK = 20, minCoverage = 0 } = {}) {
+    const tokens = [...new Set(tokenize(query))];
+    if (tokens.length === 0) return [];
+    const scores = new Map();
+    const matched = new Map();
+    const n = this.docs.length;
+    for (const token of tokens) {
+      const posting = this.postings.get(token);
+      if (posting === undefined) continue;
+      const df = posting.size;
+      const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
+      for (const [index, tf] of posting) {
+        const dl = this.docLengths[index] || 1;
+        const denom = tf + this.k1 * (1 - this.b + this.b * (dl / (this.avgLength || 1)));
+        const score = idf * ((tf * (this.k1 + 1)) / denom);
+        scores.set(index, (scores.get(index) ?? 0) + score);
+        if (!matched.has(index)) matched.set(index, new Set());
+        matched.get(index).add(token);
+      }
+    }
+    return [...scores.entries()]
+      .map(([index, score]) => ({
+        ...this.docs[index],
+        score,
+        coverage: (matched.get(index)?.size ?? 0) / tokens.length,
+      }))
+      .filter((hit) => hit.coverage >= minCoverage)
+      .sort((a, b) => b.score - a.score)
+      .slice(0, topK);
+  }
+
+  toJSON() {
+    return {
+      k1: this.k1,
+      b: this.b,
+      docs: this.docs,
+      docLengths: this.docLengths,
+      avgLength: this.avgLength,
+      postings: [...this.postings.entries()].map(([token, posting]) => [token, [...posting.entries()]]),
+    };
+  }
+
+  static fromJSON(raw) {
+    const index = new BM25Index({ k1: raw.k1, b: raw.b });
+    index.docs = raw.docs;
+    index.docLengths = raw.docLengths;
+    index.avgLength = raw.avgLength;
+    index.postings = new Map(raw.postings.map(([token, entries]) => [token, new Map(entries)]));
+    return index;
+  }
+}
+
+export function mergeRanges(hits) {
+  const sorted = [...hits].sort((a, b) => a.seqStart - b.seqStart);
+  const ranges = [];
+  for (const hit of sorted) {
+    const last = ranges[ranges.length - 1];
+    if (last !== undefined && hit.seqStart <= last.seqEnd + 1) {
+      last.seqEnd = Math.max(last.seqEnd, hit.seqEnd);
+      last.turns += 1;
+      last.lastTurn = hit.turn;
+      continue;
+    }
+    ranges.push({
+      seqStart: hit.seqStart,
+      seqEnd: hit.seqEnd,
+      turns: 1,
+      lastTurn: hit.turn,
+      score: hit.score,
+    });
+  }
+  return ranges;
+}

package/src/memory/calibration.mjs

@@ -0,0 +1,54 @@
+/**
+ * Confidence calibration utilities for the local facet head.
+ * Labels here are weak T-derived labels, so this is provisional calibration,
+ * not human-gold calibration.
+ */
+
+export function reliabilityBins(points, { bins = 10 } = {}) {
+  const buckets = Array.from({ length: bins }, (_, index) => ({
+    lower: index / bins,
+    upper: (index + 1) / bins,
+    count: 0,
+    confidenceSum: 0,
+    correct: 0,
+  }));
+  for (const point of points) {
+    const confidence = Math.max(0, Math.min(1, point.confidence));
+    const index = Math.min(bins - 1, Math.floor(confidence * bins));
+    const bucket = buckets[index];
+    bucket.count += 1;
+    bucket.confidenceSum += confidence;
+    bucket.correct += point.correct ? 1 : 0;
+  }
+  return buckets.map((bucket) => ({
+    lower: bucket.lower,
+    upper: bucket.upper,
+    count: bucket.count,
+    confidence: bucket.count === 0 ? 0 : bucket.confidenceSum / bucket.count,
+    accuracy: bucket.count === 0 ? 0 : bucket.correct / bucket.count,
+  }));
+}
+
+export function thresholdCurve(points, { steps = 20 } = {}) {
+  const curve = [];
+  for (let index = 1; index <= steps; index += 1) {
+    const threshold = index / steps;
+    const accepted = points.filter((point) => point.confidence >= threshold);
+    const correct = accepted.filter((point) => point.correct).length;
+    curve.push({
+      threshold,
+      coverage: points.length === 0 ? 0 : accepted.length / points.length,
+      precision: accepted.length === 0 ? 0 : correct / accepted.length,
+      accepted: accepted.length,
+    });
+  }
+  return curve;
+}
+
+export function chooseThreshold(points, { targetPrecision = 0.9, minCoverage = 0.1 } = {}) {
+  const curve = thresholdCurve(points);
+  const eligible = curve
+    .filter((item) => item.precision >= targetPrecision && item.coverage >= minCoverage)
+    .sort((a, b) => b.coverage - a.coverage);
+  return eligible[0] ?? null;
+}

package/src/memory/chatter.mjs

@@ -0,0 +1,29 @@
+/** Shared chatter/register score (not a topic concept). */
+import { extractKeywords } from './bc-recall.mjs';
+
+const CJK = /[\u4e00-\u9fff]{2,6}/g;
+const FUNCTION = /[了的着吗呢吧啊]/g;
+
+export function buildDf(texts) {
+  const df = new Map();
+  for (const text of texts) {
+    const tokens = [...String(text ?? '').matchAll(CJK)].map((match) => match[0]);
+    for (const token of new Set(tokens)) df.set(token, (df.get(token) ?? 0) + 1);
+  }
+  return df;
+}
+
+export function chatterScore(text, df = new Map()) {
+  const source = String(text ?? '');
+  const tokens = [...source.matchAll(CJK)].map((match) => match[0]);
+  const proper = tokens.filter((token) => (df.get(token) ?? 0) <= 3).length;
+  const properDensity = tokens.length ? proper / tokens.length : 0;
+  const fnDensity = (source.match(FUNCTION) ?? []).length / Math.max(1, source.length);
+  const keywordSparse = extractKeywords(source, { limit: 10 }).length < 3 ? 1 : 0;
+  const quotes = (source.match(/[「」“”"——]/g) ?? []).length;
+  return 0.30 * (source.length < 60 ? 1 : 0)
+    + 0.25 * Math.min(1, properDensity)
+    + 0.20 * Math.min(1, fnDensity * 20)
+    + 0.15 * keywordSparse
+    + 0.10 * (quotes > 0 ? 1 : 0);
+}

package/src/memory/cmap-file.mjs

@@ -0,0 +1,107 @@
+/**
+ * C map file: user term -> B generic concepts (+ optional K / minB / summary).
+ *
+ * Old:
+ *   跑团 => 游戏 / 故事 / 对话
+ * New:
+ *   跑团 => B:游戏/故事/剧情/对话 | K:Coopanion/Coo | minB:2 | 摘要：...
+ *
+ * One line per term; `#` starts a comment.
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+export function defaultCMapPath(home = process.env.USERPROFILE ?? process.env.HOME ?? os.homedir()) {
+  return path.join(home, '.dsh', 'continuity', 'cmap.md');
+}
+
+function splitTerms(text) {
+  return String(text ?? '').split(/[\/,，、]/).map((word) => word.trim()).filter(Boolean);
+}
+
+function defaultParseError({ line, text, message }) {
+  try {
+    const home = process.env.USERPROFILE ?? process.env.HOME ?? os.homedir();
+    const file = path.join(home, '.dsh', 'continuity', 'gate-errors.jsonl');
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.appendFileSync(file, JSON.stringify({
+      at: new Date().toISOString(),
+      code: 'cmap-parse-failed',
+      line,
+      text: String(text ?? '').slice(0, 40),
+      message,
+    }) + '\n', 'utf8');
+  } catch { /* parse logging must not break loading */ }
+}
+
+export function parseCMapText(text, { onError = defaultParseError } = {}) {
+  const entries = [];
+  const lines = String(text ?? '').split(/\r?\n/);
+  for (let index = 0; index < lines.length; index += 1) {
+    const raw = lines[index];
+    const line = raw.replace(/#.*$/, '').trim();
+    if (!line) continue;
+    const match = line.match(/^(.+?)\s*(?:=>|->|→)\s*(.+)$/);
+    if (!match) {
+      onError?.({ line: index + 1, text: raw, message: 'missing =>' });
+      continue;
+    }
+    const term = match[1].trim();
+    if (!term) {
+      onError?.({ line: index + 1, text: raw, message: 'empty term' });
+      continue;
+    }
+
+    let body = match[2].trim();
+    let summary = '';
+    const summaryMarker = body.lastIndexOf('摘要：') >= 0 ? '摘要：' : '摘要:';
+    const summaryAt = body.lastIndexOf(summaryMarker);
+    if (summaryAt >= 0) {
+      summary = body.slice(summaryAt + summaryMarker.length).trim();
+      body = body.slice(0, summaryAt).trim();
+    }
+
+    let generic = [];
+    let keywords = [];
+    let minB = 0;
+    let limit = 20;
+    if (/(?:^|\|)\s*B\s*[:：]/i.test(body)) {
+      for (const segment of body.split(/\s*\|\s*/)) {
+        const item = segment.trim();
+        if (!item) continue;
+        const pair = item.match(/^([A-Za-z]+)\s*[:：]\s*(.*)$/);
+        if (!pair) continue;
+        const key = pair[1].toLowerCase();
+        const value = pair[2].trim();
+        if (key === 'b') generic = splitTerms(value);
+        else if (key === 'k') keywords = splitTerms(value);
+        else if (key === 'minb') {
+          const parsed = Number.parseInt(value, 10);
+          if (Number.isInteger(parsed) && parsed >= 0) minB = parsed;
+        } else if (key === 'limit') {
+          const parsed = Number.parseInt(value, 10);
+          if (Number.isInteger(parsed) && parsed > 0) limit = parsed;
+        }
+        // Unknown segments are ignored on purpose.
+      }
+    } else {
+      generic = splitTerms(body);
+    }
+
+    if (generic.length === 0) {
+      onError?.({ line: index + 1, text: raw, message: 'empty B list' });
+      continue;
+    }
+    entries.push({ term, generic, keywords, minB, limit, summary });
+  }
+  return entries;
+}
+
+export function loadCMapEntries(file = defaultCMapPath(), { onError = defaultParseError } = {}) {
+  try {
+    return parseCMapText(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''), { onError });
+  } catch {
+    return [];
+  }
+}

package/src/memory/concepts-file.mjs

@@ -0,0 +1,41 @@
+/**
+ * A-layer concepts (display only).
+ *
+ * Format: name | description | optional coverage | kind
+ * `kind` is `topic` (default) or `workflow`; workflow concepts are excluded
+ * from the default A browse but remain valid labels.
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+export function defaultConceptsPath(home = process.env.USERPROFILE ?? process.env.HOME ?? os.homedir()) {
+  return path.join(home, '.dsh', 'continuity', 'concepts.md');
+}
+
+export function parseConcepts(text) {
+  const rows = [];
+  for (const raw of String(text ?? '').split(/\r?\n/)) {
+    const line = raw.replace(/#.*$/, '').trim();
+    if (!line) continue;
+    const parts = line.split('|').map((part) => part.trim());
+    if (parts.length < 2 || !parts[0] || !parts[1]) continue;
+    const coverage = Number.parseInt(parts[2] ?? '', 10);
+    const rawKind = String(parts[3] ?? '').trim().toLowerCase().replace(/^kind\s*=\s*/, '');
+    rows.push({
+      label: parts[0],
+      description: parts[1],
+      coverage: Number.isInteger(coverage) ? coverage : null,
+      kind: rawKind === 'workflow' ? 'workflow' : 'topic',
+    });
+  }
+  return rows;
+}
+
+export function loadConcepts(file = defaultConceptsPath()) {
+  try {
+    return parseConcepts(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
+  } catch {
+    return [];
+  }
+}

package/src/memory/coref-slots.mjs

@@ -0,0 +1,40 @@
+/**
+ * Coreference output -> EventModel slots and referent candidates.
+ * Input clusters are expected to be normalizeCoref() output.
+ */
+
+function isLikelyName(text) {
+  return /^[A-Z][A-Za-z0-9'_-]{1,30}$/u.test(String(text ?? '').trim());
+}
+
+export function corefReferents(clusters = []) {
+  const referents = [];
+  for (const cluster of clusters) {
+    const canonical = cluster.canonical ?? cluster.mentions?.find((m) => m.isCanonical)?.text ?? null;
+    if (!canonical) continue;
+    referents.push({
+      clusterId: cluster.clusterId ?? null,
+      canonical,
+      mentions: (cluster.mentions ?? []).map((mention) => mention.text),
+    });
+  }
+  return referents;
+}
+
+export function applyCorefToSlots(slots = {}, clusters = []) {
+  const next = {
+    characters: [...(slots.characters ?? [])],
+    location: slots.location ?? null,
+    goal: slots.goal ?? null,
+    objects: [...(slots.objects ?? [])],
+  };
+  const referents = corefReferents(clusters);
+  for (const referent of referents) {
+    if (isLikelyName(referent.canonical)) {
+      if (!next.characters.includes(referent.canonical)) next.characters.push(referent.canonical);
+    } else if (!next.objects.includes(referent.canonical)) {
+      next.objects.push(referent.canonical);
+    }
+  }
+  return { slots: next, referents };
+}

package/src/memory/event-alias-ledger.mjs

@@ -0,0 +1,87 @@
+/**
+ * Event-bound alias ledger.
+ *
+ * A term becomes an alias only when:
+ *   - it is not already a known alias
+ *   - the user/Coo has used it at least twice
+ *   - it appears in at least two distinct events
+ *   - its role is stable
+ *   - it maps to an existing facet
+ *
+ * Assistant-only mentions never promote. If no target facet exists, the term
+ * is an accommodation candidate (new concept), not an alias.
+ */
+import { DEFAULT_ALIASES } from '../facets/schema.mjs';
+
+function pickDominant(counts) {
+  const entries = Object.entries(counts).sort((a, b) => b[1] - a[1]);
+  if (entries.length === 0) return null;
+  const total = entries.reduce((sum, [, count]) => sum + count, 0);
+  return { value: entries[0][0], count: entries[0][1], total, ratio: entries[0][1] / total };
+}
+
+export class EventAliasLedger {
+  constructor({ minUser = 2, minEvents = 2, terms = new Map() } = {}) {
+    this.minUser = minUser;
+    this.minEvents = minEvents;
+    this.terms = terms;
+  }
+
+  observe({ term, source = 'user', eventId, role = null, target = null, nowTurn = 0, kind = 'mention' }) {
+    const clean = String(term ?? '').trim();
+    if (clean.length === 0) return { action: 'none', reason: 'no-term' };
+    if (DEFAULT_ALIASES[clean] !== undefined) return { action: 'none', reason: 'known-alias' };
+
+    const record = this.terms.get(clean) ?? {
+      term: clean,
+      user: 0,
+      assistant: 0,
+      events: new Set(),
+      roles: {},
+      targets: {},
+      firstSeen: nowTurn,
+      lastSeen: nowTurn,
+      strong: false,
+      proposed: false,
+      accommodated: false,
+    };
+    if (source === 'user' || (source === 'coo' || source === 'peer')) record.user += 1;
+    else record.assistant += 1;
+    record.events.add(eventId);
+    if (role !== null) record.roles[role] = (record.roles[role] ?? 0) + 1;
+    if (target !== null) record.targets[target] = (record.targets[target] ?? 0) + 1;
+    if (kind === 'definition' || kind === 'quoted') record.strong = true;
+    record.lastSeen = nowTurn;
+    this.terms.set(clean, record);
+
+    if (record.proposed) return { action: 'none', reason: 'already-proposed', record };
+    if (record.accommodated) return { action: 'none', reason: 'already-accommodated', record };
+    if (record.user < this.minUser) return { action: 'none', reason: 'need-user-repeat', record };
+    if (record.events.size < this.minEvents) return { action: 'none', reason: 'need-event-repeat', record };
+
+    const dominantRole = pickDominant(record.roles);
+    if (dominantRole !== null && dominantRole.ratio < 0.8) return { action: 'none', reason: 'role-unstable', record };
+    const dominantTarget = pickDominant(record.targets);
+    if (dominantTarget === null) {
+      record.accommodated = true;
+      return {
+        action: 'accommodate',
+        reason: 'no-target-facet',
+        term: clean,
+        text: `[新概念?] ${clean}（暂无 facet 归属）`,
+        record,
+      };
+    }
+
+    record.proposed = true;
+    return {
+      action: 'propose',
+      reason: 'cross-event-assimilation',
+      term: clean,
+      role: dominantRole?.value ?? null,
+      target: dominantTarget.value,
+      text: `[新别名?] ${clean} → ${dominantTarget.value}`,
+      record,
+    };
+  }
+}

package/src/memory/event-log.mjs

@@ -0,0 +1,95 @@
+/**
+ * Append-only event log with hash chaining.
+ * The log is a normalized projection of an immutable source; it is never
+ * rewritten in place after the initial build.
+ */
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+function canonical(value) {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+  if (value !== null && typeof value === 'object') {
+    return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+
+function sha256(text) {
+  return createHash('sha256').update(text, 'utf8').digest('hex');
+}
+
+export function eventHash(record, prevHash) {
+  return sha256(`${prevHash}\n${canonical(record)}`);
+}
+
+export function buildEventLog(rows) {
+  const records = [];
+  let prevHash = 'GENESIS';
+  for (const row of [...rows].sort((a, b) => (a.turn - b.turn) || (a.seqStart - b.seqStart))) {
+    const body = {
+      turn: row.turn,
+      seqStart: row.seqStart,
+      seqEnd: row.seqEnd,
+      text: String(row.text ?? ''),
+      facets: row.facets ?? {},
+      labelSource: row.labelSource ?? 'unknown',
+    };
+    const hash = eventHash(body, prevHash);
+    records.push({ ...body, prevHash, hash });
+    prevHash = hash;
+  }
+  return records;
+}
+
+export function verifyEventLog(records) {
+  let prevHash = 'GENESIS';
+  const errors = [];
+  for (let index = 0; index < records.length; index += 1) {
+    const record = records[index];
+    if (record.prevHash !== prevHash) errors.push({ index, reason: 'prev-hash-mismatch' });
+    const expected = eventHash({
+      turn: record.turn,
+      seqStart: record.seqStart,
+      seqEnd: record.seqEnd,
+      text: record.text,
+      facets: record.facets,
+      labelSource: record.labelSource,
+    }, prevHash);
+    if (expected !== record.hash) errors.push({ index, reason: 'hash-mismatch' });
+    prevHash = record.hash;
+  }
+  return {
+    ok: errors.length === 0,
+    count: records.length,
+    rootHash: records.length === 0 ? 'GENESIS' : records[records.length - 1].hash,
+    errors,
+  };
+}
+
+export function writeEventLog(file, records) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const text = records.map((record) => JSON.stringify(record)).join('\n') + '\n';
+  fs.writeFileSync(file, text, 'utf8');
+  return { file, bytes: Buffer.byteLength(text), ...verifyEventLog(records) };
+}
+
+export function readEventLog(file) {
+  return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
+}
+
+export function appendEvent(file, record) {
+  const records = fs.existsSync(file) ? readEventLog(file) : [];
+  const prevHash = records.length === 0 ? 'GENESIS' : records[records.length - 1].hash;
+  const body = {
+    turn: record.turn,
+    seqStart: record.seqStart,
+    seqEnd: record.seqEnd,
+    text: String(record.text ?? ''),
+    facets: record.facets ?? {},
+    labelSource: record.labelSource ?? 'unknown',
+  };
+  const next = { ...body, prevHash, hash: eventHash(body, prevHash) };
+  fs.appendFileSync(file, JSON.stringify(next) + '\n', 'utf8');
+  return next;
+}