@chatpanel/events 0.101.0 → 0.103.1

package/curate.js

@@ -1,509 +1,3 @@
-// THE MAINTENANCE PASS — the half that costs nothing.
-//
-// The wiki pattern's lint step is usually described as a model reading the whole corpus
-// looking for contradictions, which is both expensive and O(N²). Most of what it would
-// "notice" is mechanical, though: a [[link]] pointing at nothing, a record connected to
-// nothing, two titles that differ by a space, a tag spelled two ways, a subject mentioned
-// everywhere with no page. None of that needs a model, and running it first is what keeps
-// the model half small enough to bound — only candidates this pass FLAGGED are ever
-// adjudicated, and only within one subject's claims (never N×N over the corpus).
-//
-// So this module is the deterministic layer, and `surveyCorpus()` is its first caller: the
-// W0 read-only report over an existing corpus. It answers, for a real user's data, the
-// question the design cannot answer from a chair — is this corpus dense enough that a
-// derived layer beats re-deriving from retrieval every time? If the numbers are thin the
-// honest conclusion is to finish retrieval instead, and this report is what says so.
-//
-// Pure input → output. No storage, no clock, no network, no model — it runs in a service
-// worker, in Node against a backup, and in the gateway against the warm store, and gives
-// the same answer in all three.
-
-import { normalizeTag } from './tags.js';
-// The bounded Levenshtein, from the module that holds only it. Two implementations of "how
-// far apart are these strings" become two answers to "is this the same title" — but see
-// distance.js for why it is not imported from voice-intents.js, which is where it grew up.
-import { blockedPairs, editDistance } from './distance.js';
-import {
-  DEFAULT_THRESHOLD, MAX_SUBJECTS, isSelfLabel, isSubjectCandidate,
-  normalizeSubject, rankSubjects, resolveSubjects,
-} from './entity.js';
-import { isRedactionToken } from './redaction-tokens.js';
-
-/** Wikilink syntax, matching store-notes.js `extractLinks` exactly — one grammar, not two. */
-const WIKILINK_RE = /\[\[([^[\]\n]+)\]\]/g;
-
-/** Below this, two normalized titles are "the same title typed twice". */
-export const NEAR_TITLE_DISTANCE = 2;
-
-/** How much term overlap counts as a record answering a question, in `spanningQuestions`. */
-export const SPAN_MIN_TERMS = 2;
-
-/**
- * Accept either the extension's Source shape (`meta.tags` / `meta.terms` / `meta.people`)
- * or a flat record, so a caller does not have to reshape a corpus to survey it.
- */
-export function normalizeRecord(rec) {
-  if (!rec || !rec.id) return null;
-  const meta = rec.meta || {};
-  const list = (v) => (Array.isArray(v) ? v.map((x) => String(x || '').trim()).filter(Boolean) : []);
-  return {
-    id: String(rec.id),
-    type: String(rec.type || rec.kind || '').toLowerCase() || 'record',
-    title: String(rec.title || '').trim(),
-    date: Number(rec.date || rec.updatedAt || rec.startedAt || 0) || 0,
-    text: String(rec.text || rec.contentText || ''),
-    tags: list(rec.tags ?? meta.tags),
-    topics: list(rec.topics ?? meta.terms ?? meta.topics),
-    people: list(rec.people ?? meta.people),
-  };
-}
-
-export function normalizeRecords(records) {
-  // Tolerates null as well as an omitted argument: a caller reading a corpus that has not
-  // loaded yet passes null, and a survey that throws there reads as a broken tool rather
-  // than as an empty corpus.
-  return (Array.isArray(records) ? records : []).map(normalizeRecord).filter(Boolean);
-}
-
-/**
- * Every `[[target]]` in a string, de-duplicated, in first-seen order — MINUS the redaction
- * placeholders, which share the syntax exactly.
- *
- * `@chatpanel/pii` writes `[[PERSON_1]]`, so a redacted transcript looks like a document
- * full of links to pages nobody wrote. Counting those as wanted pages filed the people we
- * deliberately did not learn about as things we know. Matched by TYPE, not by shape, so a
- * real `[[Q3_2026]]` link still resolves.
- */
-export function wikilinksIn(text) {
-  const out = [];
-  WIKILINK_RE.lastIndex = 0;
-  let m;
-  while ((m = WIKILINK_RE.exec(String(text || '')))) {
-    // `[[Title|alias]]` is Obsidian's display form — the LINK is the part before the pipe.
-    const target = m[1].split('|')[0].trim();
-    if (!target || isRedactionToken(target)) continue;
-    if (!out.includes(target)) out.push(target);
-  }
-  return out;
-}
-
-/**
- * The placeholders a text carries, by type — what redaction COST the graph.
- *
- * Reported rather than silently dropped, because the loss is real and the user is the only
- * one who can decide about it. `PERSON_1` is a genuine, stable entity inside its own
- * conversation; what makes it unusable as a subject is that the vault is scoped to that
- * conversation and is never persisted, so Monday's `PERSON_1` and Friday's are different
- * people and merging them would attribute one person's decisions to another.
- *
- * Seeing "412 redacted mentions across 38 records" is what tells someone their redaction
- * level is costing them a connected graph — a trade only they can make.
- */
-export function redactedTokensIn(text) {
-  const out = new Map();
-  WIKILINK_RE.lastIndex = 0;
-  let m;
-  while ((m = WIKILINK_RE.exec(String(text || '')))) {
-    const target = m[1].split('|')[0].trim();
-    if (!isRedactionToken(target)) continue;
-    const type = /^([A-Z][A-Z0-9]*)_/.exec(target.replace(/^\[{1,2}|\]{1,2}$/g, ''))?.[1] || 'OTHER';
-    out.set(type, (out.get(type) || 0) + 1);
-  }
-  return out;
-}
-
-/** Corpus-wide: how many placeholders, of which types, across how many records. */
-export function redactionCost(records = []) {
-  const byType = {};
-  let total = 0;
-  let recordsAffected = 0;
-  for (const r of normalizeRecords(records)) {
-    const found = redactedTokensIn(r.text);
-    if (!found.size) continue;
-    recordsAffected += 1;
-    for (const [type, n] of found) { byType[type] = (byType[type] || 0) + n; total += n; }
-  }
-  return { total, records: recordsAffected, byType };
-}
-
-/**
- * Links that point at nothing — the corpus telling you which pages it wants.
- *
- * This is the highest-signal candidate source in the whole pass: a human already decided
- * the subject was worth naming, and typed it. Returns most-wanted first.
- */
-export function wantedPages(records = []) {
-  const recs = normalizeRecords(records);
-  const titles = new Set(recs.map((r) => normalizeSubject(r.title)).filter(Boolean));
-  const wanted = new Map();
-  for (const r of recs) {
-    for (const target of wikilinksIn(r.text)) {
-      const norm = normalizeSubject(target);
-      if (!norm || titles.has(norm)) continue;
-      let w = wanted.get(norm);
-      if (!w) { w = { target, norm, count: 0, records: new Set() }; wanted.set(norm, w); }
-      w.count += 1;
-      w.records.add(r.id);
-    }
-  }
-  return [...wanted.values()]
-    .map((w) => ({ target: w.target, norm: w.norm, count: w.count, recordCount: w.records.size }))
-    .sort((a, b) => b.recordCount - a.recordCount || b.count - a.count || a.norm.localeCompare(b.norm));
-}
-
-/**
- * Records connected to nothing else.
- *
- * "Connected" is deliberately generous — a resolvable wikilink in either direction, or a
- * shared tag, or a shared topic. A record that fails all three is genuinely marooned: no
- * path leads to it except full-text search, which is exactly the state a derived layer is
- * supposed to fix. Notes' Stats tab computes a notes-only version of this today; this is
- * the same question asked across chats, meetings and notes at once.
- */
-export function orphanRecords(records = []) {
-  const recs = normalizeRecords(records);
-  const byTitle = new Map();
-  for (const r of recs) {
-    const norm = normalizeSubject(r.title);
-    if (norm && !byTitle.has(norm)) byTitle.set(norm, r.id);
-  }
-  const linked = new Set();
-  const shared = new Map(); // term -> record ids
-  for (const r of recs) {
-    for (const target of wikilinksIn(r.text)) {
-      const hit = byTitle.get(normalizeSubject(target));
-      if (hit && hit !== r.id) { linked.add(r.id); linked.add(hit); }
-    }
-    for (const t of [...r.tags.map(normalizeTag), ...r.topics.map(normalizeSubject)]) {
-      if (!t) continue;
-      if (!shared.has(t)) shared.set(t, new Set());
-      shared.get(t).add(r.id);
-    }
-  }
-  for (const ids of shared.values()) {
-    if (ids.size > 1) for (const id of ids) linked.add(id);
-  }
-  return recs.filter((r) => !linked.has(r.id)).map((r) => ({ id: r.id, type: r.type, title: r.title, date: r.date }));
-}
-
-/**
- * "Atlas sync 3" and "Atlas sync 4" are a SERIES, not a typo of each other — and so are
- * "Chat 1" … "Chat 8". They sit one or two edits apart, so without this guard the near-match
- * pass reports every recurring meeting and every default chat title as one duplicate group,
- * which is precisely the noise that makes a maintenance report get ignored.
- */
-function sameSeries(a, b) {
-  const stem = (s) => s.replace(/\s*\d+$/, '').trim();
-  const sa = stem(a), sb = stem(b);
-  return sa !== a || sb !== b ? sa === sb : false;
-}
-
-/**
- * Titles that are the same thing written twice — exact collisions after normalization, and
- * near-misses within `NEAR_TITLE_DISTANCE`. Each group is a merge candidate, never a merge:
- * "Q3 Planning" and "Q4 Planning" are one character apart and must NOT be merged, which is
- * precisely why this pass reports and the promotion gate decides.
- */
-export function duplicateTitles(records = [], { distance = NEAR_TITLE_DISTANCE } = {}) {
-  const recs = normalizeRecords(records).filter((r) => normalizeSubject(r.title));
-  const groups = new Map();
-  for (const r of recs) {
-    const norm = normalizeSubject(r.title);
-    if (!groups.has(norm)) groups.set(norm, []);
-    groups.get(norm).push(r);
-  }
-  const norms = [...groups.keys()].sort();
-
-  // Near-misses first, over BLOCKED candidate pairs rather than every pair — see
-  // MAX_PAIR_COMPARISONS for what that replaced.
-  const nearOf = new Map();
-  for (const [a, b] of blockedPairs(norms)) {
-    // A title short enough that `distance` edits rewrite most of it is not a near-miss.
-    if (Math.min(a.length, b.length) <= distance * 2) continue;
-    if (sameSeries(a, b)) continue;
-    if (editDistance(a, b, distance) > distance) continue;
-    if (!nearOf.has(a)) nearOf.set(a, []);
-    if (!nearOf.has(b)) nearOf.set(b, []);
-    nearOf.get(a).push(b);
-    nearOf.get(b).push(a);
-  }
-
-  const out = [];
-  const merged = new Set();
-  for (const norm of norms) {
-    if (merged.has(norm)) continue;
-    const cluster = [norm];
-    for (const other of nearOf.get(norm) || []) {
-      if (merged.has(other) || other === norm) continue;
-      cluster.push(other);
-      merged.add(other);
-    }
-    const items = cluster.flatMap((n) => groups.get(n));
-    if (items.length > 1) {
-      out.push({
-        norm,
-        titles: [...new Set(items.map((r) => r.title))],
-        ids: items.map((r) => r.id),
-      });
-    }
-  }
-  return out.sort((a, b) => b.ids.length - a.ids.length || a.norm.localeCompare(b.norm));
-}
-
-/**
- * One vocabulary spelled several ways — "design-review" and "designreview" filed apart.
- *
- * Tags already normalize (tags.js), so a collision here is a real divergence in what the
- * user typed, not a normalization bug. Topics come from a model and drift harder.
- */
-export function vocabularyDrift(records = [], { distance = 1 } = {}) {
-  const recs = normalizeRecords(records);
-  const freq = new Map();
-  for (const r of recs) {
-    for (const raw of r.tags) {
-      const t = normalizeTag(raw);
-      if (t) freq.set(t, (freq.get(t) || 0) + 1);
-    }
-    for (const raw of r.topics) {
-      const t = normalizeTag(raw);
-      if (t) freq.set(t, (freq.get(t) || 0) + 1);
-    }
-  }
-  const terms = [...freq.keys()].sort();
-  const bare = (t) => t.replace(/-/g, '');
-
-  // Same blocking as duplicateTitles. A vocabulary is smaller than a corpus, so this has not
-  // bitten yet — but it is the identical shape, and the identical shape is what bites.
-  const nearOf = new Map();
-  for (const [a, b] of blockedPairs(terms)) {
-    if (Math.min(a.length, b.length) <= 3) continue;
-    if (bare(a) !== bare(b) && editDistance(a, b, distance) > distance) continue;
-    if (!nearOf.has(a)) nearOf.set(a, []);
-    if (!nearOf.has(b)) nearOf.set(b, []);
-    nearOf.get(a).push(b);
-    nearOf.get(b).push(a);
-  }
-
-  const out = [];
-  const taken = new Set();
-  for (const term of terms) {
-    if (taken.has(term)) continue;
-    const near = (nearOf.get(term) || []).filter((t) => !taken.has(t) && t !== term);
-    for (const t of near) taken.add(t);
-    if (near.length) {
-      out.push({ terms: [term, ...near].map((t) => ({ term: t, count: freq.get(t) || 0 })) });
-    }
-  }
-  return out.sort((a, b) => b.terms.length - a.terms.length);
-}
-
-/**
- * Every mention the corpus offers, without a model: who was on a call, what it was tagged,
- * what topics were extracted for it, what someone [[linked]] to, and what the records are
- * titled. These are the raw inputs `resolveSubjects()` folds into identities.
- */
-export function mentionsFrom(records = []) {
-  const recs = normalizeRecords(records);
-  const out = [];
-  for (const r of recs) {
-    for (const name of r.people) out.push({ kind: 'person', name, recordId: r.id });
-    for (const name of r.tags) out.push({ kind: 'tag', name, recordId: r.id });
-    for (const name of r.topics) out.push({ kind: 'topic', name, recordId: r.id });
-    for (const name of wikilinksIn(r.text)) out.push({ kind: 'title', name, recordId: r.id });
-  }
-  // A person's self-label ("You") survives candidacy HERE and is decided by
-  // `resolveSubjects`, which is the only layer that knows whether we have a name to fold it
-  // into. Filtering it out at this level threw the user out of their own corpus.
-  return out.filter((m) => isSubjectCandidate(m.name, { kind: m.kind })
-    || (m.kind === 'person' && isSelfLabel(m.name)));
-}
-
-/** Query terms, folded the way `sources-retrieval.js queryTerms` folds them. */
-function termsOf(text, { min = 3 } = {}) {
-  return [...new Set(
-    String(text || '').toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((w) => w.length >= min),
-  )];
-}
-
-/**
- * How often does answering a question require more than one record?
- *
- * The measurement that decides whether this layer is worth building at all. If almost every
- * question is answered by a single record, retrieval already wins and synthesis buys
- * nothing; if the typical question touches four, then every asking of it re-derives the
- * same join, forever, and a maintained page pays for itself.
- *
- * Deterministic and crude on purpose — term overlap, no model, no embeddings. It is a
- * ratio, not a search engine, and a ratio only has to be honest.
- */
-export function spanningQuestions(records = [], questions = [], { minTerms = SPAN_MIN_TERMS, span = 3 } = {}) {
-  const recs = normalizeRecords(records);
-  const index = recs.map((r) => ({ id: r.id, terms: new Set(termsOf(`${r.title}\n${r.text}`)) }));
-  let spanning = 0;
-  let considered = 0;
-  const hits = [];
-  for (const q of questions) {
-    const terms = termsOf(q);
-    if (terms.length < minTerms) continue; // "thanks!" is not a question about the corpus
-    considered += 1;
-    let n = 0;
-    for (const rec of index) {
-      let overlap = 0;
-      for (const t of terms) if (rec.terms.has(t)) overlap += 1;
-      if (overlap >= minTerms) n += 1;
-    }
-    hits.push(n);
-    if (n > span) spanning += 1;
-  }
-  hits.sort((a, b) => a - b);
-  return {
-    considered,
-    spanning,
-    fraction: considered ? spanning / considered : 0,
-    medianRecords: hits.length ? hits[Math.floor(hits.length / 2)] : 0,
-    span,
-  };
-}
-
-/**
- * W0 — the whole deterministic pass, as one read-only report.
- *
- * Nothing here writes, and nothing here calls a model. Run it before building the derived
- * layer, and let its numbers set the thresholds in `entity.js` rather than the other way
- * round.
- */
-export function surveyCorpus(records = [], { questions = [], threshold = DEFAULT_THRESHOLD, limit = MAX_SUBJECTS } = {}) {
-  const recs = normalizeRecords(records);
-  const byType = {};
-  let chars = 0;
-  for (const r of recs) {
-    byType[r.type] = (byType[r.type] || 0) + 1;
-    chars += r.text.length;
-  }
-  const mentions = mentionsFrom(recs);
-  const subjects = resolveSubjects(mentions);
-  const qualifying = rankSubjects(subjects, { threshold, limit });
-  const byKind = {};
-  for (const s of qualifying) byKind[s.kind] = (byKind[s.kind] || 0) + 1;
-  const wanted = wantedPages(recs);
-  const orphans = orphanRecords(recs);
-  const redaction = redactionCost(recs);
-
-  return {
-    corpus: {
-      records: recs.length,
-      byType,
-      chars,
-      oldest: recs.reduce((m, r) => (r.date && (!m || r.date < m) ? r.date : m), 0),
-      newest: recs.reduce((m, r) => Math.max(m, r.date || 0), 0),
-    },
-    subjects: {
-      total: subjects.size,
-      qualifying: qualifying.length,
-      byKind,
-      threshold: { records: threshold.records ?? DEFAULT_THRESHOLD.records, mentions: threshold.mentions ?? DEFAULT_THRESHOLD.mentions },
-      capped: subjects.size > 0 && qualifying.length === limit,
-      top: qualifying.slice(0, 25).map((s) => ({
-        key: s.key, kind: s.kind, name: s.name, aliases: s.aliases,
-        records: s.recordCount, mentions: s.mentions,
-      })),
-    },
-    wantedPages: { total: wanted.length, top: wanted.slice(0, 25) },
-    orphans: {
-      total: orphans.length,
-      fraction: recs.length ? orphans.length / recs.length : 0,
-      byType: orphans.reduce((acc, r) => ({ ...acc, [r.type]: (acc[r.type] || 0) + 1 }), {}),
-      sample: orphans.slice(0, 10),
-    },
-    redaction,
-    duplicateTitles: duplicateTitles(recs),
-    vocabularyDrift: vocabularyDrift(recs),
-    questions: spanningQuestions(recs, questions),
-  };
-}
-
-/**
- * Sensitivity — the same corpus at several thresholds, so the number is CHOSEN rather than
- * inherited. A threshold that yields 4 subjects is useless and one that yields 4000 is a
- * second corpus (C5); the report should make both visible at a glance.
- */
-export function thresholdSweep(records = [], grid = [
-  { records: 2, mentions: 2 }, { records: 3, mentions: 5 }, { records: 4, mentions: 8 }, { records: 6, mentions: 12 },
-]) {
-  const subjects = resolveSubjects(mentionsFrom(records));
-  return grid.map((threshold) => {
-    const ranked = rankSubjects(subjects, { threshold, limit: Infinity });
-    const byKind = {};
-    for (const s of ranked) byKind[s.kind] = (byKind[s.kind] || 0) + 1;
-    return { threshold, qualifying: ranked.length, byKind };
-  });
-}
-
-const pct = (x) => `${(x * 100).toFixed(1)}%`;
-const day = (ms) => (ms ? new Date(ms).toISOString().slice(0, 10) : '—');
-
-/** The report as plain text — the form W0 is actually read in. */
-export function formatSurvey(report, { sweep = null } = {}) {
-  if (!report) return 'no report';
-  const L = [];
-  const { corpus, subjects, wantedPages: wp, orphans, duplicateTitles: dupes, vocabularyDrift: drift, questions } = report;
-
-  L.push('CORPUS');
-  L.push(`  ${corpus.records} records (${Object.entries(corpus.byType).map(([k, v]) => `${v} ${k}`).join(', ') || 'none'})`);
-  L.push(`  ${corpus.chars.toLocaleString()} chars · ${day(corpus.oldest)} → ${day(corpus.newest)}`);
-
-  L.push('');
-  L.push(`SUBJECTS  (earn a page at ≥${subjects.threshold.records} records and ≥${subjects.threshold.mentions} mentions)`);
-  L.push(`  ${subjects.qualifying} of ${subjects.total} candidates qualify${subjects.capped ? ' (AT THE CEILING — raise the threshold)' : ''}`);
-  L.push(`  by kind: ${Object.entries(subjects.byKind).map(([k, v]) => `${k} ${v}`).join(', ') || 'none'}`);
-  for (const s of subjects.top) {
-    L.push(`    ${String(s.records).padStart(4)} rec ${String(s.mentions).padStart(5)} men  ${s.kind}: ${s.name}${s.aliases.length ? `  (aka ${s.aliases.join(', ')})` : ''}`);
-  }
-
-  if (sweep?.length) {
-    L.push('');
-    L.push('THRESHOLD SWEEP');
-    for (const row of sweep) {
-      L.push(`  ≥${row.threshold.records} rec / ≥${row.threshold.mentions} men → ${String(row.qualifying).padStart(5)} subjects  (${Object.entries(row.byKind).map(([k, v]) => `${k} ${v}`).join(', ') || 'none'})`);
-    }
-  }
-
-  L.push('');
-  L.push(`WANTED PAGES  ([[links]] resolving to nothing) — ${wp.total}`);
-  for (const w of wp.top) L.push(`    ${String(w.recordCount).padStart(4)} rec  ${w.target}`);
-  if (!wp.total) L.push('    none — either nobody uses [[links]] yet, or every one resolves');
-
-  L.push('');
-  L.push(`ORPHANS  (no link, no shared tag, no shared topic) — ${orphans.total} of ${corpus.records} (${pct(orphans.fraction)})`);
-  L.push(`  by type: ${Object.entries(orphans.byType).map(([k, v]) => `${k} ${v}`).join(', ') || 'none'}`);
-
-  L.push('');
-  L.push(`REDACTED MENTIONS  (placeholders that cannot become subjects) — ${report.redaction.total} across ${report.redaction.records} records`);
-  L.push(`  by type: ${Object.entries(report.redaction.byType).map(([k, v]) => `${k} ${v}`).join(', ') || 'none'}`);
-  if (report.redaction.total) {
-    L.push('  → the vault is per-conversation and is not persisted, so one PERSON_1 is not another.');
-    L.push('    Lowering the redaction level is what buys these back as real subjects.');
-  }
-
-  L.push('');
-  L.push(`DUPLICATE / NEAR-DUPLICATE TITLES — ${dupes.length} groups`);
-  for (const g of dupes.slice(0, 10)) L.push(`    ${g.ids.length}×  ${g.titles.join('  |  ')}`);
-
-  L.push('');
-  L.push(`VOCABULARY DRIFT  (one term filed several ways) — ${drift.length} clusters`);
-  for (const g of drift.slice(0, 10)) L.push(`    ${g.terms.map((t) => `${t.term}(${t.count})`).join('  |  ')}`);
-
-  L.push('');
-  L.push('QUESTIONS THAT SPAN RECORDS');
-  if (!questions.considered) {
-    L.push('  no questions supplied — pass the recent user turns to measure this');
-  } else {
-    L.push(`  ${questions.spanning} of ${questions.considered} (${pct(questions.fraction)}) touch more than ${questions.span} records`);
-    L.push(`  median records touched: ${questions.medianRecords}`);
-    L.push('');
-    L.push(questions.fraction >= 0.3
-      ? '  → synthesis has something to compound: most asking re-derives a join across records.'
-      : '  → THIN. Most questions are answered by one record; finish retrieval (docs/retrieval.md) before building briefs.');
-  }
-  return L.join('\n');
-}
+import{normalizeTag as S}from"./tags.js";import{blockedPairs as T,editDistance as k}from"./distance.js";import{DEFAULT_THRESHOLD as b,MAX_SUBJECTS as O,isSelfLabel as v,isSubjectCandidate as M,normalizeSubject as g,rankSubjects as x,resolveSubjects as E}from"./entity.js";import{isRedactionToken as j}from"./redaction-tokens.js";const $=/\[\[([^[\]\n]+)\]\]/g,L=2,N=2;function D(r){if(!r||!r.id)return null;const c=r.meta||{},t=e=>Array.isArray(e)?e.map(n=>String(n||"").trim()).filter(Boolean):[];return{id:String(r.id),type:String(r.type||r.kind||"").toLowerCase()||"record",title:String(r.title||"").trim(),date:Number(r.date||r.updatedAt||r.startedAt||0)||0,text:String(r.text||r.contentText||""),tags:t(r.tags??c.tags),topics:t(r.topics??c.terms??c.topics),people:t(r.people??c.people)}}function m(r){return(Array.isArray(r)?r:[]).map(D).filter(Boolean)}function w(r){const c=[];$.lastIndex=0;let t;for(;t=$.exec(String(r||""));){const e=t[1].split("|")[0].trim();!e||j(e)||c.includes(e)||c.push(e)}return c}function P(r){const c=new Map;$.lastIndex=0;let t;for(;t=$.exec(String(r||""));){const e=t[1].split("|")[0].trim();if(!j(e))continue;const n=/^([A-Z][A-Z0-9]*)_/.exec(e.replace(/^\[{1,2}|\]{1,2}$/g,""))?.[1]||"OTHER";c.set(n,(c.get(n)||0)+1)}return c}function q(r=[]){const c={};let t=0,e=0;for(const n of m(r)){const i=P(n.text);if(i.size){e+=1;for(const[a,u]of i)c[a]=(c[a]||0)+u,t+=u}}return{total:t,records:e,byType:c}}function z(r=[]){const c=m(r),t=new Set(c.map(n=>g(n.title)).filter(Boolean)),e=new Map;for(const n of c)for(const i of w(n.text)){const a=g(i);if(!a||t.has(a))continue;let u=e.get(a);u||(u={target:i,norm:a,count:0,records:new Set},e.set(a,u)),u.count+=1,u.records.add(n.id)}return[...e.values()].map(n=>({target:n.target,norm:n.norm,count:n.count,recordCount:n.records.size})).sort((n,i)=>i.recordCount-n.recordCount||i.count-n.count||n.norm.localeCompare(i.norm))}function H(r=[]){const c=m(r),t=new Map;for(const i of c){const a=g(i.title);a&&!t.has(a)&&t.set(a,i.id)}const e=new Set,n=new Map;for(const i of c){for(const a of w(i.text)){const u=t.get(g(a));u&&u!==i.id&&(e.add(i.id),e.add(u))}for(const a of[...i.tags.map(S),...i.topics.map(g)])a&&(n.has(a)||n.set(a,new Set),n.get(a).add(i.id))}for(const i of n.values())if(i.size>1)for(const a of i)e.add(a);return c.filter(i=>!e.has(i.id)).map(i=>({id:i.id,type:i.type,title:i.title,date:i.date}))}function _(r,c){const t=i=>i.replace(/\s*\d+$/,"").trim(),e=t(r),n=t(c);return e!==r||n!==c?e===n:!1}function U(r=[],{distance:c=L}={}){const t=m(r).filter(p=>g(p.title)),e=new Map;for(const p of t){const o=g(p.title);e.has(o)||e.set(o,[]),e.get(o).push(p)}const n=[...e.keys()].sort(),i=new Map;for(const[p,o]of T(n))Math.min(p.length,o.length)<=c*2||_(p,o)||k(p,o,c)>c||(i.has(p)||i.set(p,[]),i.has(o)||i.set(o,[]),i.get(p).push(o),i.get(o).push(p));const a=[],u=new Set;for(const p of n){if(u.has(p))continue;const o=[p];for(const d of i.get(p)||[])u.has(d)||d===p||(o.push(d),u.add(d));const s=o.flatMap(d=>e.get(d));s.length>1&&a.push({norm:p,titles:[...new Set(s.map(d=>d.title))],ids:s.map(d=>d.id)})}return a.sort((p,o)=>o.ids.length-p.ids.length||p.norm.localeCompare(o.norm))}function B(r=[],{distance:c=1}={}){const t=m(r),e=new Map;for(const o of t){for(const s of o.tags){const d=S(s);d&&e.set(d,(e.get(d)||0)+1)}for(const s of o.topics){const d=S(s);d&&e.set(d,(e.get(d)||0)+1)}}const n=[...e.keys()].sort(),i=o=>o.replace(/-/g,""),a=new Map;for(const[o,s]of T(n))Math.min(o.length,s.length)<=3||i(o)!==i(s)&&k(o,s,c)>c||(a.has(o)||a.set(o,[]),a.has(s)||a.set(s,[]),a.get(o).push(s),a.get(s).push(o));const u=[],p=new Set;for(const o of n){if(p.has(o))continue;const s=(a.get(o)||[]).filter(d=>!p.has(d)&&d!==o);for(const d of s)p.add(d);s.length&&u.push({terms:[o,...s].map(d=>({term:d,count:e.get(d)||0}))})}return u.sort((o,s)=>s.terms.length-o.terms.length)}function A(r=[]){const c=m(r),t=[];for(const e of c){for(const n of e.people)t.push({kind:"person",name:n,recordId:e.id});for(const n of e.tags)t.push({kind:"tag",name:n,recordId:e.id});for(const n of e.topics)t.push({kind:"topic",name:n,recordId:e.id});for(const n of w(e.text))t.push({kind:"title",name:n,recordId:e.id})}return t.filter(e=>M(e.name,{kind:e.kind})||e.kind==="person"&&v(e.name))}function I(r,{min:c=3}={}){return[...new Set(String(r||"").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(t=>t.length>=c))]}function K(r=[],c=[],{minTerms:t=N,span:e=3}={}){const i=m(r).map(o=>({id:o.id,terms:new Set(I(`${o.title}
+${o.text}`))}));let a=0,u=0;const p=[];for(const o of c){const s=I(o);if(s.length<t)continue;u+=1;let d=0;for(const l of i){let y=0;for(const f of s)l.terms.has(f)&&(y+=1);y>=t&&(d+=1)}p.push(d),d>e&&(a+=1)}return p.sort((o,s)=>o-s),{considered:u,spanning:a,fraction:u?a/u:0,medianRecords:p.length?p[Math.floor(p.length/2)]:0,span:e}}function Q(r=[],{questions:c=[],threshold:t=b,limit:e=O}={}){const n=m(r),i={};let a=0;for(const f of n)i[f.type]=(i[f.type]||0)+1,a+=f.text.length;const u=A(n),p=E(u),o=x(p,{threshold:t,limit:e}),s={};for(const f of o)s[f.kind]=(s[f.kind]||0)+1;const d=z(n),l=H(n),y=q(n);return{corpus:{records:n.length,byType:i,chars:a,oldest:n.reduce((f,h)=>h.date&&(!f||h.date<f)?h.date:f,0),newest:n.reduce((f,h)=>Math.max(f,h.date||0),0)},subjects:{total:p.size,qualifying:o.length,byKind:s,threshold:{records:t.records??b.records,mentions:t.mentions??b.mentions},capped:p.size>0&&o.length===e,top:o.slice(0,25).map(f=>({key:f.key,kind:f.kind,name:f.name,aliases:f.aliases,records:f.recordCount,mentions:f.mentions}))},wantedPages:{total:d.length,top:d.slice(0,25)},orphans:{total:l.length,fraction:n.length?l.length/n.length:0,byType:l.reduce((f,h)=>({...f,[h.type]:(f[h.type]||0)+1}),{}),sample:l.slice(0,10)},redaction:y,duplicateTitles:U(n),vocabularyDrift:B(n),questions:K(n,c)}}function Z(r=[],c=[{records:2,mentions:2},{records:3,mentions:5},{records:4,mentions:8},{records:6,mentions:12}]){const t=E(A(r));return c.map(e=>{const n=x(t,{threshold:e,limit:1/0}),i={};for(const a of n)i[a.kind]=(i[a.kind]||0)+1;return{threshold:e,qualifying:n.length,byKind:i}})}const C=r=>`${(r*100).toFixed(1)}%`,R=r=>r?new Date(r).toISOString().slice(0,10):"—";function V(r,{sweep:c=null}={}){if(!r)return"no report";const t=[],{corpus:e,subjects:n,wantedPages:i,orphans:a,duplicateTitles:u,vocabularyDrift:p,questions:o}=r;t.push("CORPUS"),t.push(`  ${e.records} records (${Object.entries(e.byType).map(([s,d])=>`${d} ${s}`).join(", ")||"none"})`),t.push(`  ${e.chars.toLocaleString()} chars · ${R(e.oldest)} → ${R(e.newest)}`),t.push(""),t.push(`SUBJECTS  (earn a page at ≥${n.threshold.records} records and ≥${n.threshold.mentions} mentions)`),t.push(`  ${n.qualifying} of ${n.total} candidates qualify${n.capped?" (AT THE CEILING — raise the threshold)":""}`),t.push(`  by kind: ${Object.entries(n.byKind).map(([s,d])=>`${s} ${d}`).join(", ")||"none"}`);for(const s of n.top)t.push(`    ${String(s.records).padStart(4)} rec ${String(s.mentions).padStart(5)} men  ${s.kind}: ${s.name}${s.aliases.length?`  (aka ${s.aliases.join(", ")})`:""}`);if(c?.length){t.push(""),t.push("THRESHOLD SWEEP");for(const s of c)t.push(`  ≥${s.threshold.records} rec / ≥${s.threshold.mentions} men → ${String(s.qualifying).padStart(5)} subjects  (${Object.entries(s.byKind).map(([d,l])=>`${d} ${l}`).join(", ")||"none"})`)}t.push(""),t.push(`WANTED PAGES  ([[links]] resolving to nothing) — ${i.total}`);for(const s of i.top)t.push(`    ${String(s.recordCount).padStart(4)} rec  ${s.target}`);i.total||t.push("    none — either nobody uses [[links]] yet, or every one resolves"),t.push(""),t.push(`ORPHANS  (no link, no shared tag, no shared topic) — ${a.total} of ${e.records} (${C(a.fraction)})`),t.push(`  by type: ${Object.entries(a.byType).map(([s,d])=>`${s} ${d}`).join(", ")||"none"}`),t.push(""),t.push(`REDACTED MENTIONS  (placeholders that cannot become subjects) — ${r.redaction.total} across ${r.redaction.records} records`),t.push(`  by type: ${Object.entries(r.redaction.byType).map(([s,d])=>`${s} ${d}`).join(", ")||"none"}`),r.redaction.total&&(t.push("  → the vault is per-conversation and is not persisted, so one PERSON_1 is not another."),t.push("    Lowering the redaction level is what buys these back as real subjects.")),t.push(""),t.push(`DUPLICATE / NEAR-DUPLICATE TITLES — ${u.length} groups`);for(const s of u.slice(0,10))t.push(`    ${s.ids.length}×  ${s.titles.join("  |  ")}`);t.push(""),t.push(`VOCABULARY DRIFT  (one term filed several ways) — ${p.length} clusters`);for(const s of p.slice(0,10))t.push(`    ${s.terms.map(d=>`${d.term}(${d.count})`).join("  |  ")}`);return t.push(""),t.push("QUESTIONS THAT SPAN RECORDS"),o.considered?(t.push(`  ${o.spanning} of ${o.considered} (${C(o.fraction)}) touch more than ${o.span} records`),t.push(`  median records touched: ${o.medianRecords}`),t.push(""),t.push(o.fraction>=.3?"  → synthesis has something to compound: most asking re-derives a join across records.":"  → THIN. Most questions are answered by one record; finish retrieval (docs/retrieval.md) before building briefs.")):t.push("  no questions supplied — pass the recent user turns to measure this"),t.join(`
+`)}export{L as NEAR_TITLE_DISTANCE,N as SPAN_MIN_TERMS,U as duplicateTitles,V as formatSurvey,A as mentionsFrom,D as normalizeRecord,m as normalizeRecords,H as orphanRecords,P as redactedTokensIn,q as redactionCost,K as spanningQuestions,Q as surveyCorpus,Z as thresholdSweep,B as vocabularyDrift,z as wantedPages,w as wikilinksIn};

package/distance.js

@@ -1,124 +1 @@
-// How far apart are two strings — one answer, and nothing else in the module.
-//
-// It exists as its own file for a load-time reason, and the reason is worth recording
-// because it is the second time this rule has been learned here. `voice-intents.js` needed
-// a bounded Levenshtein for wake-word matching and grew one. When the maintenance pass
-// needed the same question answered for near-duplicate titles, importing it from there was
-// the correct instinct — reuse, don't reinvent — and it pulled `voice-intents.js` (79 KB)
-// and its `structured.js` dependency (41 KB) onto the MV3 service worker's cold start, for
-// forty lines of arithmetic.
-//
-// So: a primitive two unrelated features need belongs in a module of its own, not in
-// whichever feature happened to need it first. Reuse is right; reuse through a large module
-// is a 120 KB import of one function.
-
-/**
- * Bounded Levenshtein. Returns early once the distance cannot come in under `max`, so a
- * wake scan over a long transcript — or a pairwise title sweep over a corpus — stays linear
- * in practice rather than paying for an exact answer nobody reads.
- *
- * Past `max` the return is `max + 1`: "far", not a number. Callers compare against `max`.
- */
-export function editDistance(a, b, max = Infinity) {
-  if (a === b) return 0;
-  if (Math.abs(a.length - b.length) > max) return max + 1;
-  let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
-  for (let i = 1; i <= a.length; i++) {
-    const cur = [i];
-    let best = i;
-    for (let j = 1; j <= b.length; j++) {
-      cur[j] = Math.min(
-        prev[j] + 1,
-        cur[j - 1] + 1,
-        prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
-      );
-      if (cur[j] < best) best = cur[j];
-    }
-    if (best > max) return max + 1;
-    prev = cur;
-  }
-  return prev[b.length];
-}
-
-/**
- * A ceiling on pairwise work, and the reason it exists.
- *
- * Comparing every pair is the N×N scan the knowledge design forbids, and it behaves exactly
- * as that rule predicts. Two passes learned it the hard way: near-duplicate titles ran 40s
- * over 12,000 records, and merge suggestions did not finish 12,000 SUBJECTS in two minutes.
- * Both on the UI thread, which is an unresponsive tab rather than a slow report.
- */
-export const MAX_PAIR_COMPARISONS = 200_000;
-
-const BLOCK_KEY_CHARS = 4;
-// Every block gets at least this many comparisons before the budget can starve it.
-const MIN_BLOCK_BUDGET = 2_000;
-
-/**
- * The keys a string is filed under for candidate generation.
- *
- * Three, and each earns its place: two strings within a couple of edits still agree on their
- * first few characters unless the typo is at the front — in which case they agree on their
- * last few — and two forms of one person's name ("alex rivera", "a rivera") agree on the
- * LAST TOKEN even when neither end matches. Drop the third and abbreviated first names stop
- * being found at all.
- */
-export function blockKeys(norm) {
-  const s = String(norm || '');
-  if (!s) return [];
-  const keys = new Set([`p:${s.slice(0, BLOCK_KEY_CHARS)}`, `s:${s.slice(-BLOCK_KEY_CHARS)}`]);
-  const last = s.split(' ').filter(Boolean).pop();
-  if (last && last.length >= 2) keys.add(`t:${last}`);
-  return [...keys];
-}
-
-/**
- * BLOCKING — the standard record-linkage answer to "which pairs are worth comparing".
- *
- * Files every string under `blockKeys` and yields only pairs that share one, so the work is
- * proportional to the corpus rather than to its square. Each unordered pair is yielded at
- * most once even when two strings share several keys.
- *
- * `budget` is the backstop for the pathological case — ten thousand titles that all start the
- * same way land in one block, and a block is compared pairwise. A weird corpus then costs a
- * truncated report instead of a hung page.
- */
-export function* blockedPairs(values, { budget = MAX_PAIR_COMPARISONS } = {}) {
-  const blocks = new Map();
-  for (const v of values) {
-    for (const key of blockKeys(v)) {
-      if (!blocks.has(key)) blocks.set(key, []);
-      blocks.get(key).push(v);
-    }
-  }
-  let spent = 0;
-  const seen = new Set();
-  // SMALLEST BLOCKS FIRST, and a per-block share of the budget. Both are about RECALL, not
-  // speed, and the first version got this wrong: nine thousand subjects all beginning
-  // "Unrelated Person" land in one bucket under the prefix key, and that single
-  // non-discriminating block spent the entire budget before the buckets holding the real
-  // findings were ever reached — so a big corpus returned five hundred suggestions and not
-  // one of the ones that mattered.
-  //
-  // A small block is a discriminating one: "rivera" as a last token says far more than
-  // "unre" as a prefix. Working through them in size order means the specific evidence is
-  // spent first and the vague evidence gets whatever is left.
-  const buckets = [...blocks.values()].filter((b) => b.length > 1).sort((a, b) => a.length - b.length);
-  for (const bucket of buckets) {
-    if (spent >= budget) return;
-    // No single block may consume the whole budget, however it is ordered.
-    const blockBudget = Math.min(budget - spent, Math.max(MIN_BLOCK_BUDGET, Math.floor(budget / 8)));
-    let blockSpent = 0;
-    for (let i = 0; i < bucket.length && blockSpent < blockBudget; i += 1) {
-      for (let j = i + 1; j < bucket.length && blockSpent < blockBudget; j += 1) {
-        const a = bucket[i]; const b = bucket[j];
-        const key = a < b ? `${a}\u0000${b}` : `${b}\u0000${a}`;
-        if (seen.has(key)) continue;
-        seen.add(key);
-        blockSpent += 1;
-        spent += 1;
-        yield [a, b];
-      }
-    }
-  }
-}
+function y(c,e,o=1/0){if(c===e)return 0;if(Math.abs(c.length-e.length)>o)return o+1;let s=Array.from({length:e.length+1},(l,r)=>r);for(let l=1;l<=c.length;l++){const r=[l];let n=l;for(let t=1;t<=e.length;t++)r[t]=Math.min(s[t]+1,r[t-1]+1,s[t-1]+(c[l-1]===e[t-1]?0:1)),r[t]<n&&(n=r[t]);if(n>o)return o+1;s=r}return s[e.length]}const k=2e5,p=4,M=2e3;function _(c){const e=String(c||"");if(!e)return[];const o=new Set([`p:${e.slice(0,p)}`,`s:${e.slice(-p)}`]),s=e.split(" ").filter(Boolean).pop();return s&&s.length>=2&&o.add(`t:${s}`),[...o]}function*S(c,{budget:e=k}={}){const o=new Map;for(const n of c)for(const t of _(n))o.has(t)||o.set(t,[]),o.get(t).push(n);let s=0;const l=new Set,r=[...o.values()].filter(n=>n.length>1).sort((n,t)=>n.length-t.length);for(const n of r){if(s>=e)return;const t=Math.min(e-s,Math.max(M,Math.floor(e/8)));let u=0;for(let i=0;i<n.length&&u<t;i+=1)for(let g=i+1;g<n.length&&u<t;g+=1){const f=n[i],h=n[g],a=f<h?`${f}\0${h}`:`${h}\0${f}`;l.has(a)||(l.add(a),u+=1,s+=1,yield[f,h])}}}export{k as MAX_PAIR_COMPARISONS,_ as blockKeys,S as blockedPairs,y as editDistance};