@discretion/core 0.1.0

package/src/checksums/weighted.ts

@@ -0,0 +1,108 @@
+/**
+ * The weighted modular sum — the single algorithm behind most national
+ * identifier check digits.
+ *
+ * SPEC.md calls for implementing each algorithm once and reusing it: "Many
+ * national IDs are the same algorithm with different weights." That is
+ * precisely true of this family. Turkey's TCKN, Poland's PESEL, Brazil's CPF,
+ * China's RIC, Singapore's NRIC, Australia's TFN and dozens of others are all
+ *
+ *     Σ(digitᵢ × weightᵢ) mod m
+ *
+ * differing only in the weight vector, the modulus, and how the remainder is
+ * turned back into an expected check character.
+ *
+ * DESIGN NOTE. This module deliberately stops at the remainder rather than
+ * trying to model every "and then…" rule as configuration. The final step
+ * varies too much to generalize honestly — some schemes use the remainder
+ * directly, some its complement, some map 10→'X', some declare 10 invalid,
+ * some subtract from 11 and fold both 10 and 11 to 0. Encoding all of that in
+ * one options object produces a function nobody can read and whose branches
+ * are individually untested. Each detector applies its own documented final
+ * rule to the remainder this module returns. The shared, error-prone part —
+ * the weighted sum itself — is written and tested exactly once.
+ */
+
+/**
+ * Σ(digitᵢ × weightᵢ).
+ *
+ * Returns `null` when there are fewer weights than digits, which is always a
+ * programming error rather than bad user input. Extra trailing weights are
+ * ignored so a scheme can declare the full weight vector including the check
+ * position and pass only the payload digits.
+ */
+export function weightedSum(digits: readonly number[], weights: readonly number[]): number | null {
+  if (weights.length < digits.length) return null;
+  let sum = 0;
+  for (let i = 0; i < digits.length; i++) {
+    sum += digits[i]! * weights[i]!;
+  }
+  return sum;
+}
+
+/**
+ * Σ(digitᵢ × weightᵢ) mod `modulus`.
+ *
+ * Returns `null` on a weight/digit length mismatch or a non-positive modulus.
+ */
+export function weightedMod(
+  digits: readonly number[],
+  weights: readonly number[],
+  modulus: number,
+): number | null {
+  if (modulus <= 0) return null;
+  const sum = weightedSum(digits, weights);
+  if (sum === null) return null;
+  return sum % modulus;
+}
+
+/**
+ * Σ(digitᵢ × weightᵢ) mod `modulus`, where the weight for position i is
+ * generated by a function rather than read from a table.
+ *
+ * Several schemes define weights positionally — China's RIC uses 2^(n-i) mod
+ * 11, ISO 7064 uses a running product — where writing the vector out would be
+ * both longer and easier to get wrong than stating the rule.
+ */
+export function weightedModBy(
+  digits: readonly number[],
+  weightAt: (index: number) => number,
+  modulus: number,
+): number | null {
+  if (modulus <= 0) return null;
+  let sum = 0;
+  for (let i = 0; i < digits.length; i++) {
+    sum += digits[i]! * weightAt(i);
+  }
+  return sum % modulus;
+}
+
+/**
+ * A cycling weight vector: position i uses `weights[i % weights.length]`.
+ *
+ * Used by schemes whose weights repeat over a long identifier (Russia's INN
+ * and SNILS, several VAT formats) rather than being listed per position.
+ */
+export function cyclicWeightedMod(
+  digits: readonly number[],
+  weights: readonly number[],
+  modulus: number,
+): number | null {
+  if (modulus <= 0 || weights.length === 0) return null;
+  let sum = 0;
+  for (let i = 0; i < digits.length; i++) {
+    sum += digits[i]! * weights[i % weights.length]!;
+  }
+  return sum % modulus;
+}
+
+/**
+ * The common "complement" closing rule: `(modulus − (sum mod modulus)) mod
+ * modulus`.
+ *
+ * Extracted because getting the outer `mod` wrong is the classic bug — when
+ * the remainder is 0 the complement must be 0, not `modulus`.
+ */
+export function complement(sum: number, modulus: number): number {
+  return (modulus - (sum % modulus)) % modulus;
+}

package/src/context/documentProfile.ts

@@ -0,0 +1,293 @@
+/**
+ * Stage 3, DOCUMENT TYPE.
+ *
+ * SPEC.md: "DOCUMENT TYPE — detect whether input is prose, source code (and
+ * which language), JSON, YAML, CSV, a log dump, a markdown table, or an email
+ * thread. Each mode shifts weights. Code mode raises secret sensitivity and
+ * lowers person-name sensitivity, since identifiers in code are usually not
+ * people."
+ *
+ * Format detection is evidence-scored rather than first-match: several forms
+ * genuinely overlap (a YAML file is a superset of many key/value shapes, a CV
+ * in prose is full of "Label: value" lines that look like YAML), so each
+ * candidate format accumulates evidence and the strongest wins. The evidence
+ * strings are kept and reported, which is what makes a misclassification
+ * diagnosable instead of mysterious.
+ *
+ * The DOMAIN axis is deliberately separate from format — see the note on
+ * `DocumentDomain`. A medical record can arrive as prose, CSV, or JSON.
+ */
+
+import { buildStructureIndex, type StructureIndex } from './structure.js';
+import { foldForMatch } from './triggers.js';
+import type { DocumentDomain, DocumentFormat, DocumentProfile } from './types.js';
+
+/** Domain terminology, keyed by domain then language. Supplied by the data package. */
+export type DomainLexicon = Readonly<Partial<Record<Exclude<DocumentDomain, 'general'>, readonly string[]>>>;
+
+/** Minimum distinct domain-term hits before a domain is claimed over 'general'. */
+const MIN_DOMAIN_HITS = 2;
+
+/** Fraction of non-empty lines a form must claim to be considered dominant. */
+const DOMINANT_LINE_RATIO = 0.34;
+
+interface FormatScore {
+  format: DocumentFormat;
+  score: number;
+  evidence: string[];
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Line-level probes
+// ─────────────────────────────────────────────────────────────────────────────
+
+const TABLE_SEPARATOR = /^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$/;
+/** ISO-8601, syslog, and bracketed-timestamp line starts. */
+const LOG_TIMESTAMP =
+  /^\s*[[(]?(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}|[A-Z][a-z]{2} {1,2}\d{1,2} \d{2}:\d{2}:\d{2})/;
+const LOG_LEVEL = /\b(TRACE|DEBUG|INFO|WARN|WARNING|ERROR|FATAL|CRITICAL)\b/;
+/**
+ * Keywords that are strong evidence of source code.
+ *
+ * Deliberately excludes words that are ordinary English: `new`, `for`, `from`,
+ * `if`, `else`, `while`, `return`, `class`, `try`, `throw`, `package` and
+ * `public` all appear constantly in prose. Including them classified "everyone
+ * agreed to the new schedule" as source code, and code mode shifts PERSON
+ * sensitivity down — so the cost of a loose keyword list is missed names.
+ */
+const CODE_KEYWORD =
+  /(^|\s)(const|let|var|function|def|elif|lambda|func|fn|impl|struct|async|await|=>|null|undefined|nullptr|println|printf|console\.log|System\.out)\b/;
+const CODE_PUNCTUATION = /(=>|::|->|\)\s*\{|\{\s*$|;\s*$|\)\s*;)/;
+const COMMENT_LINE = /^\s*(\/\/|#|\/\*|\*|--)/;
+/** RFC-822-style headers that mark an email thread. */
+const EMAIL_HEADER = /^\s*(From|To|Cc|Bcc|Subject|Date|Sent|Reply-To)\s*:\s+\S/i;
+const EMAIL_QUOTE = /^\s*>+\s?\S/;
+
+function nonEmptyLines(text: string): string[] {
+  return text.split('\n').filter((l) => l.trim().length > 0);
+}
+
+function ratio(count: number, total: number): number {
+  return total === 0 ? 0 : count / total;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Format scoring
+// ─────────────────────────────────────────────────────────────────────────────
+
+function scoreJson(text: string, lines: readonly string[], scores: FormatScore[]): void {
+  const trimmed = text.trim();
+  const braced = /^[[{]/.test(trimmed) && /[\]}]$/.test(trimmed);
+  const keyLines = lines.filter((l) => /^\s*"[^"]+"\s*:/.test(l)).length;
+  const keyRatio = ratio(keyLines, lines.length);
+  if (!braced && keyRatio < DOMINANT_LINE_RATIO) return;
+
+  const evidence: string[] = [];
+  let score = 0;
+  if (braced) {
+    score += 2;
+    evidence.push('wrapped in braces/brackets');
+  }
+  if (keyRatio >= DOMINANT_LINE_RATIO) {
+    score += 2 + keyRatio;
+    evidence.push(`${Math.round(keyRatio * 100)}% quoted-key lines`);
+  }
+  scores.push({ format: 'json', score, evidence });
+}
+
+function scoreYaml(lines: readonly string[], index: StructureIndex, scores: FormatScore[]): void {
+  const yamlSlots = index.slots.filter((s) => s.kind === 'yaml').length;
+  const yamlRatio = ratio(yamlSlots, lines.length);
+  const documentMarker = lines.some((l) => /^---\s*$/.test(l));
+  const indented = lines.filter((l) => /^\s+\S/.test(l)).length;
+  if (yamlRatio < DOMINANT_LINE_RATIO && !documentMarker) return;
+
+  const evidence: string[] = [];
+  let score = 0;
+  if (yamlRatio >= DOMINANT_LINE_RATIO) {
+    score += 1.5 + yamlRatio;
+    evidence.push(`${Math.round(yamlRatio * 100)}% identifier-key lines`);
+  }
+  if (documentMarker) {
+    score += 1;
+    evidence.push('--- document marker');
+  }
+  if (indented > 0) {
+    score += 0.5;
+    evidence.push('indented block structure');
+  }
+  scores.push({ format: 'yaml', score, evidence });
+}
+
+function scoreCsv(lines: readonly string[], scores: FormatScore[]): void {
+  if (lines.length < 2) return;
+  const counts = lines.map((l) => (l.match(/,/g) ?? []).length);
+  const first = counts[0];
+  if (first === undefined || first < 1) return;
+  const consistent = counts.filter((c) => c === first).length;
+  const consistentRatio = ratio(consistent, counts.length);
+  if (consistentRatio < 0.8) return;
+
+  scores.push({
+    format: 'csv',
+    score: 2 + first * 0.1 + consistentRatio,
+    evidence: [`${consistent}/${counts.length} lines with ${first} commas`],
+  });
+}
+
+function scoreMarkdownTable(lines: readonly string[], scores: FormatScore[]): void {
+  const separators = lines.filter((l) => TABLE_SEPARATOR.test(l) && l.includes('-')).length;
+  if (separators === 0) return;
+  const piped = lines.filter((l) => l.includes('|')).length;
+  scores.push({
+    format: 'markdown-table',
+    score: 2.5 + ratio(piped, lines.length),
+    evidence: [`${separators} table separator row(s)`, `${piped} pipe-delimited lines`],
+  });
+}
+
+function scoreLog(lines: readonly string[], scores: FormatScore[]): void {
+  const stamped = lines.filter((l) => LOG_TIMESTAMP.test(l)).length;
+  const levelled = lines.filter((l) => LOG_LEVEL.test(l)).length;
+  const stampRatio = ratio(stamped, lines.length);
+  if (stampRatio < DOMINANT_LINE_RATIO && ratio(levelled, lines.length) < DOMINANT_LINE_RATIO) return;
+
+  const evidence: string[] = [];
+  let score = 1;
+  if (stampRatio > 0) {
+    score += 1.5 + stampRatio;
+    evidence.push(`${Math.round(stampRatio * 100)}% timestamped lines`);
+  }
+  if (levelled > 0) {
+    score += ratio(levelled, lines.length);
+    evidence.push(`${levelled} lines with a log level`);
+  }
+  scores.push({ format: 'log', score, evidence });
+}
+
+function scoreCode(lines: readonly string[], scores: FormatScore[]): void {
+  const keyworded = lines.filter((l) => CODE_KEYWORD.test(l)).length;
+  const punctuated = lines.filter((l) => CODE_PUNCTUATION.test(l)).length;
+  const commented = lines.filter((l) => COMMENT_LINE.test(l)).length;
+  // Punctuation is required, not merely additive. Prose can borrow a keyword
+  // but it does not end lines with `;`, `{` or `);` — that is what separates
+  // a sentence containing "function" from a line of code.
+  if (punctuated === 0) return;
+  const signal = ratio(keyworded + punctuated, lines.length);
+  if (signal < 0.25) return;
+
+  const evidence: string[] = [`${keyworded} keyword lines`, `${punctuated} code-punctuation lines`];
+  if (commented > 0) evidence.push(`${commented} comment lines`);
+  scores.push({ format: 'code', score: 1 + signal * 2, evidence });
+}
+
+function scoreEmail(lines: readonly string[], scores: FormatScore[]): void {
+  const headers = lines.filter((l) => EMAIL_HEADER.test(l)).length;
+  const quoted = lines.filter((l) => EMAIL_QUOTE.test(l)).length;
+  if (headers < 2 && quoted === 0) return;
+
+  const evidence: string[] = [];
+  let score = 0;
+  if (headers >= 2) {
+    score += 2 + headers * 0.2;
+    evidence.push(`${headers} message headers`);
+  }
+  if (quoted > 0) {
+    score += 0.5;
+    evidence.push(`${quoted} quoted-reply lines`);
+  }
+  scores.push({ format: 'email', score, evidence });
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Domain scoring
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Does `text` contain `term` as a whole word?
+ *
+ * Plain substring matching found "lan" inside "balance" and counted it as
+ * financial vocabulary. Scripts that do not space their words cannot use a
+ * boundary test, so for those the substring match stands — which is correct,
+ * since that is how those scripts are written.
+ */
+function containsAsWord(text: string, term: string): boolean {
+  if (!/^[\p{L}\p{M}\p{N} '-]+$/u.test(term)) return text.includes(term);
+
+  let at = text.indexOf(term);
+  while (at !== -1) {
+    const before = at === 0 ? '' : text[at - 1] ?? '';
+    const after = text[at + term.length] ?? '';
+    if (!/\p{L}/u.test(before) && !/\p{L}/u.test(after)) return true;
+    at = text.indexOf(term, at + 1);
+  }
+  return false;
+}
+
+function detectDomain(
+  folded: string,
+  lexicon: DomainLexicon,
+): { domain: DocumentDomain; evidence: string[] } {
+  let best: { domain: DocumentDomain; evidence: string[] } = { domain: 'general', evidence: [] };
+  let bestCount = 0;
+
+  for (const [domain, terms] of Object.entries(lexicon)) {
+    const hits: string[] = [];
+    for (const term of terms ?? []) {
+      const foldedTerm = foldForMatch(term);
+      if (foldedTerm.length > 0 && containsAsWord(folded, foldedTerm)) hits.push(foldedTerm);
+    }
+    if (hits.length > bestCount) {
+      bestCount = hits.length;
+      best = { domain: domain as DocumentDomain, evidence: hits.slice(0, 8) };
+    }
+  }
+
+  return bestCount >= MIN_DOMAIN_HITS ? best : { domain: 'general', evidence: [] };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Entry point
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Classify a document's format and subject domain.
+ *
+ * `structureIndex` is accepted rather than rebuilt so callers that already
+ * have one (the analyzer does) pay for it once.
+ */
+export function profileDocument(
+  text: string,
+  options: { readonly structureIndex?: StructureIndex; readonly domainLexicon?: DomainLexicon } = {},
+): DocumentProfile {
+  const lines = nonEmptyLines(text);
+  const index = options.structureIndex ?? buildStructureIndex(text);
+
+  const scores: FormatScore[] = [];
+  scoreMarkdownTable(lines, scores);
+  scoreJson(text, lines, scores);
+  scoreCsv(lines, scores);
+  scoreYaml(lines, index, scores);
+  scoreLog(lines, scores);
+  scoreCode(lines, scores);
+  scoreEmail(lines, scores);
+
+  scores.sort((a, b) => b.score - a.score);
+  const winner = scores[0];
+
+  const { domain, evidence: domainEvidence } = detectDomain(
+    foldForMatch(text),
+    options.domainLexicon ?? {},
+  );
+
+  if (winner === undefined) {
+    return {
+      format: lines.length > 0 ? 'prose' : 'unknown',
+      domain,
+      formatEvidence: lines.length > 0 ? ['no structural form matched'] : ['empty document'],
+      domainEvidence,
+    };
+  }
+
+  return { format: winner.format, domain, formatEvidence: winner.evidence, domainEvidence };
+}