@arela/uploader 1.1.3 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/scoring/scoring-engine.js

@@ -0,0 +1,251 @@
+/**
+ * Scoring-based document classification engine (PROTOTYPE).
+ *
+ * Replaces the "first-match-wins" selection in `document-type-shared.js`
+ * (`extractDocumentFields`) with "best-match": every applicable matcher is
+ * scored by the weight of the clues it satisfies, and the highest score wins.
+ * This removes the order-dependent registration that the current registry
+ * relies on (e.g. `factura_inter_agencia` MUST be evaluated before
+ * `facturas_comerciales`) — precedence now lives in clue weights / `required`
+ * / `negative` flags, not in array order.
+ *
+ * Matcher shape (see `matchers-seed.js`):
+ *   {
+ *     documentType, extensions[], minScore, priority,
+ *     clues: [{ kind, pattern, flags?, weight=1, group?, required=false, negative=false }],
+ *     qualify?: [{ <group>: minCount, ... }, ...],   // OR-of-ANDs gate
+ *     extractors, resolveType?, extractNumPedimento?, extractPedimentoYear?
+ *   }
+ *
+ * Two separable concerns:
+ *   - QUALIFICATION (does this matcher apply at all?): `required`/`negative`
+ *     clues plus an optional `qualify` rule set. `qualify` is a list of
+ *     alternative rules (OR); a rule is a map of `group -> minimum matched
+ *     clues` (AND across its entries). This expresses the grouped boolean gates
+ *     of the original matchers, e.g. `(cfdi>=2) OR (invoice>=1 AND customs>=1)`.
+ *     When `qualify` is absent, the gate falls back to `score >= minScore`.
+ *   - RANKING (which qualifying matcher wins?): always the weighted sum of
+ *     matched clues (`score`), tie-broken by fraction -> priority -> type.
+ *
+ * Selection only depends on clues/qualify. The winning matcher's `extractors` /
+ * `resolveType` / `extractNumPedimento` / `extractPedimentoYear` run AFTER
+ * selection with the same post-processing as `extractDocumentFields`, so a
+ * side-by-side comparison isolates the selection change.
+ */
+import path from 'path';
+
+import { FieldResult } from '../document-type-shared.js';
+
+// Compile cache for string patterns (from DB matchers): stable across every
+// document in a run, so compile once instead of per document. Invalid patterns
+// cache as null and are treated as a non-match (parity with the TS engine).
+const regexCache = new Map();
+const REGEX_CACHE_MAX = 5000;
+
+function toRegExp(clue) {
+  if (clue.pattern instanceof RegExp) {
+    // Local-seed patterns are already compiled; reset lastIndex so a g/y flag
+    // can't make repeated .test() calls stateful across documents.
+    clue.pattern.lastIndex = 0;
+    return clue.pattern;
+  }
+  const safeFlags = (clue.flags ?? '').replace(/[gy]/g, '');
+  const key = `${safeFlags} ${clue.pattern}`;
+  let re = regexCache.get(key);
+  if (re === undefined) {
+    try {
+      re = new RegExp(clue.pattern, safeFlags);
+    } catch {
+      re = null;
+    }
+    if (regexCache.size >= REGEX_CACHE_MAX) regexCache.clear();
+    regexCache.set(key, re);
+  }
+  return re;
+}
+
+// Bound the text a single regex runs on so an allowed (quadratic) pattern
+// can't blow up on a megabyte-scale adversarial string. Real documents can
+// legitimately exceed the cap (edocument XMLs embed multi-MB base64 PDFs with
+// signature tags AFTER the blob), so instead of truncating we sample a HEAD +
+// TAIL window — signatures live at the extremes, the blob in the middle.
+// (Parity with the API TS engine.)
+const MATCH_HEAD_CAP = 1_000_000;
+const MATCH_TAIL_CAP = 262_144;
+
+function clueTarget(clue, ctx) {
+  // FILENAME_REGEX tests the file name; every other kind tests the content.
+  const raw =
+    clue.kind === 'FILENAME_REGEX' ? (ctx.fileName ?? '') : (ctx.source ?? '');
+  if (raw.length <= MATCH_HEAD_CAP + MATCH_TAIL_CAP) return raw;
+  return `${raw.slice(0, MATCH_HEAD_CAP)}\n${raw.slice(-MATCH_TAIL_CAP)}`;
+}
+
+/**
+ * Score a single matcher against a document context.
+ * @returns {null} when the matcher does not apply (extension mismatch),
+ *          `{ disqualified: true, reason }` when a `required`/`negative` clue
+ *          rules it out, otherwise a scored result object.
+ */
+export function scoreMatcher(matcher, ctx) {
+  const ext = (ctx.extension ?? '').toLowerCase();
+  if (
+    Array.isArray(matcher.extensions) &&
+    matcher.extensions.length > 0 &&
+    ext &&
+    !matcher.extensions.includes(ext)
+  ) {
+    return null;
+  }
+
+  let matchedWeight = 0;
+  let totalWeight = 0;
+  const matchedClues = [];
+  const groupCounts = {};
+
+  for (const clue of matcher.clues ?? []) {
+    const weight = clue.weight ?? 1;
+    const re = toRegExp(clue);
+    const hit = re ? re.test(clueTarget(clue, ctx)) : false;
+
+    if (clue.negative) {
+      if (hit) {
+        return {
+          documentType: matcher.documentType,
+          disqualified: true,
+          reason: `negative:${clue.pattern}`,
+        };
+      }
+      continue;
+    }
+
+    if (clue.required && !hit) {
+      return {
+        documentType: matcher.documentType,
+        disqualified: true,
+        reason: `required-missing:${clue.pattern}`,
+      };
+    }
+
+    totalWeight += weight;
+    if (hit) {
+      matchedWeight += weight;
+      matchedClues.push(clue);
+      if (clue.group) {
+        groupCounts[clue.group] = (groupCounts[clue.group] ?? 0) + 1;
+      }
+    }
+  }
+
+  // Qualification gate: `qualify` rules (OR-of-ANDs over group counts) when
+  // present, otherwise the weighted-score threshold.
+  const passed = Array.isArray(matcher.qualify)
+    ? matcher.qualify.some((rule) =>
+        Object.entries(rule).every(
+          ([group, min]) => (groupCounts[group] ?? 0) >= min,
+        ),
+      )
+    : matchedWeight >= (matcher.minScore ?? 1);
+
+  return {
+    documentType: matcher.documentType,
+    matcher,
+    disqualified: false,
+    score: matchedWeight,
+    totalWeight,
+    fraction: totalWeight > 0 ? matchedWeight / totalWeight : 0,
+    priority: matcher.priority ?? 0,
+    passed,
+    matchedClues,
+    groupCounts,
+  };
+}
+
+/**
+ * All qualifying candidates, sorted best-first.
+ * Order: score desc → fraction desc → priority desc → documentType (stable).
+ */
+export function scoreAll(matchers, ctx) {
+  const candidates = [];
+  for (const matcher of matchers) {
+    const result = scoreMatcher(matcher, ctx);
+    if (!result || result.disqualified || !result.passed) continue;
+    candidates.push(result);
+  }
+  candidates.sort(
+    (a, b) =>
+      b.score - a.score ||
+      b.fraction - a.fraction ||
+      b.priority - a.priority ||
+      String(a.documentType).localeCompare(String(b.documentType)),
+  );
+  return candidates;
+}
+
+export function selectBestMatch(matchers, ctx) {
+  return scoreAll(matchers, ctx)[0] ?? null;
+}
+
+/**
+ * Full classification: pick the best matcher, then run ITS extractors /
+ * resolveType / pedimento helpers. Post-selection logic mirrors
+ * `extractDocumentFields` so a comparison isolates the selection change.
+ *
+ * @returns {{ detectedType, fields, detectedPedimento, detectedPedimentoYear,
+ *             winner, candidates }}
+ */
+export function classifyDocument(matchers, { source, extension, filePath }) {
+  const ctx = {
+    source,
+    extension,
+    fileName: filePath ? path.basename(filePath) : '',
+  };
+  const candidates = scoreAll(matchers, ctx);
+  const winner = candidates[0] ?? null;
+
+  if (!winner) {
+    return {
+      detectedType: null,
+      fields: [],
+      detectedPedimento: null,
+      detectedPedimentoYear: null,
+      winner: null,
+      candidates,
+    };
+  }
+
+  const def = winner.matcher;
+  const fields = [];
+  for (const extractor of def.extractors ?? []) {
+    try {
+      fields.push(extractor.extract(source));
+    } catch {
+      fields.push(new FieldResult(extractor.field, false, null));
+    }
+  }
+
+  const resolvedType = def.resolveType
+    ? def.resolveType(fields, filePath)
+    : def.documentType;
+  const pedimento = def.extractNumPedimento
+    ? def.extractNumPedimento(source, fields, filePath)
+    : null;
+  const year = def.extractPedimentoYear
+    ? def.extractPedimentoYear(source, fields, filePath)
+    : null;
+
+  // Backfill numPedimento as a field (same as extractDocumentFields) so
+  // downstream consumers (composeArelaPath) see a consistent shape.
+  if (pedimento && !fields.some((f) => f.name === 'numPedimento')) {
+    fields.push(new FieldResult('numPedimento', true, pedimento));
+  }
+
+  return {
+    detectedType: resolvedType,
+    fields,
+    detectedPedimento: pedimento,
+    detectedPedimentoYear: year,
+    winner,
+    candidates,
+  };
+}