clearotron 0.3.3 → 0.4.0-beta.1

package/providers/_shared/answer-memory.mjs

@@ -0,0 +1,199 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026 Cordillera Sàrl. Additional terms under section 7 of the AGPL-3.0 apply — see ADDITIONAL-TERMS.md
+// answer-memory.mjs — a run remembers the register's answer to a question it has already asked.
+//
+// WHY. A register that bills per request charges again for every repeat, and a run repeats itself: a
+// repair re-sends a whole axis, a re-attempt re-asks questions that already answered, a proposal is
+// asked again later in the run. The memory sits at the provider's one HTTP chokepoint, so every caller
+// is covered without each of them learning to share.
+//
+// WHERE IT LIVES. In the run's own `_driver/` folder, because the requests come from two kinds of
+// process — the long-lived driver and the tool servers a stage spawns — and the repeats cross between
+// them. Each already knows the run's record log: the driver passes it on `tctx.recordLog`, and a
+// spawned server has it in CLEAROTRON_REGISTER_RECORD_LOG. The memory is the folder beside that file,
+// so nothing new is handed to either.
+//
+// ONE ATTEMPT. The driver clears the folder when an attempt starts and removes it when the attempt
+// ends, so a resume asks the register again. A folder whose attempt began more than a day ago is
+// ignored, for an attempt that died without cleaning up. A process that finds no folder — a bare
+// probe, a test, a register with no run — asks the register exactly as it always has.
+//
+// EXCEPT A RECORD, WHICH IS HELD FOR THE WHOLE RUN. Clarivate's record reuse (`heldRecords` in its core)
+// reads the run's record log, not this folder, and that log keeps every attempt's records. So a run
+// resumed days later screens a record from the copy an earlier attempt fetched, and its row says what the
+// register said then. That is the ruling on this memory, that a run never fetches the same record twice;
+// the questions above are asked again on a resume, the records are not.
+//
+// THREE MODES, fixed when the attempt starts from the register's own switch (ANSWER_MEMORY_PROVIDERS):
+//   off    nothing is remembered and nothing is written.
+//   watch  every request still goes to the register. Per request, the memory records whether it held
+//          an answer to the identical question, and whether the fresh answer has the same total, the
+//          same record ids and the same order as the one it held. This is the evidence the switch to
+//          `on` waits for.
+//   on     a held answer is returned and the register is not asked.
+//
+// WHAT IS KEPT is the provider's decision, because only the provider knows which bodies are complete
+// answers (see `rememberableAnswer` in the Signa core). An answer that points at a next page is held
+// only for NEXT_PAGE_FRESH_MS: the cursor it carries is the register's, and nothing says how long the
+// register honours it.
+//
+// NEVER THROWS. Every failure here reads as "nothing held" or "not stored", which is the behaviour
+// the run had before the memory existed.
+
+import { appendFileSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { createHash } from "node:crypto";
+import { gunzipSync, gzipSync } from "node:zlib";
+import { driverDir } from "../../shared/driver-dir.mjs";
+import { RUN_RECORD_LOG_FILE } from "./ledger-path.mjs";
+
+export const ANSWER_MEMORY_MODES = Object.freeze(["off", "watch", "on"]);
+
+/**
+ * The registers whose core consults this memory, the environment switch that sets it for each (`env`, the
+ * table shape the configuration audit reads), and the mode a run takes when that switch is unset. Every other register is `off`, whatever any switch says.
+ *
+ * SIGNA starts `off`: it goes `on` only after runs in watch mode show no mismatch, by the owner's ruling.
+ * CLARIVATE starts `on`, with no watch period, also by his ruling: the repeated answers of its past runs
+ * were already compared and matched.
+ */
+export const ANSWER_MEMORY_PROVIDERS = Object.freeze({
+  signa: Object.freeze({ env: "CLEAROTRON_SIGNA_ANSWER_MEMORY", defaultMode: "off" }),
+  clarivate: Object.freeze({ env: "CLEAROTRON_CLARIVATE_ANSWER_MEMORY", defaultMode: "on" }),
+});
+export const ANSWER_MEMORY_SWITCH = ANSWER_MEMORY_PROVIDERS.signa.env;
+export const ANSWER_MEMORY_DIR = "register-answers";
+export const ANSWER_WATCH_LOG = "register-answer-watch.jsonl";
+const ATTEMPT_FILE = "attempt.json";
+export const ATTEMPT_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+// One hour: a next-page link has been seen to work an hour after the register gave it, and nothing says it
+// works for longer. A held answer older than this is simply asked again. A park ends the attempt anyway,
+// and a resumed run starts with an empty memory, so no link survives a park.
+export const NEXT_PAGE_FRESH_MS = 60 * 60 * 1000;
+
+/**
+ * The mode a register's switch asks for. Unset, or naming none of the three, is that register's default,
+ * and a value that named none of them is returned as `unknown` so the run can say so.
+ */
+export function answerMemoryMode(env = process.env, provider = "signa") {
+  const spec = ANSWER_MEMORY_PROVIDERS[String(provider ?? "")];
+  if (!spec) return { mode: "off", unknown: null };
+  const raw = String(env?.[spec.env] ?? "").trim().toLowerCase();
+  if (!raw) return { mode: spec.defaultMode, unknown: null };
+  return ANSWER_MEMORY_MODES.includes(raw) ? { mode: raw, unknown: null } : { mode: spec.defaultMode, unknown: raw };
+}
+
+/**
+ * Begin an attempt: whatever an earlier attempt left is removed, and a folder is made only for
+ * `watch` or `on`. Returns the folder, or null when the memory is off. Called by the driver alone.
+ */
+export function startAnswerMemory(runDir, mode, { now = Date.now } = {}) {
+  const dir = driverDir(runDir, ANSWER_MEMORY_DIR);
+  try { rmSync(dir, { recursive: true, force: true }); } catch { /* nothing to clear */ }
+  if (mode !== "watch" && mode !== "on") return null;
+  try {
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, ATTEMPT_FILE), JSON.stringify({ mode, started_ms: now() }) + "\n");
+    return dir;
+  } catch { return null; }
+}
+
+/**
+ * The driver's half of an attempt: the mode this run uses, with the folder begun for it. `applies` is
+ * false for a register that does not use the memory, and then nothing is written at all, so that
+ * register's run folder is exactly what it was. A folder that cannot be made reads as `off`.
+ */
+export function beginAnswerMemory(runDir, provider, { env = process.env, now = Date.now } = {}) {
+  const spec = ANSWER_MEMORY_PROVIDERS[String(provider ?? "")];
+  if (!spec) return { mode: "off", unknown: null, applies: false, switch: null };
+  const { mode, unknown } = answerMemoryMode(env, provider);
+  const dir = startAnswerMemory(runDir, mode, { now });
+  return { mode: dir ? mode : "off", unknown, applies: true, switch: spec.env };
+}
+
+/** End an attempt. The watch log stays; the held answers go. */
+export function endAnswerMemory(runDir) {
+  try { rmSync(driverDir(runDir, ANSWER_MEMORY_DIR), { recursive: true, force: true }); } catch { /* already gone */ }
+}
+
+/**
+ * The run's memory for this request, or null. Found from the run's record log, which is the only
+ * address both kinds of process already hold. The box-global ledger's file has a different name, so a
+ * process pointed at it finds no run and no memory.
+ */
+export function openAnswerMemory(recordLog = null, { env = process.env, now = Date.now } = {}) {
+  try {
+    const log = typeof recordLog === "string" && recordLog.trim()
+      ? recordLog.trim() : String(env?.CLEAROTRON_REGISTER_RECORD_LOG ?? "").trim();
+    if (!log || basename(log) !== RUN_RECORD_LOG_FILE) return null;
+    const dir = join(dirname(log), ANSWER_MEMORY_DIR);
+    const attempt = JSON.parse(readFileSync(join(dir, ATTEMPT_FILE), "utf8"));
+    if (attempt?.mode !== "watch" && attempt?.mode !== "on") return null;
+    if (!Number.isFinite(attempt.started_ms) || now() - attempt.started_ms > ATTEMPT_MAX_AGE_MS) return null;
+    return { mode: attempt.mode, dir, watchLog: join(dirname(log), ANSWER_WATCH_LOG) };
+  } catch { return null; }
+}
+
+/** The identity of a question: everything that decides the answer, and nothing that does not. */
+export function answerKey(question) {
+  return createHash("sha256").update(JSON.stringify(question)).digest("hex");
+}
+
+/**
+ * The held answer to a question, or null. An answer that points at a next page is returned `stale`
+ * once NEXT_PAGE_FRESH_MS has passed: it is not served, and the watch log records that it was held, so
+ * the log shows what the rule costs.
+ */
+export function recallAnswer(mem, key, { now = Date.now } = {}) {
+  try {
+    const held = JSON.parse(gunzipSync(readFileSync(join(mem.dir, `${key}.json.gz`))).toString("utf8"));
+    const age = now() - held.stored_ms;
+    return { ...held, age_ms: age, stale: held?.summary?.next_page === true && age > NEXT_PAGE_FRESH_MS };
+  } catch { return null; }
+}
+
+/**
+ * Keep an answer. Written whole and renamed into place, so a process reading at the same moment sees
+ * the old state or the new one and never half of either. Never makes the folder: an attempt that has
+ * ended has no memory to write into.
+ */
+export function rememberAnswer(mem, key, entry, { now = Date.now } = {}) {
+  const dest = join(mem.dir, `${key}.json.gz`);
+  const tmp = `${dest}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
+  try {
+    writeFileSync(tmp, gzipSync(JSON.stringify({ ...entry, stored_ms: now() })));
+    renameSync(tmp, dest);
+    return true;
+  } catch {
+    try { rmSync(tmp, { force: true }); } catch { /* nothing written */ }
+    return false;
+  }
+}
+
+/** Drop a held answer, so the next identical question goes to the register. Never throws. */
+export function forgetAnswer(mem, key) {
+  try { rmSync(join(mem.dir, `${key}.json.gz`), { force: true }); return true; } catch { return false; }
+}
+
+/** One line per request in the watch log. Small, so concurrent appends from two processes stay whole. */
+export function noteAnswer(mem, row) {
+  try { appendFileSync(mem.watchLog, JSON.stringify(row) + "\n"); } catch { /* the log is evidence, never a gate */ }
+}
+
+/**
+ * Three separate answers, because they mean different things. A different total or a different set
+ * of ids is a different answer, and the memory must not go live while either happens. A different
+ * order of the same ids is a register that breaks ties differently from one request to the next: the
+ * memory returns one of the orders the register itself gives.
+ */
+export function compareAnswers(held, fresh) {
+  const a = Array.isArray(held?.ids) ? held.ids : [];
+  const b = Array.isArray(fresh?.ids) ? fresh.ids : [];
+  const sa = [...a].sort(), sb = [...b].sort();
+  const same_ids = sa.length === sb.length && sa.every((v, i) => v === sb[i]);
+  return {
+    same_total: (held?.total ?? null) === (fresh?.total ?? null),
+    same_ids,
+    same_order: same_ids && a.every((v, i) => v === b[i]),
+  };
+}

package/providers/_shared/ledger-path.mjs

@@ -305,7 +305,7 @@ export function retiredGlobalRecordLogNotice(env = process.env) {
   if (announced.has(key)) return null;
   announced.add(key);
   const bytes = sizeOf(r.path);
-  return `[ledger] the box-global record log ${r.path}${bytes === null ? "" : ` (${bytes} bytes)`} is RETIRED (#743) — `
+  return `[ledger] the box-global record log ${r.path}${bytes === null ? "" : ` (${bytes} bytes)`} is RETIRED — `
     + `register response bodies now live in each run's _driver/${RUN_RECORD_LOG_FILE} and are archived and purged with the run. `
     + `Nothing writes to or reads this file any more; archive it once (mv it aside) at your convenience.`;
 }

package/providers/_shared/ledger.mjs

@@ -135,15 +135,21 @@ export function makeLedger(provider) {
 //
 // Run-scoped ledgers only. The box-wide fallback file holds many runs, and a record one run fetched is not
 // a record another run holds.
-const RUN_INDEXES = new Map();   // dest → { offset, byTarget: Map<key, { hash, refreshed, ts }> }
+const RUN_INDEXES = new Map();   // dest → { offset, byTarget: Map<key, { hash, refreshed, ts, start, len }>, byGuid: Map<guid, key> }
 const targetKey = (t) => String(t ?? "").toLowerCase();   // the key every reader of this file matches on
 const bodyHash = (b) => createHash("sha1").update(JSON.stringify(b ?? null)).digest("hex");
+// The record's own id: the last segment of `/mark/<office>/<id>`. The office segment is a hint that can
+// differ between two answers for the same record, so a lookup by record keys on this and never on it.
+const guidOf = (t) => { const k = targetKey(t); const i = k.lastIndexOf("/"); return i >= 0 ? k.slice(i + 1) : k; };
 
-function indexRow(ix, line) {
+// `start` and `len` are the line's byte range, so one record can be read back without reading the file.
+function indexRow(ix, line, start = null, len = null) {
   if (!line.trim()) return;
   try {
     const r = JSON.parse(line);
-    ix.byTarget.set(targetKey(r?.target), { hash: bodyHash(r?.body), refreshed: r?.refreshed === true, ts: r?.ts ?? null });
+    const key = targetKey(r?.target);
+    ix.byTarget.set(key, { hash: bodyHash(r?.body), refreshed: r?.refreshed === true, ts: r?.ts ?? null, start, len });
+    if (key) ix.byGuid.set(guidOf(key), key);
   } catch { /* a torn or foreign line indexes nothing, and is left where it is */ }
 }
 
@@ -154,7 +160,7 @@ function indexOf(dest) {
   let ix = RUN_INDEXES.get(dest);
   // A replacement elsewhere renames a NEW file over this one: same path, different inode, and an offset
   // into the old file means nothing in the new one. Re-read from the start.
-  if (!ix || size < ix.offset || ix.ino !== ino) { ix = { offset: 0, ino, byTarget: new Map() }; RUN_INDEXES.set(dest, ix); }
+  if (!ix || size < ix.offset || ix.ino !== ino) { ix = { offset: 0, ino, byTarget: new Map(), byGuid: new Map() }; RUN_INDEXES.set(dest, ix); }
   if (size <= ix.offset) return ix;
   const fd = openSync(dest, "r");
   try {
@@ -167,9 +173,14 @@ function indexOf(dest) {
       if (n <= 0) break;
       pos += n;
       const data = carry.length ? Buffer.concat([carry, buf.subarray(0, n)]) : buf.subarray(0, n);
+      const base = pos - data.length;   // the file offset of data[0]
       const last = data.lastIndexOf(0x0a);
       if (last < 0) { carry = data; continue; }
-      for (const line of data.subarray(0, last).toString("utf8").split("\n")) indexRow(ix, line);
+      for (let at = 0; at <= last;) {
+        const nl = data.indexOf(0x0a, at);
+        indexRow(ix, data.subarray(at, nl).toString("utf8"), base + at, nl - at);
+        at = nl + 1;
+      }
       carry = data.subarray(last + 1);   // a line still being written waits for the next look
       ix.offset = pos - carry.length;
     }
@@ -177,6 +188,37 @@ function indexOf(dest) {
   return ix;
 }
 
+/**
+ * The record bodies this run already holds, for the given record ids: Map<id, body>, lowercased ids.
+ *
+ * Read off the run's own record log, which every process that fetches a record for the run writes to,
+ * so a record the driver fetched is held for a tool server and the other way round. A run-scoped log only:
+ * the box-wide fallback file holds many runs, and one run's record is not another's. Anything that cannot
+ * be read reads as not held, which costs a fetch and never an answer.
+ */
+export function heldRecordBodies(dest, ids) {
+  const out = new Map();
+  if (typeof dest !== "string" || basename(dest) !== RUN_RECORD_LOG_FILE) return out;
+  let ix;
+  try { ix = indexOf(dest); } catch { return out; }
+  const wanted = [...new Set((ids ?? []).map((g) => String(g ?? "").toLowerCase()).filter(Boolean))]
+    .map((g) => [g, ix.byTarget.get(ix.byGuid.get(g) ?? "")]).filter(([, e]) => e && Number.isInteger(e.start) && Number.isInteger(e.len));
+  if (!wanted.length) return out;
+  let fd;
+  try {
+    fd = openSync(dest, "r");
+    for (const [g, e] of wanted) {
+      try {
+        const buf = Buffer.alloc(e.len);
+        if (readSync(fd, buf, 0, e.len, e.start) !== e.len) continue;
+        const row = JSON.parse(buf.toString("utf8"));
+        if (row?.body && typeof row.body === "object") out.set(g, row.body);
+      } catch { /* a line that moved under us is simply not held */ }
+    }
+  } catch { /* unreadable: nothing held */ } finally { if (fd !== undefined) try { closeSync(fd); } catch { /* closed */ } }
+  return out;
+}
+
 const sleepMs = (ms) => { try { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } catch { /* no sleep available */ } };
 
 /** Run `fn` holding the ledger's lock; false (and `fn` not run) when the lock could not be had in time. */

package/providers/_shared/script-form.mjs

@@ -44,6 +44,25 @@ export const NON_LATIN_RE = /[^\p{Script=Latin}\p{Script=Common}\p{Script=Inheri
 
 export function isNonLatinTerm(term) { return NON_LATIN_RE.test(String(term ?? "")); }
 
+/**
+ * Does the term MIX Latin letters with Greek or Cyrillic ones? `τιmbεr` does; `timber`, `τιμβερ` and
+ * `тимбер` do not, and neither does a term whose only other characters are digits or punctuation, which
+ * belong to no script of their own.
+ *
+ * The look-alike generator in driver/form-neighbourhood.mjs writes such terms: it swaps every letter
+ * that has a Greek or Cyrillic twin and leaves the rest Latin. A register may answer that spelling as
+ * if the non-Latin letters were not there, returning marks that share only the Latin remainder, or may
+ * not take it at all. `capabilities.mixedScriptQuery: false` declares that the register cannot search
+ * such a spelling as written, and the form band leaves these spellings out there and lists them as not
+ * searched.
+ */
+const LATIN_LETTER_RE = /\p{Script=Latin}/u;
+const GREEK_OR_CYRILLIC_RE = /[\p{Script=Greek}\p{Script=Cyrillic}]/u;
+export function mixesLatinWithGreekOrCyrillic(term) {
+  const s = String(term ?? "");
+  return LATIN_LETTER_RE.test(s) && GREEK_OR_CYRILLIC_RE.test(s);
+}
+
 /**
  * The declaration-driven policy. Given a provider's capability contract and the MARK TERMS a query is
  * about to carry, return the plain-English gap reason — or null when the slice may be dispatched.
@@ -121,15 +140,15 @@ export function romanizationSpellings(value) {
  *
  * Two folds fused, and the seam between them is the whole point:
  *   - LATIN text is accent-folded: lowercase, NFKD, then combining marks stripped — but ONLY the
- *     marks sitting on a Latin base letter — then punctuation/whitespace dropped. "Tikí-Slush" and
- *     "CORAL FREEZE" collide, exactly as the owner-formative dedup has always wanted.
+ *     marks sitting on a Latin base letter — then punctuation/whitespace dropped. "Wavó-Slush" and
+ *     "WAVO SLUSH" collide, exactly as the owner-formative dedup has always wanted.
  *   - NON-LATIN text keeps every combining mark. In most non-Latin scripts a combining mark is not
  *     an accent, it is part of WHICH LETTER this is: Japanese dakuten/handakuten (タ=ta vs ダ=da),
  *     Thai vowel signs, Devanagari matras, Arabic diacritics. A fold that strips them (the previous
  *     shape of this function: bare NFKD + strip all \p{M}) collapses MARK-DISTINGUISHED SIBLINGS
- *     into one key: ティキスラッシュ
- *     (TIKI SURASSHU) and ディキスラッシュ (DIKI SURASSHU) keyed identically, so the romanisation
- *     lookup handed one sibling the OTHER's romanisation and the dictated DIKI form was never
+ *     into one key: ワボスラッシュ
+ *     (WABO SURASSHU) and ワホスラッシュ (WAHO SURASSHU) keyed identically, so the romanisation
+ *     lookup handed one sibling the OTHER's romanisation and the dictated WAHO form was never
  *     searched anywhere — a silent wrong-query false clean, the exact class the carriage fix exists
  *     to kill. NFKD is kept for its width folding (half-width ｶﾞ and full-width ガ are the same
  *     letter); only the mark-stripping is script-scoped.

package/providers/_shared/term-shape.mjs

@@ -6,12 +6,12 @@
 // Two defect classes shipped as SILENT CLEANS, and both were shape-vs-predicate
 // disagreements nobody checked:
 //
-//   * WILDCARD-UNDER-LITERAL: the frozen plan carried {predicate:"exact", term:"TIKI*"} ×4. Dispatch
+//   * WILDCARD-UNDER-LITERAL: the frozen plan carried {predicate:"exact", term:"WAVO*"} ×4. Dispatch
 //     never inspects term characters on a literal predicate, so the provider searched the star as a
 //     character, found nothing, and the band recorded state:"enumerated", total_hits:0 — a
 //     schema-level confident clean over a slice that was never really searched.
-//   * LABEL-AS-TERM: a frame-diff directive's display label ("Reverse-order TIKI composites
-//     (TROPICAL TIKI, ISLAND TIKI)") was dispatched verbatim as a mark term. Structured transport,
+//   * LABEL-AS-TERM: a frame-diff directive's display label ("Reverse-order WAVO composites
+//     (TROPICAL WAVO, ISLAND WAVO)") was dispatched verbatim as a mark term. Structured transport,
 //     prose value — same nil search, same false clean.
 //
 // This module is the shared detector all four seams call: the plan freeze-lint
@@ -88,7 +88,7 @@ export function termPredicateIssue(term, predicate) {
 // state:"enumerated", total_hits:0 — a false clean, quieter than the failure.
 //
 // The three arms are the ones the issue names, and NOT the bracket: `predicate:"owner"` rows carry
-// parenthesised company names ("Delphi Technologies (BorgWarner Inc.)") and a bracket rule breaks
+// parenthesised company names ("Korphi Technologies (BorgWarner Inc.)") and a bracket rule breaks
 // owner search, which is the very lane this screen protects. Every arm fires at ANY word count.
 //
 //   `**` / `__`     markdown emphasis. No register indexes it; a mark cannot contain it.
@@ -178,7 +178,7 @@ export function termShapeIssue(term) {
  * TWO EXEMPTIONS, AND THEY ARE NOT THE SAME EXEMPTION:
  *
  *   `predicate: "owner"` is exempt from EVERYTHING, first, before any other test. Owner names are
- *   long, prose-shaped and parenthesised by nature ("Delphi Technologies (BorgWarner Inc.)"), they
+ *   long, prose-shaped and parenthesised by nature ("Korphi Technologies (BorgWarner Inc.)"), they
  *   ride their own field, and the cross-check lane mints them — so this is what makes it safe to
  *   screen that lane at all.
  *

package/providers/clarivate/src/capabilities.js

@@ -280,6 +280,17 @@ export const CAPABILITIES = Object.freeze({
   // transliteration is itself a silent zero, where the same term under contains answers). A slice
   // rescued that way is answerable and is never refused; only a native term with no romanisation defers.
   nativeScriptIndex: false,
+  // A term mixing Latin letters with Greek or Cyrillic ones is not searched here. The index holds
+  // non-Latin filings by transliteration only (`nativeScriptIndex` above), so such a term used to be
+  // compiled and then deferred, and a deferred look-alike was counted as a search not completed. `false`
+  // leaves those spellings out of the form band and lists them there as not searched, as on Signa.
+  mixedScriptQuery: false,
+  // A knockout's listing already holds its count. The listing's `/search` asks the count lane's exact
+  // question, with the same body `/count` takes, and returns every matching id, so its length is the
+  // register's total. A knockout therefore lists first and takes the identical and close counts from the
+  // listing (driver/register-count.mjs listingAnswers). A term the listing did not answer, including one
+  // `/search` refuses for exceeding the result ceiling, is counted with `/count` as before.
+  listingAnswersCount: true,
   // No phoneme expansion knob: PHONETIC_WORD_MARK_SPECIFICATION is the whole surface; the client cannot
   // hand it a variant list. (/similarity/word/* — which would be the expansion surface — is genuinely
   // not available on this provider; the endpoint answers 403. Do NOT wire it.)