lazycodex-ai 5.1.45 → 5.1.46

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-claims.mjs

@@ -0,0 +1,89 @@
+// fidelity-claims.mjs - the claim-kind checks of the deterministic fidelity gate.
+//
+// A mark (the data-claim-kind attribute) binds per cited fact: a data-fact element takes the kind
+// of its nearest mark (its own, else the closest marked ancestor), and only that kind is checked
+// against its ledger row (claim-kinds.mjs CLAIM_KINDS); a row that fails is the kind's defect
+// (unsupported_verification for `verified`). A fact under two marks of different kinds is a
+// claim_kind_mismatch, and a mark that owns no fact of its own kind (every fact under it cites
+// nothing or is shadowed by another kind) is its kind's defect: shadowing never satisfies a parent.
+// An unmarked fact citing a row neither fact nor quote accepts (an assumed row) is a
+// claim_kind_mismatch naming the mark it needs. A sentence that matches a kind's unmarked patterns warns (minor, verification_claim_unmarked)
+// unless every fact it cites is marked that kind; a sentence citing no fact warns unless its block
+// is marked. No model, no network.
+
+import { CLAIM_ATTRIBUTE, CLAIM_KINDS, claimSatisfied, unmarkedKinds, unmarkedRequires } from "./claim-kinds.mjs"
+import { proseSentences } from "./fidelity-binding.mjs"
+import { clip } from "./fidelity-trace.mjs"
+import { proseText } from "./html-lite.mjs"
+
+const marked = (node) => node.attrs?.[CLAIM_ATTRIBUTE] !== undefined
+const kindOf = (node) => node.attrs[CLAIM_ATTRIBUTE].trim()
+const isFact = (node) => node.attrs["data-fact"] !== undefined
+
+/** The marks over a node, nearest first: its own mark, then each marked ancestor. */
+function marksOver(node) {
+	const marks = []
+	for (let cur = node; cur; cur = cur.parent) if (marked(cur)) marks.push(cur)
+	return marks
+}
+
+const nearestKind = (node) => {
+	const [mark] = marksOver(node)
+	return mark ? kindOf(mark) : null
+}
+
+/** Whether `mark` binds `fact`: every mark from the fact up to `mark` names the same kind. */
+function binds(mark, fact) {
+	const marks = marksOver(fact)
+	const at = marks.indexOf(mark)
+	return at >= 0 && marks.slice(0, at + 1).every((cur) => kindOf(cur) === kindOf(mark))
+}
+
+/**
+ * @param {object} ctx buildContext(...) from gates-static.mjs
+ * @param {Map<string, object>} rows ledger rows by fact_id
+ * @param {(row: object) => string | undefined} provenanceOf meta.json kind of the snapshot holding the row's own locator artifact
+ * @param {(code: string, node: object, message: string) => void} flag
+ */
+export function claimChecks(ctx, rows, provenanceOf, flag) {
+	for (const node of ctx.all.filter(marked)) {
+		const name = kindOf(node)
+		const entry = CLAIM_KINDS[name]
+		if (!entry) {
+			flag("claim_kind_mismatch", node, `${CLAIM_ATTRIBUTE}="${name}" is not a claim kind (${Object.keys(CLAIM_KINDS).join(", ")})`)
+			continue
+		}
+		const cited = [node, ...ctx.util.descendants(node)].filter(isFact)
+		if (cited.some((fact) => binds(node, fact))) continue
+		const why = cited.length === 0 ? "cites no fact" : "cites no fact of its own (a nested mark of another kind never satisfies it)"
+		flag(entry.defect, node, `"${clip(proseText(node))}" is marked ${name} but ${why}`)
+	}
+	for (const fact of ctx.all.filter(isFact)) {
+		const [nearest, ...outer] = marksOver(fact)
+		const id = fact.attrs["data-fact"]
+		if (!nearest) {
+			const need = unmarkedRequires(rows.get(id))
+			const mark = need && `{{${CLAIM_KINDS[need].prefix}:${id}}} (${CLAIM_ATTRIBUTE}="${need}")`
+			if (need) flag("claim_kind_mismatch", fact, `${id} is a ${rows.get(id).kind} row cited with no mark; an unmarked numeral reads as a fact, so mark it ${mark}`)
+			continue
+		}
+		const name = kindOf(nearest)
+		const conflict = outer.find((mark) => kindOf(mark) !== name)
+		if (conflict) flag("claim_kind_mismatch", fact, `${id} is marked ${name} inside a ${kindOf(conflict)} mark; a fact carries one claim kind`)
+		const entry = CLAIM_KINDS[name]
+		const row = rows.get(id)
+		if (!entry || !row || claimSatisfied(name, row, provenanceOf(row))) continue
+		const source = entry.sourceKinds ? ` and not one whose own locator artifact sits in a ${entry.sourceKinds.join(" or ")} snapshot` : ""
+		flag(entry.defect, fact, `${row.fact_id} is marked ${name}, but it is a ${row.kind} row${source}`)
+	}
+	for (const block of proseSentences(ctx)) {
+		if (block.kind === "sources" || block.kind === "toc") continue
+		for (const sentence of block.sentences) {
+			for (const name of unmarkedKinds(sentence.text)) {
+				const covered = sentence.facts.length > 0 ? sentence.facts.every((fact) => nearestKind(fact.node) === name) : nearestKind(block.owner) === name
+				if (covered) continue
+				flag(CLAIM_KINDS[name].unmarked.code, block.owner, `"${clip(sentence.text)}" reads as a ${name} claim but cites a fact with no ${CLAIM_ATTRIBUTE}="${name}" mark`)
+			}
+		}
+	}
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-lexicon.mjs

@@ -0,0 +1,145 @@
+// fidelity-lexicon.mjs - closed-list lexical checks over (source quote, report sentence) pairs.
+//
+// lexiconChecks compares each pair with the lexicon.mjs lists: hedge_dropped, direction_flipped,
+// causality_added, qualifier_dropped (six-token window around the fact span; a :q rendering puts
+// the qualifier inside the span), scope_widened_warn, and the acronym-aware name_outside_quote.
+// The hedge-position heuristic never yields a code; it only sets force_strong for the meaning
+// verifier, as do name_outside_quote and scope_widened_warn. Latin terms match whole words (terms
+// of four or more letters also match their inflections); Korean terms match as substrings, single
+// syllables as tokens. No model, no IO. Plain Node ESM; Korean strings are \u escapes.
+
+import { CAUSAL, CONCESSIVE, DIRECTION, HEDGES, NEGATION, QUALIFIERS, SCOPE_NARROW_TO_BROAD } from "./lexicon.mjs"
+import { normalizeText } from "./text-normalize.mjs"
+import { normalizeUnit } from "./units.mjs"
+
+/** Ledger qualifier terms -> their surface forms in either language (closed mapping). */
+export const QUALIFIER_FORMS = Object.freeze({
+	upper_inclusive: ["up to", "at most", "\uCD5C\uB300", "\uC774\uD558", "\uAE4C\uC9C0"],
+	lower_inclusive: ["at least", "\uCD5C\uC18C", "\uC774\uC0C1"],
+	approximate: ["about", "approximately", "\uC57D", "\uB0B4\uC678"],
+	near: ["nearly"],
+	lower_exclusive: ["over", "more than", "\uCD08\uACFC"],
+	upper_exclusive: ["under", "less than", "\uBBF8\uB9CC"],
+})
+/** Irregular and synonym direction forms the stem match cannot reach ("rose", "reduced"). */
+const DIRECTION_FORMS = { up: ["rose", "risen", "rising", "grew", "grown", "gain"], down: ["reduc", "fell", "fallen", "falling", "drop", "lower"] }
+const KOREAN = /[\uAC00-\uD7A3]/
+/** True when the term holds a Hangul syllable (Korean terms match by the Korean rules below). */
+export const isKoreanTerm = (term) => KOREAN.test(term)
+/** The guarded hedge and qualifier vocabulary: the union of HEDGES, QUALIFIERS and every QUALIFIER_FORMS surface form. */
+export const guardedTerms = () => [...new Set([...HEDGES, ...QUALIFIERS, ...Object.values(QUALIFIER_FORMS).flat()])]
+const ACRONYM = /(?<![\p{L}\p{N}])[A-Z]{2,6}(?![\p{L}\p{N}])/gu
+const WINDOW = 6
+const cache = new Map()
+
+const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+
+/**
+ * The lexical pattern engine. mode: "stem" (inflections, Korean one-syllable token prefix), "exact",
+ * "word", "suffix" (Korean attached to a value), "guard" (single Latin words of four or more letters
+ * take inflections, multiword terms never do; Korean terms match as plain substrings).
+ */
+export function lexicalPattern(term, mode) {
+	const key = `${mode}\u0001${term}`
+	if (cache.has(key)) return cache.get(key)
+	let source
+	if (!KOREAN.test(term)) {
+		const stems = term.length >= 4 && (mode === "stem" || (mode === "guard" && !term.includes(" ")))
+		source = `(?<![\\p{L}\\p{N}])${escape(term).replace(/ /g, "\\s+")}${stems ? "\\p{L}*" : ""}(?![\\p{L}\\p{N}])`
+	} else if (mode === "guard") source = escape(term)
+	else if (mode === "suffix") source = `(?<=^|[\\s\\p{P}\\p{N}A-Za-z])${escape(term)}(?=$|[\\s\\p{P}])`
+	else if ([...term].length >= 2 || mode === "word") source = escape(term)
+	else source = `(?<=^|[\\s\\p{P}])${escape(term)}${mode === "stem" ? "" : "(?=$|[\\s\\p{P}])"}`
+	const re = new RegExp(source, "giu")
+	cache.set(key, re)
+	return re
+}
+
+function positions(text, terms, mode) {
+	const out = []
+	for (const term of terms) for (const match of text.matchAll(lexicalPattern(term, mode))) out.push({ term, at: match.index })
+	return out.sort((a, b) => a.at - b.at)
+}
+
+const has = (text, terms, mode) => positions(text, terms, mode).length > 0
+const tokens = (text) => text.split(/\s+/).filter(Boolean)
+const stripQualifiers = (text) => QUALIFIERS.reduce((rest, term) => rest.replace(lexicalPattern(term, "exact"), " "), text)
+
+function directions(text) {
+	const rest = stripQualifiers(text)
+	return {
+		up: has(rest, DIRECTION.up, "stem") || has(rest, DIRECTION_FORMS.up, "stem"),
+		down: has(rest, DIRECTION.down, "stem") || has(rest, DIRECTION_FORMS.down, "stem"),
+	}
+}
+
+/** The fact span plus up to six tokens on each side, never crossing the sentence. */
+function qualifierWindow(sentence, span) {
+	const [start, end] = span
+	return [...tokens(sentence.slice(0, start)).slice(-WINDOW), sentence.slice(start, end), ...tokens(sentence.slice(end)).slice(0, WINDOW)].join(" ")
+}
+
+function qualifierDropped(pair) {
+	if (!pair.qualifier || !Array.isArray(pair.span)) return false
+	const forms = QUALIFIER_FORMS[pair.qualifier] ?? [pair.qualifier]
+	const window = qualifierWindow(pair.sentence, pair.span)
+	return !forms.some((form) => has(window, [form], KOREAN.test(form) ? "suffix" : "exact"))
+}
+
+/** A hedge before the sentence's last concessive while the quote hedges its own final clause. */
+function hedgeMoved(quote, sentence) {
+	const lastConcessive = (text) => positions(text, CONCESSIVE, "word").at(-1)?.at ?? -1
+	const sentenceHedge = positions(sentence, HEDGES, "stem")[0]?.at
+	const quoteHedge = positions(quote, HEDGES, "stem").at(-1)?.at
+	const sentenceConcessive = lastConcessive(sentence)
+	return sentenceHedge !== undefined && quoteHedge !== undefined && sentenceHedge < sentenceConcessive && quoteHedge > lastConcessive(quote)
+}
+
+function pairFindings(pair) {
+	const quote = normalizeText(pair.quote)
+	const sentence = normalizeText(pair.sentence)
+	const found = []
+	const add = (code, message) => found.push({ pair_id: pair.pair_id, code, message })
+	const dropped = qualifierDropped(pair)
+	if (dropped) add("qualifier_dropped", `the source bounds the value (${pair.qualifier}) and the sentence drops it`)
+	const sourceHedges = positions(quote, HEDGES, "stem")
+	const qualifierOnly = sourceHedges.every(({ term }) => QUALIFIERS.includes(term))
+	if (sourceHedges.length > 0 && !has(sentence, HEDGES, "stem") && !(dropped && qualifierOnly)) {
+		add("hedge_dropped", `the source hedges ("${sourceHedges[0].term}") and the sentence does not`)
+	}
+	const [from, to] = [directions(quote), directions(sentence)]
+	const negated = has(quote, NEGATION, "word") || has(sentence, NEGATION, "word")
+	if (!negated && ((from.up && !from.down && to.down && !to.up) || (from.down && !from.up && to.up && !to.down))) {
+		add("direction_flipped", `the source says ${from.up ? "up" : "down"} and the sentence says ${to.up ? "up" : "down"}`)
+	}
+	const causal = positions(sentence, CAUSAL, "word").find(({ term }) => !has(quote, [term], "word"))
+	if (causal) add("causality_added", `"${causal.term}" states a cause the source does not`)
+	const context = [quote, ...(pair.context ?? []).map(normalizeText)]
+	for (const [narrow, broad] of SCOPE_NARROW_TO_BROAD) {
+		if (has(quote, [narrow], "word") && has(sentence, [broad], "word") && !context.some((text) => has(text, [broad], "word"))) {
+			add("scope_widened_warn", `the source says "${narrow}" and the sentence says "${broad}"`)
+		}
+	}
+	const acronyms = pair.acronyms ?? {}
+	const quoteNames = new Set(quote.match(ACRONYM) ?? [])
+	const stray = (sentence.match(ACRONYM) ?? []).find((name) => !quoteNames.has(name) && !Object.hasOwn(acronyms, name) && !normalizeUnit(name))
+	if (stray) add("name_outside_quote", `"${stray}" is in neither the quote nor the source's acronym table`)
+	return found
+}
+
+/**
+ * @param {{ pair_id: string, quote: string, sentence: string, span?: [number, number] | null,
+ *   qualifier?: string | null, acronyms?: Record<string, string>, context?: string[] }[]} pairs
+ * @returns {{ findings: { pair_id: string, code: string, message: string }[], forceStrong: string[] }}
+ */
+export function lexiconChecks(pairs) {
+	const findings = []
+	const forceStrong = []
+	for (const pair of pairs) {
+		const found = pairFindings(pair)
+		findings.push(...found)
+		const routed = found.some(({ code }) => code === "name_outside_quote" || code === "scope_widened_warn")
+		if (routed || hedgeMoved(normalizeText(pair.quote), normalizeText(pair.sentence))) forceStrong.push(pair.pair_id)
+	}
+	return { findings, forceStrong }
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-records.mjs

@@ -0,0 +1,163 @@
+// fidelity-records.mjs - the fidelity record schema and the delivery barrier of `outcome verify`
+// and `outcome finish`.
+//
+// The table and the schema are the spec for the barrier. Every fidelity record is written through
+// writeFidelityRecord, which stamps one envelope (RECORD_ENVELOPE: record_kind, artifact_sha256,
+// report, written_at, defects) plus the fields its kind declares, and read through
+// readFidelityRecord, which refuses a missing, unknown, or ill-typed field (fail closed).
+// FIDELITY_RECORDS holds every per-record rule as a column: where the records are (targets), when a
+// delivered session must carry one (requiredWhen), which envelope field binds it to the delivered
+// bytes (bindsTo), and which defects block delivery (blocking). The barrier is one loop over the
+// table with no record-specific branch; a new fidelity step adds a row here and nothing else.
+//
+// HTML/PDF/DOCX need a gate record or an explicitly logged not_run reason, even without a ledger.
+// That logged exemption covers deliveries this harness never gated: once a ledger or any session
+// fidelity record exists, the barrier refuses the unverified edition regardless of it, and the same
+// exemption is the only way a required record may be absent.
+
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { CliError, readJsonFile, sha256Of, writeJsonAtomic } from "./cli-support.mjs"
+import { LEDGER_FILENAME } from "./fact-ledger.mjs"
+import { meaningCheckEnabled, requireLockedBrief } from "./fidelity-brief.mjs"
+import { blocksDelivery, deliveryBlockReason } from "./gates-fidelity.mjs"
+
+export const FIDELITY_RECORD = "fidelity-defects.json"
+export const MEANING_RECORD = "fidelity-meaning-defects.json"
+
+/** The fields every fidelity record carries, whatever its kind. */
+export const RECORD_ENVELOPE = Object.freeze(["record_kind", "artifact_sha256", "report", "written_at", "defects"])
+
+/** Formats that need a fidelity record even in a session with no ledger and no record. */
+const GATED_FORMATS = Object.freeze(["html", "pdf", "docx"])
+
+/** The meaning record is required unless the locked brief turns the check off; a malformed block cannot prove off. */
+function meaningRequired({ sessionDir }) {
+	const lock = requireLockedBrief(join(sessionDir, "brief.md"))
+	return lock.exitCode === 2 || (lock.locked && meaningCheckEnabled(lock.parsed))
+}
+
+const subjectOf = (row) => row.renderedFrom ?? row.path ?? null
+
+/** One record per delivered edition: the file its row names with --verified-by, bound to that edition's bytes. */
+const perEdition = (sessionDir, delivered) =>
+	delivered.map((row) => ({ path: row.verifiedBy ?? null, declared: row.verifiedBy !== undefined, format: row.format, at: `${row.format}: `, subjects: [subjectOf(row)].filter((subject) => subject !== null) }))
+
+/** One record per session, in the session directory, bound to any delivered edition's bytes. */
+const perSession = (file) => (sessionDir, delivered) => [{ path: join(sessionDir, file), declared: false, format: null, at: "", subjects: delivered.map(subjectOf).filter((subject) => subject !== null) }]
+
+/** Every fidelity record the barrier reads, in check order; each column is one rule the barrier applies. */
+export const FIDELITY_RECORDS = Object.freeze([
+	Object.freeze({
+		kind: "fidelity",
+		file: FIDELITY_RECORD,
+		step: "fidelity gate",
+		targets: perEdition,
+		requiredWhen: ({ hasFidelity }, target) => hasFidelity || GATED_FORMATS.includes(target.format),
+		remedy: (target) => `run fact verify on ${target.subjects[0] ?? "the edition"} and set it with --verified-by`,
+		bindsTo: "artifact_sha256",
+		blocking: blocksDelivery,
+		payload: Object.freeze(["command", "status", "exitCode", "format", "counts", "gatesRun"]),
+	}),
+	Object.freeze({
+		kind: "meaning",
+		file: MEANING_RECORD,
+		step: "meaning gate",
+		targets: perSession(MEANING_RECORD),
+		requiredWhen: meaningRequired,
+		remedy: () => "run fact pairs and fact meaning-check before delivery",
+		bindsTo: "artifact_sha256",
+		blocking: blocksDelivery,
+		payload: Object.freeze(["command", "status", "exitCode", "run_id", "pairs_sha256", "counts", "dropped", "unverified", "calibration_path", "calibration_written", "calibration_replaced", "promotion_eligibility"]),
+	}),
+])
+
+const entryOf = (kind) => {
+	const entry = FIDELITY_RECORDS.find((row) => row.kind === kind)
+	if (entry === undefined) throw new CliError(`unknown fidelity record kind ${kind}`)
+	return entry
+}
+
+/** Why a value is not a valid record of this kind, or null. */
+function envelopeProblem(record, entry) {
+	if (record === null || typeof record !== "object" || Array.isArray(record)) return "not a record object"
+	const missing = RECORD_ENVELOPE.filter((key) => !Object.hasOwn(record, key))
+	if (missing.length > 0) return `missing ${missing.join(", ")}`
+	const unknown = Object.keys(record).filter((key) => !RECORD_ENVELOPE.includes(key) && !entry.payload.includes(key))
+	if (unknown.length > 0) return `unknown field(s) ${unknown.join(", ")}`
+	if (record.record_kind !== entry.kind) return `record_kind ${JSON.stringify(record.record_kind)} is not ${entry.kind}`
+	if (typeof record.artifact_sha256 !== "string" || !/^[0-9a-f]{64}$/u.test(record.artifact_sha256)) return "artifact_sha256 is not a sha256 hex digest"
+	if (typeof record.report !== "string" || record.report === "") return "report is not a path"
+	if (typeof record.written_at !== "string" || Number.isNaN(Date.parse(record.written_at))) return "written_at is not an ISO time"
+	if (!Array.isArray(record.defects) || !record.defects.every((defect) => typeof defect?.code === "string")) return "defects is not a list of coded defects"
+	return null
+}
+
+/**
+ * Write one fidelity record of a kind in the table: the envelope plus that kind's payload fields.
+ * @param {string} path
+ * @param {string} kind a FIDELITY_RECORDS kind
+ * @param {{ artifact_sha256: string, report: string, defects: object[], [field: string]: unknown }} fields the envelope values plus the kind's payload
+ */
+export function writeFidelityRecord(path, kind, { artifact_sha256, report, defects, ...payload }, now = new Date()) {
+	const entry = entryOf(kind)
+	const record = { record_kind: kind, artifact_sha256, report, written_at: now.toISOString(), defects, ...payload }
+	const problem = envelopeProblem(record, entry)
+	if (problem !== null) throw new CliError(`refusing to write a malformed ${kind} record: ${problem}`)
+	writeJsonAtomic(path, record)
+	return record
+}
+
+/** Read one fidelity record of a kind; anything but a valid envelope is a CliError naming the path. */
+export function readFidelityRecord(path, kind) {
+	const entry = entryOf(kind)
+	let record
+	try {
+		record = readJsonFile(path)
+	} catch (error) {
+		throw new CliError(`${kind} record ${path} is malformed: ${error.message}`)
+	}
+	const problem = envelopeProblem(record, entry)
+	if (problem !== null) throw new CliError(`${kind} record ${path} is malformed: ${problem}`)
+	return record
+}
+
+const unresolved = (format, codes, path) => ({ code: "fidelity_unresolved", format, codes, message: codes.map((code) => deliveryBlockReason({ code })).join("; "), path })
+const hashOf = (path) => (existsSync(path) ? sha256Of(readFileSync(path)) : null)
+
+/** Barrier problems for `outcome verify` and `outcome finish`; empty when nothing blocks delivery. */
+export function fidelityProblems(sessionDir, manifest) {
+	const delivered = manifest.deliverables.filter((row) => row.status === "delivered")
+	const hasFidelity = existsSync(join(sessionDir, LEDGER_FILENAME)) || FIDELITY_RECORDS.some((entry) => existsSync(join(sessionDir, entry.file)))
+	const lastFidelity = manifest.gateHistory?.findLast((entry) => entry.gate === "fidelity")
+	const exempt = !hasFidelity && manifest.gates.fidelity === "not_run" && lastFidelity?.status === "not_run" && Boolean(lastFidelity.reason?.trim())
+	const context = { sessionDir, hasFidelity }
+	const problems = []
+	const counted = new Set()
+	for (const entry of FIDELITY_RECORDS) {
+		for (const target of entry.targets(sessionDir, delivered)) {
+			const refuse = (message, path = target.path) => problems.push({ code: "edition_unverified", format: target.format, message: `${target.at}${message}`, path })
+			if (target.path === null || !existsSync(target.path)) {
+				if (target.declared) refuse(`${entry.kind} record ${target.path} is missing`)
+				else if (delivered.length > 0 && !exempt && entry.requiredWhen(context, target)) refuse(`${entry.step} not run; ${entry.remedy(target)}`, target.path ?? target.subjects[0] ?? null)
+				continue
+			}
+			let record
+			try {
+				record = readFidelityRecord(target.path, entry.kind)
+			} catch (error) {
+				refuse(error.message)
+				continue
+			}
+			if (target.subjects.length > 0 && !target.subjects.some((subject) => hashOf(subject) === record[entry.bindsTo])) {
+				refuse(`${entry.kind} record ${target.path} is for a different edition than ${target.subjects.join(", ")}`)
+			}
+			if (counted.has(target.path)) continue
+			counted.add(target.path)
+			const codes = [...new Set(record.defects.filter(entry.blocking).map((defect) => defect.code))].sort()
+			if (codes.length > 0) problems.push(unresolved(target.format, codes, target.path))
+		}
+	}
+	return problems
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-state.mjs

@@ -1,6 +1,7 @@
 // fidelity-state.mjs - per-sentence failure counts for the fidelity repair loop. A sentence is
 // keyed by its section slug and the sorted fact ids it cites, so a reworded sentence that
-// cites the same facts keeps its count. node: builtins only.
+// cites the same facts keeps its count. The exact slug and id rules are in the comment above
+// slugify in fact-fill.mjs. node: builtins only.
 
 import { existsSync } from "node:fs"
 import { join } from "node:path"

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-trace.mjs

@@ -0,0 +1,85 @@
+// fidelity-trace.mjs - the numeral and quotation trace of the deterministic fidelity gate.
+//
+// Every numeral in prose and table cells outside sources, toc and chart figures (dates included)
+// must sit in a data-fact, data-translates, q or blockquote element, or at the site of a fill-map
+// entry; otherwise untraceable_number. Quoted runs with no Q row are quote_unbound, and leftover
+// {{...}} pointers unresolved_placeholder. No model, no network.
+//
+// Fill-map assumption (fact-fill.mjs fill(), #11141): format "html" wraps every value and quote in
+// <data data-fact> / <q data-fact> / <blockquote data-fact>, which this trace skips by element, so
+// it needs no fill map. Format "md" writes bare text, and each fillMap entry is
+// { fact_id, rendered, start, end } with start/end indexing the filled Markdown, not the HTML this
+// gate reads. fill() pushes entries in output order and the Markdown-to-HTML step keeps text order,
+// so each entry, in array order, is matched to the next untraced occurrence of its rendered text in
+// document order that is a whole numeral and reads as the entry's row there (fits: "1,237 million"
+// is not F2's 1,237), and strips that one occurrence only. A copy of the same text that no entry
+// accounts for stays in the scan; an entry whose text is gone strips nothing.
+
+import { NOT_CLAIMS } from "./numerals.mjs"
+
+/** Structural numerals: versions, figure/table/section references, times (NOT_CLAIMS 7-11), pages, citations, placeholders. */
+export const STRUCTURAL = [...NOT_CLAIMS.slice(7, 12), /\b(?:p|pp|page|pages)\.?\s*\d+(?:\s*[-\u2013]\s*\d+)?/gi, /\[(?:S|Source\s*)\d+\]|\[\^\d+\]/gi, /\{\{[^}]*\}\}/g]
+const NUMERAL = /\d+(?:[.,\-/:]\d+)*/g
+const TRACED = "\u0000"
+const QUOTE_RUNS = [/"([^"]+)"/g, /\u201C([^\u201C\u201D]+)\u201D/g, /\u300C([^\u300C\u300D]+)\u300D/g, /\u300E([^\u300E\u300F]+)\u300F/g]
+
+export const clip = (text) => String(text).replace(/\s+/g, " ").trim().slice(0, 80)
+
+function ownerTexts(ctx, charts) {
+	const owners = new Map()
+	for (const entry of ctx.texts) {
+		const kind = ctx.kindOf(entry.scope)
+		if (kind === "sources" || kind === "toc") continue
+		if (charts.has(ctx.util.closest(entry.node, (cur) => cur.tag === "figure"))) continue
+		const hidden = ctx.util.closest(entry.node, (cur) => cur.attrs["data-fact"] !== undefined || cur.attrs["data-translates"] !== undefined || cur.tag === "q" || cur.tag === "blockquote")
+		owners.set(entry.owner, (owners.get(entry.owner) ?? "") + (hidden ? ` ${TRACED} ` : entry.node.text))
+	}
+	return owners
+}
+
+function scanOwner(ctx, owner, raw, flag) {
+	let text = raw
+	if (/^h[1-6]$/.test(owner.tag)) text = text.replace(/^\s*\d+(?:[.-]\d+)*[.):]?\s+/, " ")
+	if (text.includes("{{")) flag("unresolved_placeholder", owner, `leftover placeholder: "${clip(text.slice(text.indexOf("{{")))}"`)
+	for (const re of QUOTE_RUNS) {
+		for (const match of text.matchAll(re)) {
+			const inner = match[1].replaceAll(TRACED, " ").trim()
+			const after = text.slice(match.index + match[0].length).trim().split(/\s+/).slice(0, 6).join(" ")
+			if (inner && (inner.split(/\s+/).length >= 3 || ctx.util.citationMarker.test(after))) flag("quote_unbound", owner, `direct quotation with no Q row: "${clip(match[0])}"`)
+		}
+	}
+	const rest = STRUCTURAL.reduce((acc, re) => acc.replace(re, " "), text)
+	for (const match of rest.matchAll(NUMERAL)) flag("untraceable_number", owner, `number "${match[0]}" has no ledger row: "${clip(rest.slice(Math.max(0, match.index - 30), match.index + 30))}"`)
+}
+
+/** Strip each fill-map entry at its own fill site: the next untraced occurrence after the previous entry's site. */
+function traceFills(owners, fillMap, fits) {
+	const keys = [...owners.keys()]
+	const texts = [...owners.values()]
+	let [index, from] = [0, 0]
+	for (const entry of fillMap ?? []) {
+		const { rendered } = entry
+		if (!rendered) continue
+		for (let i = index, at = from; i < texts.length; i += 1, at = 0) {
+			let hit = texts[i].indexOf(rendered, at)
+			const glued = (pos) => /\d/.test(texts[i][pos - 1] ?? "") || /\d/.test(texts[i][pos + rendered.length] ?? "")
+			while (hit !== -1 && (glued(hit) || !fits(entry, texts[i].slice(hit)))) hit = texts[i].indexOf(rendered, hit + 1)
+			if (hit === -1) continue
+			texts[i] = `${texts[i].slice(0, hit)}${TRACED}${texts[i].slice(hit + rendered.length)}`
+			;[index, from] = [i, hit + TRACED.length]
+			break
+		}
+	}
+	return new Map(keys.map((key, i) => [key, texts[i]]))
+}
+
+/**
+ * @param {object} ctx buildContext(...) from gates-static.mjs
+ * @param {Map<object, unknown>} charts chart figures, whose numerals chart checks own
+ * @param {{ fact_id: string, rendered: string, start?: number, end?: number }[] | null} fillMap Markdown fill map, or null
+ * @param {(code: string, node: object, message: string) => void} flag
+ * @param {(entry: object, tail: string) => boolean} fits whether the text from a candidate site on reads as the entry's row
+ */
+export function traceChecks(ctx, charts, fillMap, flag, fits) {
+	for (const [owner, text] of traceFills(ownerTexts(ctx, charts), fillMap, fits)) scanOwner(ctx, owner, text, flag)
+}