lazycodex-ai 5.1.45 → 5.1.46

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-fill.mjs

@@ -0,0 +1,194 @@
+// fact-fill.mjs - the fill step: substitute every fact pointer in a report source from the
+// ledger, all or nothing. Fatal errors (pointer_label_mismatch, unresolved_placeholder,
+// quote_anchor_unresolved, unit_duplicated) return out null; warnings (particle_unverified,
+// fallback_without_entity) never block. A twice-failed sentence can be replaced by an attributed
+// block quote of its source row (replaceFailed). No IO, but not pure: unless options.view is set,
+// it records each replaced sentence in options.replaceFailed through markReplaced. node: builtins only.
+
+import { finalReading, rewriteParticle } from "./particles-ko.mjs"
+import { failureCount, markReplaced, sentenceKey } from "./fidelity-state.mjs"
+import { findPlaceholders, lead, qualifiedText, renderBlockquote, renderQuote, renderValueHtml, subSpan, valueParts } from "./placeholder.mjs"
+import { splitSentences } from "./sentence-split.mjs"
+
+const FORMATS = Object.freeze(["html", "md"])
+const TRANSLATION = Object.freeze({
+	html: /^\s*<span data-translates="(Q\d+)">[\s\S]*?<\/span>/u,
+	md: /^\s*<!-- translates: (Q\d+) -->[^\n]*/u,
+})
+const HEADING = Object.freeze({ html: /<h[1-6]\b([^>]*)>([\s\S]*?)<\/h[1-6]>/giu, md: /^#{1,6}[ \t]+(.+)$/gmu })
+
+const lineOf = (text, at) => text.slice(0, at).split("\n").length
+
+function resolveValue(row, parsed, format) {
+	const { valueText, unitText } = valueParts(row)
+	const text = { full: row.raw, value: valueText, unit: unitText, q: qualifiedText(row) }[parsed.mode]
+	if (text === null) return { code: "quote_anchor_unresolved", message: `${row.fact_id} has qualifier ${row.qualifier}, but its quote holds no qualifier beside "${row.raw}"` }
+	return { row, parsed, visible: text, rendered: format === "html" ? renderValueHtml(row, text) : text }
+}
+
+function resolveQuote(row, parsed, format) {
+	const whole = { text: row.quote, start: 0, end: row.quote.length }
+	const selection = parsed.mode === "lead" ? lead(row.quote, parsed.lead) : parsed.mode === "span" ? subSpan(row.quote, parsed.from, parsed.to) : whole
+	if (selection === null) {
+		return { code: "quote_anchor_unresolved", message: `from "${parsed.from}" and to "${parsed.to}" must each occur exactly once in ${row.fact_id}'s quote, in that order` }
+	}
+	const rendered = renderQuote(row, parsed.selector, selection.text, format)
+	return { row, parsed, visible: `[${row.source_id}]`, rendered, selector: parsed.selector, quote_span: [selection.start, selection.end] }
+}
+
+function resolve(found, rows, format) {
+	const { parsed } = found
+	if (parsed.error) return { code: "unresolved_placeholder", message: parsed.error }
+	const row = rows.get(parsed.id)
+	if (row === undefined) return { code: "unresolved_placeholder", message: `${parsed.id} is not in the fact ledger` }
+	if (row.status !== "active") return { code: "unresolved_placeholder", message: `${parsed.id} is ${row.status}; cite its replacement` }
+	if (row.label !== parsed.label) return { code: "pointer_label_mismatch", message: `${parsed.id} has label "${row.label}", not "${parsed.label}"` }
+	return parsed.kind === "Q" ? resolveQuote(row, parsed, format) : resolveValue(row, parsed, format)
+}
+
+/** A unit typed right after a pointer that already renders it: never absorbed, always an error. */
+function unitDuplicated(text, item) {
+	if (item.parsed.kind !== "F" || (item.parsed.mode !== "full" && item.parsed.mode !== "q")) return false
+	const suffix = item.row.raw.replace(/^[\s\S]*\d/u, "").trim()
+	const units = [item.row.unit, valueParts(item.row).unitText, suffix].filter((unit) => typeof unit === "string" && unit !== "")
+	const following = text.slice(item.end).replace(/^[ \t]+/u, "")
+	return units.some((unit) => following.startsWith(unit) && !/[A-Za-z0-9]/u.test(following[unit.length] ?? ""))
+}
+
+/*
+ * Sentence-key rules (fidelity-state.mjs sentenceKey(slug, ids)); any other producer of keys must match:
+ * - slug source: an HTML heading's id attribute wins over its text; md headings use their text.
+ * - slugify: strip tags, lowercase, turn each run of non-letters/digits (\p{L}\p{N}) into "-",
+ *   trim leading and trailing "-".
+ * - a sentence takes the nearest heading that starts strictly before the sentence's start offset;
+ *   a sentence before the first heading gets slug "".
+ * - fact ids cited in the sentence are de-duplicated before sentenceKey sorts them.
+ */
+function slugify(text) {
+	return text
+		.replace(/<[^>]*>/gu, "")
+		.toLowerCase()
+		.replace(/[^\p{L}\p{N}]+/gu, "-")
+		.replace(/^-+|-+$/gu, "")
+}
+
+function headings(text, format) {
+	return [...text.matchAll(HEADING[format])].map((match) => {
+		const id = format === "html" ? /\bid="([^"]*)"/u.exec(match[1])?.[1] : undefined
+		return { at: match.index, slug: slugify(id ?? match[match.length - 1]) }
+	})
+}
+
+/** Sentence offsets over the source with pointers, comments and (html) tags blanked to equal length. */
+function sentencesOf(text, found, format) {
+	let masked = text
+	for (const item of found) masked = masked.slice(0, item.start) + "x".repeat(item.end - item.start) + masked.slice(item.end)
+	const blank = (part) => part.replace(/[^\n]/gu, " ")
+	masked = masked.replace(/<!--[\s\S]*?-->/gu, blank)
+	if (format === "html") masked = masked.replace(/<[^>]*>/gu, blank)
+	return splitSentences(masked)
+}
+
+function replacements(text, found, items, options, warnings) {
+	const { format, replaceFailed: state } = options
+	const marks = headings(text, format)
+	const out = []
+	for (const sentence of sentencesOf(text, found, format)) {
+		const inside = items.filter((item) => item.start >= sentence.start && item.end <= sentence.end)
+		if (inside.length === 0) continue
+		const slug = marks.filter((mark) => mark.at < sentence.start).at(-1)?.slug ?? ""
+		const key = sentenceKey(slug, [...new Set(inside.map((item) => item.row.fact_id))])
+		if (failureCount(state, key) < 2) continue
+		const rows = inside.map((item) => item.row)
+		const source = rows.find((row) => row.kind === "quote") ?? rows.find((row) => row.source_id !== null && row.quote !== "")
+		if (source === undefined) continue
+		let rendered = options.fallback === "omit" ? "" : renderBlockquote(source, key, format)
+		if (rendered !== "" && format === "md") {
+			if (sentence.start > 0 && text[sentence.start - 1] !== "\n") rendered = `\n\n${rendered}`
+			if (sentence.end < text.length && text[sentence.end] !== "\n") rendered = `${rendered}\n\n`
+		}
+		if (rendered !== "" && source.entity === null) {
+			warnings.push({ code: "fallback_without_entity", token: key, line: lineOf(text, sentence.start), message: `${source.fact_id} has no entity; the block quote cites [${source.source_id}] alone` })
+		}
+		out.push({ start: sentence.start, end: sentence.end, key, source, rendered })
+	}
+	return out
+}
+
+/** The text after a pointer: rewrite a following Korean particle, and map a translation span. */
+function afterPlaceholder(segment, item, context) {
+	const { format, text, warnings, fillMap } = context
+	const probeEnd = format === "html" && segment.includes("<") ? segment.indexOf("<") : segment.length
+	const result = rewriteParticle(segment.slice(0, probeEnd), finalReading(item.visible, null))
+	let next = segment
+	if (result.applied) next = result.text + segment.slice(probeEnd)
+	else if (result.candidate) {
+		warnings.push({ code: "particle_unverified", token: item.token, line: lineOf(text, item.start), message: `the particle after ${item.token} was not rewritten; check it by eye` })
+	}
+	const translation = item.parsed.kind === "Q" ? TRANSLATION[format].exec(next) : null
+	if (translation !== null && translation[1] === item.row.fact_id) {
+		const span = translation[0].trimStart()
+		const start = context.out.length + translation[0].length - span.length
+		fillMap.push({ token: span, fact_id: translation[1], translates: translation[1], start, end: start + span.length, rendered: span })
+	}
+	return next
+}
+
+function assemble(text, parts, context) {
+	let cursor = 0
+	let previous = null
+	const flush = (until) => {
+		const segment = text.slice(cursor, until)
+		context.out += previous?.parsed ? afterPlaceholder(segment, previous, context) : segment
+	}
+	for (const part of parts) {
+		flush(part.start)
+		const start = context.out.length
+		if (part.parsed) {
+			const extra = part.selector === undefined ? {} : { selector: part.selector, quote_span: part.quote_span }
+			context.fillMap.push({ token: part.token, fact_id: part.row.fact_id, start, end: start + part.rendered.length, rendered: part.rendered, ...extra })
+		} else if (part.rendered !== "") {
+			context.fillMap.push({ token: null, fact_id: part.source.fact_id, replaced: part.key, start, end: start + part.rendered.length, rendered: part.rendered })
+		}
+		context.out += part.rendered
+		cursor = part.end
+		previous = part
+	}
+	flush(text.length)
+	return context.out
+}
+
+/**
+ * @param {string} sourceText the report source with {{F..}} / {{Q..}} pointers
+ * @param {{ rows: object[] }} ledger
+ * @param {{ format?: "html"|"md", replaceFailed?: object, fallback?: "quote"|"omit", view?: boolean }} [options]
+ * @returns {{ out: string|null, fillMap: object[], errors: object[], warnings: object[] }}
+ */
+export function fill(sourceText, ledger, options = {}) {
+	const format = options.format ?? "html"
+	if (!FORMATS.includes(format)) throw new TypeError(`format must be one of ${FORMATS.join(", ")}`)
+	const rows = new Map(ledger.rows.map((row) => [row.fact_id, row]))
+	const found = findPlaceholders(sourceText)
+	const errors = []
+	const warnings = []
+	const items = []
+	for (const pointer of found) {
+		const resolved = resolve(pointer, rows, format)
+		const line = lineOf(sourceText, pointer.start)
+		if (resolved.code) errors.push({ code: resolved.code, token: pointer.token, line, message: resolved.message })
+		else {
+			const item = { ...pointer, ...resolved }
+			if (unitDuplicated(sourceText, item)) {
+				errors.push({ code: "unit_duplicated", token: pointer.token, line, message: `${pointer.token} already renders its unit; use {{${item.row.fact_id}:${item.row.label}:value}}` })
+			} else items.push(item)
+		}
+	}
+	if (errors.length > 0) return { out: null, fillMap: [], errors, warnings }
+	const replaced = options.replaceFailed ? replacements(sourceText, found, items, { ...options, format }, warnings) : []
+	const kept = items.filter((item) => !replaced.some((block) => item.start >= block.start && item.end <= block.end))
+	const parts = [...kept, ...replaced].sort((a, b) => a.start - b.start)
+	const context = { format, text: sourceText, warnings, fillMap: [], out: "" }
+	const out = assemble(sourceText, parts, context)
+	if (!options.view) for (const block of replaced) markReplaced(options.replaceFailed, block.key)
+	return { out, fillMap: context.fillMap, errors, warnings }
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-ledger-kinds.mjs

@@ -5,6 +5,11 @@
 // Also the value primitives the schema shares. Pure functions with no IO; node: builtins only.
 
 import { isCliEntry } from "./entry-guard.mjs"
+import { canonicalUnit, inputScale, RuleError } from "./unit-algebra.mjs"
+import { metadataErrors } from "./unit-metadata.mjs"
+import { mul } from "./rational.mjs"
+import { placesOf, roundPlaces, toBig } from "./unit-precision.mjs"
+import { OPERATION_RULES } from "./unit-rules.mjs"
 
 export const DECIMAL = /^-?\d+(?:\.\d+)?$/
 const INTEGER = /^-?\d+$/
@@ -16,7 +21,15 @@ export const isString = (value) => typeof value === "string"
 export const isNonEmpty = (value) => typeof value === "string" && value.length > 0
 export const isCount = (value) => Number.isInteger(value) && value >= 0
 const isDecimal = (value) => isString(value) && DECIMAL.test(value)
-const isRational = (exact) => isObject(exact) && isString(exact.num) && isString(exact.den) && INTEGER.test(exact.num) && /^\d+$/.test(exact.den) && !/^0+$/.test(exact.den)
+export function isRational(exact) {
+	if (!isObject(exact) || Object.keys(exact).sort().join(",") !== "den,num" || !isString(exact.num) || !isString(exact.den) || !INTEGER.test(exact.num) || !/^[1-9]\d*$/.test(exact.den)) return false
+	let a = BigInt(exact.num)
+	let b = BigInt(exact.den)
+	if (String(a) !== exact.num) return false
+	if (a < 0n) a = -a
+	while (b !== 0n) [a, b] = [b, a % b]
+	return a === 1n
+}
 
 /** The exact rational of a decimal string, reduced. */
 export function exactFromDecimal(value) {
@@ -72,22 +85,27 @@ export const RULES = Object.freeze({
 		test: (v, row) => isRational(v) && (!isDecimal(row.value) || (v.num === exactFromDecimal(row.value).num && v.den === exactFromDecimal(row.value).den)),
 	},
 	rational_or_null: { cell: "rational or null", says: "null or { num: integer string, den: positive integer string }", test: orNull(isRational) },
-	by_derive: { cell: "written by derive (rational or null)", says: "null or { num: integer string, den: positive integer string }", test: orNull(isRational) },
+	by_derive: { cell: "reduced rational", says: "a reduced { num: integer string, den: positive integer string }", test: isRational },
+	context_by_derive: { cell: "written by derive (object or null)", says: "null or an object", test: orNull(isObject) },
+	derived_operation: { cell: "sum/diff/growth/share/ratio/expr", says: "a derived operation", test: (v) => Object.hasOwn(OPERATION_RULES, v) && OPERATION_RULES[v].kind === "derived" },
+	convert_operation: { cell: "`convert`", says: "convert", test: (v) => Object.hasOwn(OPERATION_RULES, v) && OPERATION_RULES[v].kind === "converted" },
+	precision: { cell: "`{ places, sigfigs }`", says: "nonnegative places and positive sigfigs", test: (v) => isObject(v) && Object.keys(v).sort().join(",") === "places,sigfigs" && Number.isSafeInteger(v.places) && v.places >= 0 && Number.isSafeInteger(v.sigfigs) && v.sigfigs >= 1 },
+	compatibility: { cell: "checked or unchecked (expr)", says: "the operation's compatibility", test: (v, r) => Object.hasOwn(OPERATION_RULES, r.operation) && v === OPERATION_RULES[r.operation].compatibility },
 })
 
 /** The grammar's per-kind columns, in table order; `exact` follows `value` because it reads it. */
-export const KIND_COLUMNS = Object.freeze(["fact_id", "value", "source_id", "locator", "snapshot_sha256", "rationale", "exact"])
+export const KIND_COLUMNS = Object.freeze(["fact_id", "value", "source_id", "locator", "snapshot_sha256", "rationale", "exact", "context", "operation", "precision", "compatibility"])
 
 const sourced = { source_id: "source", locator: "locator", snapshot_sha256: "sha256" }
 const unsourced = { source_id: "source_or_null", locator: "locator_or_null", snapshot_sha256: "sha256_or_null" }
-const row = (cells) => Object.freeze(Object.fromEntries(KIND_COLUMNS.map((column) => [column, cells[column]])))
+const row = (cells) => Object.freeze(Object.fromEntries(KIND_COLUMNS.map((column) => [column, cells[column] ?? (["precision", "compatibility"].includes(column) && cells.operation !== "null" ? column : "null")])))
 
 export const KIND_RULES = Object.freeze({
-	measured: row({ fact_id: "f_id", value: "decimal", ...sourced, rationale: "string_or_null", exact: "from_value" }),
-	quote: row({ fact_id: "q_id", value: "null", ...sourced, rationale: "string_or_null", exact: "null" }),
-	derived: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "string_or_null", exact: "by_derive" }),
-	assumed: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "non_empty", exact: "rational_or_null" }),
-	converted: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "string_or_null", exact: "by_derive" }),
+	measured: row({ fact_id: "f_id", value: "decimal", ...sourced, rationale: "string_or_null", exact: "from_value", context: "null", operation: "null" }),
+	quote: row({ fact_id: "q_id", value: "null", ...sourced, rationale: "string_or_null", exact: "null", context: "null", operation: "null" }),
+	derived: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "string_or_null", exact: "by_derive", context: "context_by_derive", operation: "derived_operation" }),
+	assumed: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "non_empty", exact: "rational_or_null", context: "null", operation: "null" }),
+	converted: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "string_or_null", exact: "by_derive", context: "context_by_derive", operation: "convert_operation" }),
 })
 
 /** The id prefix nextId assigns for a kind; an unknown kind gets F and fails validation later. */
@@ -100,15 +118,101 @@ export function exactComesFromValue(kind) {
 	return KIND_RULES[kind]?.exact === "from_value"
 }
 
+/** Columns added after schema version 1 shipped; a ledger written before them still loads (dimension 31). */
+export const LATER_COLUMNS = Object.freeze(["context", "precision", "compatibility"])
+const DERIVED_KINDS = Object.freeze(["derived", "converted"])
+const PRE_REGISTRY_COLUMNS = KIND_COLUMNS.slice(0, KIND_COLUMNS.indexOf("exact") + 1)
+
+/** A derived or converted row written before the later columns existed: loadable, never an operand. */
+export const isLegacyDerived = (row) => isObject(row) && DERIVED_KINDS.includes(row.kind) && LATER_COLUMNS.every((column) => !Object.hasOwn(row, column))
+/** A derived or converted row carrying this version's columns: the only rows checked strictly at read time. */
+export const isCurrentDerived = (row) => isObject(row) && DERIVED_KINDS.includes(row.kind) && !isLegacyDerived(row)
+
+/** An exact carrying keys beyond num and den: never normalized, never silently stripped, flagged on use. */
+export const hasUnknownExactKeys = (exact) => isObject(exact) && isString(exact.num) && isString(exact.den) && Object.keys(exact).sort().join(",") !== "den,num"
+/** Any integer pair dev's writer accepted, reduced: sign on num, no leading zeros, gcd 1; anything else as given. */
+function readExact(exact) {
+	if (!isObject(exact) || hasUnknownExactKeys(exact) || !isString(exact.num) || !isString(exact.den) || !INTEGER.test(exact.num) || !/^\d+$/.test(exact.den) || /^0+$/.test(exact.den)) return exact
+	const { num, den } = mul({ num: BigInt(exact.num), den: BigInt(exact.den) }, 1n)
+	return { num: String(num), den: String(den) }
+}
+
+const readDecimal = (value) => (isDecimal(value) ? value.replace(/^(-?)0+(?=\d)/, "$1") : value)
+
+/** A source unit read through the alias table; unknown, scaled, or unparseable text stays as given. */
+export function sourceUnit(unit) {
+	try {
+		const canonical = canonicalUnit(unit)
+		return isString(canonical) && !inputScale(unit) ? canonical : unit
+	} catch (error) {
+		if (error instanceof RuleError) return unit
+		throw error
+	}
+}
+
+/**
+ * A row as every reader sees it (dimension 31 wins at read time): rows not written by derive in this
+ * version get reduced exacts and decimals; source rows also get canonical units and null later columns.
+ * Current derived rows stay as stored so tampering is still detected. Idempotent.
+ */
+export function readRow(row) {
+	if (!isObject(row) || !Object.hasOwn(KIND_RULES, row.kind) || isCurrentDerived(row)) return row
+	const read = { ...row, value: readDecimal(row.value), exact: exactComesFromValue(row.kind) ? row.exact : readExact(row.exact) }
+	if (DERIVED_KINDS.includes(row.kind)) return read
+	read.unit = sourceUnit(row.unit)
+	for (const column of LATER_COLUMNS) if (!Object.hasOwn(read, column)) read[column] = null
+	return read
+}
+
+const carriesDerivation = (row) => row.operation !== null || row.operands?.length || row.derived_from?.length || row.formula !== null || row.factor !== null
+
+/** Dimension 32: a stored exact must equal the row's own value at the value's decimal places. */
+function valueExactDisagreement(row) {
+	const exact = readExact(row.exact)
+	const value = readDecimal(row.value)
+	if (!isRational(exact) || !isDecimal(value) || roundPlaces(toBig(exact), placesOf(value)) === value) return null
+	return `value ${value} disagrees with exact ${exact.num}/${exact.den} at ${placesOf(value)} decimal places`
+}
+function unitFlaw(unit) {
+	try {
+		return isString(unit) && (canonicalUnit(unit) !== unit || inputScale(unit)) ? `unit ${unit} has no canonical stored text` : null
+	} catch (error) {
+		if (error instanceof RuleError) return `unit ${unit}: ${error.message}`
+		throw error
+	}
+}
+
+/**
+ * What a loadable source row (read through readRow) still breaks under the strict dimensions: dev's
+ * writer accepted it, so it loads, but derive and recompute refuse it as an operand with this shape.
+ * addRow refuses a new row that has any. Empty for derived and converted rows.
+ */
+export function readFlaws(row) {
+	if (!isObject(row) || DERIVED_KINDS.includes(row.kind)) return []
+	const flaws = []
+	if (carriesDerivation(row)) flaws.push({ shape: "dimension 1", detail: "a source row carries an operation or derivation fields", expected: "null operation and empty derivation fields on measured, quote, and assumed rows" })
+	const unit = unitFlaw(row.unit)
+	if (unit) flaws.push({ shape: "dimension 6", detail: unit, expected: "a registered or canonical unit without scale" })
+	for (const detail of metadataErrors(row)) flaws.push({ shape: "dimension 18", detail, expected: "present, type-valid metadata; explicit null when unknown" })
+	const unknownKeys = hasUnknownExactKeys(row.exact)
+	const disagreement = unknownKeys ? null : valueExactDisagreement(row)
+	if (unknownKeys) flaws.push({ shape: "dimension 7", detail: `exact has unknown keys: ${Object.keys(row.exact).filter((key) => key !== "num" && key !== "den").join(", ")}`, expected: "an exact with exactly num and den (the stored row is kept as written)" })
+	if (disagreement) flaws.push({ shape: "dimension 32", detail: disagreement, expected: "an exact equal to value at the value's decimal places" })
+	return flaws
+}
+
 /** @returns {string[]} every per-kind column error of one row; empty for an unknown kind. */
 export function kindRuleErrors(candidate, at) {
 	const rules = KIND_RULES[candidate.kind]
 	if (rules === undefined) return []
 	const errors = []
-	for (const column of KIND_COLUMNS) {
-		const value = candidate[column]
-		if (value === undefined) continue
-		const rule = RULES[rules[column]]
+	const legacy = isLegacyDerived(candidate)
+	const columns = legacy ? PRE_REGISTRY_COLUMNS : isCurrentDerived(candidate) ? KIND_COLUMNS : KIND_COLUMNS.filter((column) => column !== "operation")
+	for (const column of columns) {
+		const value = candidate[column] === undefined && LATER_COLUMNS.includes(column) ? null : candidate[column]
+		const rule = RULES[legacy && column === "exact" ? "rational_or_null" : rules[column]]
+		// An exact with unknown keys loads as written (never stripped); readFlaws refuses it at use and at write.
+		if (column === "exact" && hasUnknownExactKeys(value) && !isCurrentDerived(candidate)) continue
 		if (rule.test(value, candidate)) continue
 		errors.push(`${at}.${column} must be ${rule.says} for a ${candidate.kind} row`)
 		if (rule.detail) errors.push(...rule.detail(value, `${at}.${column}`))
@@ -116,6 +220,16 @@ export function kindRuleErrors(candidate, at) {
 	return errors
 }
 
+/** At-rest rules of current derived rows; source rows answer to readFlaws at use and at write instead. */
+export function storedErrors(row) {
+	if (!isCurrentDerived(row)) return []
+	const errors = metadataErrors(row)
+	try {
+		if (canonicalUnit(row.unit) !== row.unit || inputScale(row.unit)) errors.push("unit must be canonical stored text")
+	} catch (error) { errors.push(error.message) }
+	return errors
+}
+
 export const KIND_TABLE_BEGIN = "<!-- kind-rules:begin (rendered from KIND_RULES; do not edit by hand) -->"
 export const KIND_TABLE_END = "<!-- kind-rules:end -->"
 

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-ledger-schema.mjs

@@ -4,7 +4,8 @@
 // KIND_RULES (fact-ledger-kinds.mjs), never here.
 // Pure functions with no IO; fact-ledger.mjs owns reading, writing, and row changes. node: builtins only.
 
-import { DECIMAL, isCount, isNonEmpty, isObject, isString, kindRuleErrors } from "./fact-ledger-kinds.mjs"
+import { DECIMAL, isCount, isCurrentDerived, isNonEmpty, isObject, isString, kindRuleErrors, LATER_COLUMNS, readRow, storedErrors } from "./fact-ledger-kinds.mjs"
+import { malformed } from "./unit-rules.mjs"
 
 export { DECIMAL, isNonEmpty, isString } from "./fact-ledger-kinds.mjs"
 
@@ -50,8 +51,13 @@ export const ROW_DEFAULTS = Object.freeze({
 	formula: null,
 	factor: null,
 	rationale: null,
+	context: null,
+	precision: null,
+	compatibility: null,
 	status: "active",
 })
+/** Fields added after schema version 1 shipped: written on every new row, optional when reading an older ledger. */
+const OPTIONAL_FIELDS = new Set(LATER_COLUMNS)
 const REQUIRED = ["fact_id", "kind", "raw", "quote", "extracted_by", "observed_at", "label"]
 export const ROW_FIELDS = Object.freeze([...REQUIRED, ...Object.keys(ROW_DEFAULTS)])
 
@@ -70,10 +76,16 @@ function nullOr(value, test) {
 function validateRow(row, index, errors) {
 	const at = `rows[${index}]`
 	if (!isObject(row)) return errors.push(`${at} must be an object`)
-	for (const key of ROW_FIELDS) check(errors, Object.hasOwn(row, key), `${at}.${key} is required`)
+	for (const key of ROW_FIELDS) check(errors, OPTIONAL_FIELDS.has(key) || Object.hasOwn(row, key), `${at}.${key} is required`)
 	for (const key of Object.keys(row)) check(errors, ROW_FIELDS.includes(key), `${at}.${key} is not a ledger field`)
 	check(errors, FACT_KINDS.includes(row.kind), `${at}.kind must be one of ${FACT_KINDS.join(", ")}`)
-	errors.push(...kindRuleErrors(row, at))
+	// Dimension 31 wins at read time: per-kind rules see the row as readRow normalizes it.
+	errors.push(...kindRuleErrors(readRow(row), at))
+	errors.push(...storedErrors(row).map((e) => `${at}.${e}`))
+	if (isCurrentDerived(row)) {
+		const problem = malformed(row)
+		if (problem) errors.push(`${at}: ${problem}`)
+	}
 	check(errors, row.unit === undefined || nullOr(row.unit, isNonEmpty), `${at}.unit must be null or a non-empty token`)
 	for (const key of ["raw", "quote"]) check(errors, row[key] === undefined || isString(row[key]), `${at}.${key} must be a string`)
 	check(errors, row.extracted_by === undefined || isNonEmpty(row.extracted_by), `${at}.extracted_by must be a non-empty string`)
@@ -100,6 +112,12 @@ function validateRow(row, index, errors) {
 	check(errors, row.status === undefined || FACT_STATUSES.includes(row.status), `${at}.status must be one of ${FACT_STATUSES.join(", ")}`)
 }
 
+export function rowErrors(row) {
+	const errors = []
+	validateRow(row, 0, errors)
+	return errors
+}
+
 // The status machine and event rule of references/fact-ledger-grammar.md. validateLedger replays
 // events through transitionError, and every writer in fact-ledger.mjs calls it before appending.
 export const TRANSITIONS = Object.freeze({ active: Object.freeze(["retracted", "superseded"]), retracted: Object.freeze([]), superseded: Object.freeze([]) })
@@ -163,20 +181,19 @@ export function provenanceErrors(rows) {
 		}
 	})
 	const state = new Map()
-	const visit = (id, path) => {
-		if (state.get(id) === "done") return null
-		if (state.get(id) === "open") return [...path.slice(path.indexOf(id)), id]
-		state.set(id, "open")
-		for (const next of refs.get(id)) {
-			const cycle = next !== id && refs.has(next) ? visit(next, [...path, id]) : null
-			if (cycle) return cycle
+	for (const root of refs.keys()) {
+		const stack = [{ id: root, finish: false }]
+		const path = []
+		while (stack.length) {
+			const { id, finish } = stack.pop()
+			if (state.get(id) === "done") continue
+			if (finish) { state.set(id, "done"); path.pop(); continue }
+			if (state.get(id) === "open") return [...errors, `${id} derives from itself through cycle ${[...path, id].join(" -> ")}`]
+			state.set(id, "open")
+			path.push(id)
+			stack.push({ id, finish: true })
+			for (const next of refs.get(id)) if (next !== id && refs.has(next)) stack.push({ id: next, finish: false })
 		}
-		state.set(id, "done")
-		return null
-	}
-	for (const id of refs.keys()) {
-		const cycle = visit(id, [])
-		if (cycle) return [...errors, `${cycle[0]} derives from itself through ${cycle.join(" -> ")}`]
 	}
 	return errors
 }

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-ledger.mjs

@@ -7,8 +7,9 @@ import { existsSync } from "node:fs"
 import { join } from "node:path"
 
 import { CliError, readJsonFile, writeJsonAtomic } from "./cli-support.mjs"
-import { DECIMAL, exactComesFromValue, exactFromDecimal, idPrefix, isString } from "./fact-ledger-kinds.mjs"
+import { DECIMAL, exactComesFromValue, exactFromDecimal, idPrefix, isString, readFlaws, readRow } from "./fact-ledger-kinds.mjs"
 import { LEDGER_SCHEMA_VERSION, ROW_DEFAULTS, ROW_FIELDS, transitionError, validateLedger } from "./fact-ledger-schema.mjs"
+import { normalizeInput } from "./unit-algebra.mjs"
 
 export { exactFromDecimal } from "./fact-ledger-kinds.mjs"
 export { FACT_KINDS, FACT_STATUSES, LEDGER_SCHEMA_VERSION, OPERATIONS, QUALIFIERS, ROW_FIELDS, TRANSITIONS, transitionError, validateLedger } from "./fact-ledger-schema.mjs"
@@ -24,11 +25,24 @@ function assertValid(ledger) {
 	if (errors.length > 0) throw new CliError(`invalid fact ledger: ${errors.join("; ")}`)
 }
 
+/** The row's fields in ROW_FIELDS order, then any unknown field (which validation reports). */
+function ordered(row) {
+	const out = Object.fromEntries(ROW_FIELDS.filter((key) => Object.hasOwn(row, key)).map((key) => [key, row[key]]))
+	for (const key of Object.keys(row)) if (!ROW_FIELDS.includes(key)) out[key] = row[key]
+	return out
+}
+
+/**
+ * Every row comes back as readRow normalizes it (reduced exacts and decimals; source rows with canonical
+ * units and null later columns) before any strict rule runs; the file changes on the next write.
+ */
 export function readLedger(dir) {
 	const file = join(dir, LEDGER_FILENAME)
 	if (!existsSync(file)) return emptyLedger()
 	const ledger = readJsonFile(file)
 	assertValid(ledger)
+	ledger.rows = ledger.rows.map((row) => ordered(readRow(row)))
+	assertValid(ledger)
 	return ledger
 }
 
@@ -44,12 +58,12 @@ const ID_NUMBER = Object.freeze({ F: /^F(\d+)$/, Q: /^Q(\d+)$/ })
 /** The next id with the kind's prefix from KIND_RULES (F<n> for value rows, Q<n> for quotes). */
 export function nextId(ledger, kind) {
 	const prefix = idPrefix(kind)
-	let max = 0
+	let max = 0n
 	for (const row of ledger.rows) {
 		const match = ID_NUMBER[prefix].exec(row.fact_id ?? "")
-		if (match) max = Math.max(max, Number(match[1]))
+		if (match && BigInt(match[1]) > max) max = BigInt(match[1])
 	}
-	return `${prefix}${max + 1}`
+	return `${prefix}${max + 1n}`
 }
 
 /**
@@ -60,14 +74,20 @@ export function nextId(ledger, kind) {
  */
 export function addRow(ledger, row) {
 	const filled = { ...structuredClone(ROW_DEFAULTS), ...row, fact_id: row.fact_id ?? nextId(ledger, row.kind) }
+	if (!["derived", "converted"].includes(filled.kind)) {
+		const normalized = normalizeInput(filled.unit, filled.value)
+		if (normalized.value !== filled.value && filled.exact !== null) throw new CliError("scaled input must omit exact; it is computed after normalization")
+		Object.assign(filled, readRow({ ...filled, ...normalized }))
+	}
 	if (exactComesFromValue(filled.kind) && filled.exact === null && isString(filled.value) && DECIMAL.test(filled.value)) {
 		filled.exact = exactFromDecimal(filled.value)
 	}
-	const ordered = Object.fromEntries(ROW_FIELDS.filter((key) => Object.hasOwn(filled, key)).map((key) => [key, filled[key]]))
-	for (const key of Object.keys(filled)) if (!ROW_FIELDS.includes(key)) ordered[key] = filled[key]
-	assertValid({ ...ledger, rows: [...ledger.rows, ordered] })
-	ledger.rows.push(ordered)
-	return ordered
+	const added = ordered(filled)
+	const flaws = readFlaws(added)
+	if (flaws.length > 0) throw new CliError(`invalid fact ledger: ${flaws.map((flaw) => `${flaw.shape}: ${flaw.detail}`).join("; ")}`)
+	assertValid({ ...ledger, rows: [...ledger.rows, added] })
+	ledger.rows.push(added)
+	return added
 }
 
 /** Build the event from the row's current status and append it only if transitionError allows it. */