lazycodex-ai 5.1.45 → 5.1.46

package/packages/shared-skills/skills/ulw-research/scripts/fact-fill.mjs

@@ -0,0 +1,194 @@
+// fact-fill.mjs - the fill step: substitute every fact pointer in a report source from the
+// ledger, all or nothing. Fatal errors (pointer_label_mismatch, unresolved_placeholder,
+// quote_anchor_unresolved, unit_duplicated) return out null; warnings (particle_unverified,
+// fallback_without_entity) never block. A twice-failed sentence can be replaced by an attributed
+// block quote of its source row (replaceFailed). No IO, but not pure: unless options.view is set,
+// it records each replaced sentence in options.replaceFailed through markReplaced. node: builtins only.
+
+import { finalReading, rewriteParticle } from "./particles-ko.mjs"
+import { failureCount, markReplaced, sentenceKey } from "./fidelity-state.mjs"
+import { findPlaceholders, lead, qualifiedText, renderBlockquote, renderQuote, renderValueHtml, subSpan, valueParts } from "./placeholder.mjs"
+import { splitSentences } from "./sentence-split.mjs"
+
+const FORMATS = Object.freeze(["html", "md"])
+const TRANSLATION = Object.freeze({
+	html: /^\s*<span data-translates="(Q\d+)">[\s\S]*?<\/span>/u,
+	md: /^\s*<!-- translates: (Q\d+) -->[^\n]*/u,
+})
+const HEADING = Object.freeze({ html: /<h[1-6]\b([^>]*)>([\s\S]*?)<\/h[1-6]>/giu, md: /^#{1,6}[ \t]+(.+)$/gmu })
+
+const lineOf = (text, at) => text.slice(0, at).split("\n").length
+
+function resolveValue(row, parsed, format) {
+	const { valueText, unitText } = valueParts(row)
+	const text = { full: row.raw, value: valueText, unit: unitText, q: qualifiedText(row) }[parsed.mode]
+	if (text === null) return { code: "quote_anchor_unresolved", message: `${row.fact_id} has qualifier ${row.qualifier}, but its quote holds no qualifier beside "${row.raw}"` }
+	return { row, parsed, visible: text, rendered: format === "html" ? renderValueHtml(row, text) : text }
+}
+
+function resolveQuote(row, parsed, format) {
+	const whole = { text: row.quote, start: 0, end: row.quote.length }
+	const selection = parsed.mode === "lead" ? lead(row.quote, parsed.lead) : parsed.mode === "span" ? subSpan(row.quote, parsed.from, parsed.to) : whole
+	if (selection === null) {
+		return { code: "quote_anchor_unresolved", message: `from "${parsed.from}" and to "${parsed.to}" must each occur exactly once in ${row.fact_id}'s quote, in that order` }
+	}
+	const rendered = renderQuote(row, parsed.selector, selection.text, format)
+	return { row, parsed, visible: `[${row.source_id}]`, rendered, selector: parsed.selector, quote_span: [selection.start, selection.end] }
+}
+
+function resolve(found, rows, format) {
+	const { parsed } = found
+	if (parsed.error) return { code: "unresolved_placeholder", message: parsed.error }
+	const row = rows.get(parsed.id)
+	if (row === undefined) return { code: "unresolved_placeholder", message: `${parsed.id} is not in the fact ledger` }
+	if (row.status !== "active") return { code: "unresolved_placeholder", message: `${parsed.id} is ${row.status}; cite its replacement` }
+	if (row.label !== parsed.label) return { code: "pointer_label_mismatch", message: `${parsed.id} has label "${row.label}", not "${parsed.label}"` }
+	return parsed.kind === "Q" ? resolveQuote(row, parsed, format) : resolveValue(row, parsed, format)
+}
+
+/** A unit typed right after a pointer that already renders it: never absorbed, always an error. */
+function unitDuplicated(text, item) {
+	if (item.parsed.kind !== "F" || (item.parsed.mode !== "full" && item.parsed.mode !== "q")) return false
+	const suffix = item.row.raw.replace(/^[\s\S]*\d/u, "").trim()
+	const units = [item.row.unit, valueParts(item.row).unitText, suffix].filter((unit) => typeof unit === "string" && unit !== "")
+	const following = text.slice(item.end).replace(/^[ \t]+/u, "")
+	return units.some((unit) => following.startsWith(unit) && !/[A-Za-z0-9]/u.test(following[unit.length] ?? ""))
+}
+
+/*
+ * Sentence-key rules (fidelity-state.mjs sentenceKey(slug, ids)); any other producer of keys must match:
+ * - slug source: an HTML heading's id attribute wins over its text; md headings use their text.
+ * - slugify: strip tags, lowercase, turn each run of non-letters/digits (\p{L}\p{N}) into "-",
+ *   trim leading and trailing "-".
+ * - a sentence takes the nearest heading that starts strictly before the sentence's start offset;
+ *   a sentence before the first heading gets slug "".
+ * - fact ids cited in the sentence are de-duplicated before sentenceKey sorts them.
+ */
+function slugify(text) {
+	return text
+		.replace(/<[^>]*>/gu, "")
+		.toLowerCase()
+		.replace(/[^\p{L}\p{N}]+/gu, "-")
+		.replace(/^-+|-+$/gu, "")
+}
+
+function headings(text, format) {
+	return [...text.matchAll(HEADING[format])].map((match) => {
+		const id = format === "html" ? /\bid="([^"]*)"/u.exec(match[1])?.[1] : undefined
+		return { at: match.index, slug: slugify(id ?? match[match.length - 1]) }
+	})
+}
+
+/** Sentence offsets over the source with pointers, comments and (html) tags blanked to equal length. */
+function sentencesOf(text, found, format) {
+	let masked = text
+	for (const item of found) masked = masked.slice(0, item.start) + "x".repeat(item.end - item.start) + masked.slice(item.end)
+	const blank = (part) => part.replace(/[^\n]/gu, " ")
+	masked = masked.replace(/<!--[\s\S]*?-->/gu, blank)
+	if (format === "html") masked = masked.replace(/<[^>]*>/gu, blank)
+	return splitSentences(masked)
+}
+
+function replacements(text, found, items, options, warnings) {
+	const { format, replaceFailed: state } = options
+	const marks = headings(text, format)
+	const out = []
+	for (const sentence of sentencesOf(text, found, format)) {
+		const inside = items.filter((item) => item.start >= sentence.start && item.end <= sentence.end)
+		if (inside.length === 0) continue
+		const slug = marks.filter((mark) => mark.at < sentence.start).at(-1)?.slug ?? ""
+		const key = sentenceKey(slug, [...new Set(inside.map((item) => item.row.fact_id))])
+		if (failureCount(state, key) < 2) continue
+		const rows = inside.map((item) => item.row)
+		const source = rows.find((row) => row.kind === "quote") ?? rows.find((row) => row.source_id !== null && row.quote !== "")
+		if (source === undefined) continue
+		let rendered = options.fallback === "omit" ? "" : renderBlockquote(source, key, format)
+		if (rendered !== "" && format === "md") {
+			if (sentence.start > 0 && text[sentence.start - 1] !== "\n") rendered = `\n\n${rendered}`
+			if (sentence.end < text.length && text[sentence.end] !== "\n") rendered = `${rendered}\n\n`
+		}
+		if (rendered !== "" && source.entity === null) {
+			warnings.push({ code: "fallback_without_entity", token: key, line: lineOf(text, sentence.start), message: `${source.fact_id} has no entity; the block quote cites [${source.source_id}] alone` })
+		}
+		out.push({ start: sentence.start, end: sentence.end, key, source, rendered })
+	}
+	return out
+}
+
+/** The text after a pointer: rewrite a following Korean particle, and map a translation span. */
+function afterPlaceholder(segment, item, context) {
+	const { format, text, warnings, fillMap } = context
+	const probeEnd = format === "html" && segment.includes("<") ? segment.indexOf("<") : segment.length
+	const result = rewriteParticle(segment.slice(0, probeEnd), finalReading(item.visible, null))
+	let next = segment
+	if (result.applied) next = result.text + segment.slice(probeEnd)
+	else if (result.candidate) {
+		warnings.push({ code: "particle_unverified", token: item.token, line: lineOf(text, item.start), message: `the particle after ${item.token} was not rewritten; check it by eye` })
+	}
+	const translation = item.parsed.kind === "Q" ? TRANSLATION[format].exec(next) : null
+	if (translation !== null && translation[1] === item.row.fact_id) {
+		const span = translation[0].trimStart()
+		const start = context.out.length + translation[0].length - span.length
+		fillMap.push({ token: span, fact_id: translation[1], translates: translation[1], start, end: start + span.length, rendered: span })
+	}
+	return next
+}
+
+function assemble(text, parts, context) {
+	let cursor = 0
+	let previous = null
+	const flush = (until) => {
+		const segment = text.slice(cursor, until)
+		context.out += previous?.parsed ? afterPlaceholder(segment, previous, context) : segment
+	}
+	for (const part of parts) {
+		flush(part.start)
+		const start = context.out.length
+		if (part.parsed) {
+			const extra = part.selector === undefined ? {} : { selector: part.selector, quote_span: part.quote_span }
+			context.fillMap.push({ token: part.token, fact_id: part.row.fact_id, start, end: start + part.rendered.length, rendered: part.rendered, ...extra })
+		} else if (part.rendered !== "") {
+			context.fillMap.push({ token: null, fact_id: part.source.fact_id, replaced: part.key, start, end: start + part.rendered.length, rendered: part.rendered })
+		}
+		context.out += part.rendered
+		cursor = part.end
+		previous = part
+	}
+	flush(text.length)
+	return context.out
+}
+
+/**
+ * @param {string} sourceText the report source with {{F..}} / {{Q..}} pointers
+ * @param {{ rows: object[] }} ledger
+ * @param {{ format?: "html"|"md", replaceFailed?: object, fallback?: "quote"|"omit", view?: boolean }} [options]
+ * @returns {{ out: string|null, fillMap: object[], errors: object[], warnings: object[] }}
+ */
+export function fill(sourceText, ledger, options = {}) {
+	const format = options.format ?? "html"
+	if (!FORMATS.includes(format)) throw new TypeError(`format must be one of ${FORMATS.join(", ")}`)
+	const rows = new Map(ledger.rows.map((row) => [row.fact_id, row]))
+	const found = findPlaceholders(sourceText)
+	const errors = []
+	const warnings = []
+	const items = []
+	for (const pointer of found) {
+		const resolved = resolve(pointer, rows, format)
+		const line = lineOf(sourceText, pointer.start)
+		if (resolved.code) errors.push({ code: resolved.code, token: pointer.token, line, message: resolved.message })
+		else {
+			const item = { ...pointer, ...resolved }
+			if (unitDuplicated(sourceText, item)) {
+				errors.push({ code: "unit_duplicated", token: pointer.token, line, message: `${pointer.token} already renders its unit; use {{${item.row.fact_id}:${item.row.label}:value}}` })
+			} else items.push(item)
+		}
+	}
+	if (errors.length > 0) return { out: null, fillMap: [], errors, warnings }
+	const replaced = options.replaceFailed ? replacements(sourceText, found, items, { ...options, format }, warnings) : []
+	const kept = items.filter((item) => !replaced.some((block) => item.start >= block.start && item.end <= block.end))
+	const parts = [...kept, ...replaced].sort((a, b) => a.start - b.start)
+	const context = { format, text: sourceText, warnings, fillMap: [], out: "" }
+	const out = assemble(sourceText, parts, context)
+	if (!options.view) for (const block of replaced) markReplaced(options.replaceFailed, block.key)
+	return { out, fillMap: context.fillMap, errors, warnings }
+}

package/packages/shared-skills/skills/ulw-research/scripts/fact-fill.test.ts

@@ -0,0 +1,221 @@
+import { describe, expect, test } from "bun:test"
+import { readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { fill } from "./fact-fill.mjs"
+import { retractRow, supersedeRow, validateLedger } from "./fact-ledger.mjs"
+import { failureCount, sentenceKey } from "./fidelity-state.mjs"
+
+const LEDGER = JSON.parse(readFileSync(join(import.meta.dir, "tests/fixtures/fidelity/ledger-basic.json"), "utf8"))
+const F12_DATA = '<data class="fact" value="320" data-fact="F12" data-unit="m" data-lineage="MEASURED">320 m</data>'
+const QUOTE = "we estimate completion no earlier than 2027"
+// Korean particles and words, as escapes: eul, girok, i, ollatda, ga, isang.
+const EUL = "\uC744"
+const GIROK = "\uAE30\uB85D"
+const I = "\uC774"
+const OLLATDA = "\uC62C\uB790\uB2E4"
+const GA = "\uAC00"
+const ISANG = "\uC774\uC0C1"
+
+type State = { schemaVersion: number; failures: Record<string, number>; replaced: string[] }
+function failedTwice(key: string): State {
+	return { schemaVersion: 1, failures: { [key]: 2 }, replaced: [] }
+}
+const codes = (list: { code: string }[]) => list.map((item) => item.code)
+
+describe("fill: fixture", () => {
+	test("the ledger fixture is a valid ledger", () => {
+		expect(validateLedger(LEDGER)).toEqual([])
+	})
+})
+
+describe("fill: value placeholders (html)", () => {
+	test("{{F12:span_len}} renders the <data> element with the raw text", () => {
+		const result = fill("<p>The span is {{F12:span_len}}.</p>", LEDGER, { format: "html" })
+		expect(result.errors).toEqual([])
+		expect(result.out).toBe(`<p>The span is ${F12_DATA}.</p>`)
+		expect(result.fillMap).toHaveLength(1)
+		expect(result.fillMap[0]).toMatchObject({ token: "{{F12:span_len}}", fact_id: "F12", rendered: F12_DATA })
+	})
+
+	test(":q copies the source qualifier with the quantity", () => {
+		const result = fill("<p>{{F12:span_len:q}}</p>", LEDGER, { format: "html" })
+		expect(result.out).toContain('data-lineage="MEASURED">up to 320 m</data>')
+	})
+
+	test(":value and :unit render one part each", () => {
+		const result = fill("<p>{{F12:span_len:value}} metres, unit {{F12:span_len:unit}}</p>", LEDGER, { format: "md" })
+		expect(result.out).toBe("<p>320 metres, unit m</p>")
+	})
+
+	test("a converted row renders DERIVED lineage", () => {
+		const result = fill("<p>{{F40:span_mm}}</p>", LEDGER, { format: "html" })
+		expect(result.out).toContain('data-fact="F40" data-unit="mm" data-lineage="DERIVED">320000 mm</data>')
+	})
+
+	test("an all-English template fills with no warnings", () => {
+		const result = fill("The span is {{F12:span_len}} long.", LEDGER, { format: "html" })
+		expect(result.out).not.toBeNull()
+		expect(result.warnings).toEqual([])
+	})
+})
+
+describe("fill: fatal errors write nothing", () => {
+	test("a label mismatch is pointer_label_mismatch with out null", () => {
+		const result = fill("<p>Line one.</p>\n<p>{{F12:wrong}}</p>", LEDGER, { format: "html" })
+		expect(result.out).toBeNull()
+		expect(result.errors[0]).toMatchObject({ code: "pointer_label_mismatch", token: "{{F12:wrong}}", line: 2 })
+	})
+
+	test("an unknown id is unresolved_placeholder; no row is guessed", () => {
+		const result = fill("<p>{{F99:x}}</p>", LEDGER, { format: "html" })
+		expect(result.out).toBeNull()
+		expect(codes(result.errors)).toEqual(["unresolved_placeholder"])
+	})
+
+	test("a malformed pointer is unresolved_placeholder", () => {
+		expect(codes(fill("<p>{{F12}}</p>", LEDGER, { format: "html" }).errors)).toEqual(["unresolved_placeholder"])
+	})
+
+	test("a unit typed after a value placeholder is unit_duplicated, never absorbed", () => {
+		const result = fill("<p>{{F13:rate}}%</p>", LEDGER, { format: "html" })
+		expect(result.out).toBeNull()
+		expect(codes(result.errors)).toEqual(["unit_duplicated"])
+		expect(fill("<p>{{F13:rate:value}}%</p>", LEDGER, { format: "html" }).errors).toEqual([])
+	})
+
+	test("a retracted row is unresolved_placeholder; fill never renders it", () => {
+		const ledger = structuredClone(LEDGER)
+		retractRow(ledger, "F12", "source withdrawn")
+		expect(validateLedger(ledger)).toEqual([])
+		const result = fill("<p>Line one.</p>\n<p>{{F12:span_len}}</p>", ledger, { format: "html" })
+		expect(result.out).toBeNull()
+		expect(result.fillMap).toEqual([])
+		expect(result.errors).toEqual([{ code: "unresolved_placeholder", token: "{{F12:span_len}}", line: 2, message: "F12 is retracted; cite its replacement" }])
+	})
+
+	test("a superseded row is unresolved_placeholder; fill never renders it", () => {
+		const ledger = structuredClone(LEDGER)
+		supersedeRow(ledger, "F12", "F40", "restated in mm")
+		expect(validateLedger(ledger)).toEqual([])
+		const result = fill("{{F12:span_len:value}}", ledger, { format: "md" })
+		expect(result.out).toBeNull()
+		expect(result.fillMap).toEqual([])
+		expect(result.errors).toEqual([{ code: "unresolved_placeholder", token: "{{F12:span_len:value}}", line: 1, message: "F12 is superseded; cite its replacement" }])
+	})
+
+	test("an anchor occurring twice is quote_anchor_unresolved", () => {
+		const result = fill('<p>{{Q7:completion:from="ti":to="2027"}}</p>', LEDGER, { format: "html" })
+		expect(result.out).toBeNull()
+		expect(codes(result.errors)).toEqual(["quote_anchor_unresolved"])
+	})
+})
+
+describe("fill: quote placeholders", () => {
+	test("from/to renders the sub-span in quotes plus the source marker", () => {
+		const result = fill('<p>{{Q7:completion:from="no earlier":to="2027"}}</p>', LEDGER, { format: "html" })
+		expect(result.out).toBe('<p><q data-fact="Q7" data-quote-sel="from=no earlier;to=2027">\u2026no earlier than 2027</q> [S4]</p>')
+	})
+
+	test("full and lead selectors", () => {
+		expect(fill("{{Q7:completion}}", LEDGER, { format: "html" }).out).toBe(`<q data-fact="Q7" data-quote-sel="full">${QUOTE}</q> [S4]`)
+		expect(fill("{{Q7:completion:lead=2}}", LEDGER, { format: "html" }).out).toBe('<q data-fact="Q7" data-quote-sel="lead=2">we estimate\u2026</q> [S4]')
+	})
+
+	test("md format puts the selector into fillMap", () => {
+		const result = fill("{{Q7:completion:lead=2}}", LEDGER, { format: "md" })
+		expect(result.out).toBe('"we estimate\u2026" [S4]')
+		expect(result.fillMap[0]).toMatchObject({ fact_id: "Q7", selector: "lead=2", quote_span: [0, 11] })
+	})
+
+	test("a translation span after a quote placeholder is preserved and mapped", () => {
+		const source = '{{Q7:completion}} <span data-translates="Q7">translation</span> rest'
+		const result = fill(source, LEDGER, { format: "html" })
+		expect(result.out).toContain('<span data-translates="Q7">translation</span> rest')
+		const entry = result.fillMap.find((item: { translates?: string }) => item.translates === "Q7")
+		expect(entry).toBeDefined()
+		expect(result.out.slice(entry.start, entry.end)).toBe(entry.rendered)
+		const md = fill("{{Q7:completion}} <!-- translates: Q7 --> translation", LEDGER, { format: "md" })
+		expect(md.fillMap.some((item: { translates?: string }) => item.translates === "Q7")).toBe(true)
+	})
+})
+
+describe("fill: Korean particles", () => {
+	test("a closed final keeps eul", () => {
+		const result = fill(`<p>{{F21:hbm_rev}}${EUL} ${GIROK}</p>`, LEDGER, { format: "html" })
+		expect(result.out).toContain(`</data>${EUL} ${GIROK}`)
+		expect(result.warnings).toEqual([])
+	})
+
+	test("percent is open: i becomes ga", () => {
+		const result = fill(`<p>{{F13:rate}}${I} ${OLLATDA}</p>`, LEDGER, { format: "html" })
+		expect(result.out).toContain(`15.2%</data>${GA} ${OLLATDA}`)
+	})
+
+	test("a candidate particle with no boundary is untouched and warned", () => {
+		const result = fill(`<p>{{F13:rate}}${ISANG}</p>`, LEDGER, { format: "html" })
+		expect(result.out).toContain(`</data>${ISANG}</p>`)
+		expect(codes(result.warnings)).toEqual(["particle_unverified"])
+	})
+})
+
+describe("fill: md maps", () => {
+	test("fillMap offsets slice to the rendered text", () => {
+		const result = fill(`# Findings\n\nSpan {{F12:span_len}}, rate {{F13:rate}}${I} ${OLLATDA}, {{Q7:completion:lead=3}}.\n`, LEDGER, { format: "md" })
+		expect(result.errors).toEqual([])
+		expect(result.fillMap).toHaveLength(3)
+		for (const entry of result.fillMap) expect(result.out.slice(entry.start, entry.end)).toBe(entry.rendered)
+		expect(result.fillMap[0].rendered).toBe("320 m")
+		expect(result.out).toContain(`15.2%${GA}`)
+	})
+})
+
+describe("fill: replace-failed", () => {
+	const SOURCE = '<h2 id="findings">Findings</h2>\n<p>Completion slips: {{Q7:completion}}. Other text stays.</p>'
+	const KEY = sentenceKey("findings", ["Q7"])
+
+	test("a key that failed twice becomes the attributed blockquote; no entity warns", () => {
+		const state = failedTwice(KEY)
+		const result = fill(SOURCE, LEDGER, { format: "html", replaceFailed: state })
+		expect(result.out).not.toBeNull()
+		expect(result.out).toContain(`<blockquote data-fact="Q7" data-replaced="${KEY}">`)
+		expect(result.out).toContain(`"${QUOTE}"`)
+		expect(result.out).toContain("<footer>[S4]</footer>")
+		expect(result.out).toContain("Other text stays.</p>")
+		expect(result.out).not.toContain("Completion slips")
+		expect(codes(result.warnings)).toEqual(["fallback_without_entity"])
+		expect(state.replaced).toEqual([KEY])
+	})
+
+	test("a row with entity and valid_at carries them in the footer", () => {
+		const source = `<h2>Market</h2>\n<p>Revenue was {{F21:hbm_rev}}.</p>`
+		const result = fill(source, LEDGER, { format: "html", replaceFailed: failedTwice(sentenceKey("market", ["F21"])) })
+		expect(result.out).toContain("<footer>[S5], on HBM3E (2026 Q3)</footer>")
+		expect(result.warnings).toEqual([])
+	})
+
+	test("a key below two failures is filled normally", () => {
+		const state: State = { schemaVersion: 1, failures: { [KEY]: 1 }, replaced: [] }
+		const result = fill(SOURCE, LEDGER, { format: "html", replaceFailed: state })
+		expect(result.out).toContain("Completion slips:")
+		expect(state.replaced).toEqual([])
+	})
+
+	test("fallback omit removes the sentence", () => {
+		const result = fill(SOURCE, LEDGER, { format: "html", replaceFailed: failedTwice(KEY), fallback: "omit" })
+		expect(result.out).toBe('<h2 id="findings">Findings</h2>\n<p> Other text stays.</p>')
+	})
+
+	test("view leaves the state unchanged", () => {
+		const state = failedTwice(KEY)
+		const result = fill(SOURCE, LEDGER, { format: "html", replaceFailed: state, view: true })
+		expect(result.out).toContain("<blockquote")
+		expect(state.replaced).toEqual([])
+		expect(failureCount(state, KEY)).toBe(2)
+	})
+
+	test("md renders a > block with the footer line", () => {
+		const result = fill("## Findings\n\nCompletion slips: {{Q7:completion}}.\n", LEDGER, { format: "md", replaceFailed: failedTwice(KEY) })
+		expect(result.out).toBe(`## Findings\n\n> "${QUOTE}"\n> [S4]\n`)
+	})
+})

package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger-grammar.test.ts

@@ -39,11 +39,14 @@ function derivedRow(label: string, from: string[], kind = "derived") {
 		extracted_by: "fact derive",
 		observed_at: "2026-10-23T08:00:00Z",
 		label,
-		operation: kind === "converted" ? "convert" : "sum",
+		operation: kind === "converted" ? "convert" : "expr",
 		operands: [...from],
 		derived_from: [...from],
 		factor: kind === "converted" ? "1000" : null,
 		exact: { num: "2", den: "1" },
+		formula: from.join(" + "), render: "2", precision: { places: 0, sigfigs: 1 },
+		compatibility: kind === "converted" ? "checked" : "unchecked",
+		context: kind === "converted" ? { target: "km" } : null,
 	}
 }
 
@@ -90,7 +93,11 @@ function randomLedger(seed: number) {
 		const roll = random()
 		if (roll < 0.3 || values.length < 2) addRow(ledger, measured(`m${step}`))
 		else if (roll < 0.4) addRow(ledger, quoteRow(`q${step}`))
-		else if (roll < 0.6) addRow(ledger, derivedRow(`d${step}`, [...new Set([pick(random, values), pick(random, values)])], pick(random, ["derived", "converted"])))
+		else if (roll < 0.6) {
+			const kind = pick(random, ["derived", "converted"])
+			const from = kind === "converted" ? [pick(random, values)] : [...new Set([pick(random, values), pick(random, values)])]
+			addRow(ledger, derivedRow(`d${step}`, from, kind))
+		}
 		else if (roll < 0.8 && active.length > 0) retractRow(ledger, pick(random, active), "wrong page")
 		else if (active.length > 1) {
 			const id = pick(random, active)

package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger-kinds.mjs

@@ -5,6 +5,11 @@
 // Also the value primitives the schema shares. Pure functions with no IO; node: builtins only.
 
 import { isCliEntry } from "./entry-guard.mjs"
+import { canonicalUnit, inputScale, RuleError } from "./unit-algebra.mjs"
+import { metadataErrors } from "./unit-metadata.mjs"
+import { mul } from "./rational.mjs"
+import { placesOf, roundPlaces, toBig } from "./unit-precision.mjs"
+import { OPERATION_RULES } from "./unit-rules.mjs"
 
 export const DECIMAL = /^-?\d+(?:\.\d+)?$/
 const INTEGER = /^-?\d+$/
@@ -16,7 +21,15 @@ export const isString = (value) => typeof value === "string"
 export const isNonEmpty = (value) => typeof value === "string" && value.length > 0
 export const isCount = (value) => Number.isInteger(value) && value >= 0
 const isDecimal = (value) => isString(value) && DECIMAL.test(value)
-const isRational = (exact) => isObject(exact) && isString(exact.num) && isString(exact.den) && INTEGER.test(exact.num) && /^\d+$/.test(exact.den) && !/^0+$/.test(exact.den)
+export function isRational(exact) {
+	if (!isObject(exact) || Object.keys(exact).sort().join(",") !== "den,num" || !isString(exact.num) || !isString(exact.den) || !INTEGER.test(exact.num) || !/^[1-9]\d*$/.test(exact.den)) return false
+	let a = BigInt(exact.num)
+	let b = BigInt(exact.den)
+	if (String(a) !== exact.num) return false
+	if (a < 0n) a = -a
+	while (b !== 0n) [a, b] = [b, a % b]
+	return a === 1n
+}
 
 /** The exact rational of a decimal string, reduced. */
 export function exactFromDecimal(value) {
@@ -72,22 +85,27 @@ export const RULES = Object.freeze({
 		test: (v, row) => isRational(v) && (!isDecimal(row.value) || (v.num === exactFromDecimal(row.value).num && v.den === exactFromDecimal(row.value).den)),
 	},
 	rational_or_null: { cell: "rational or null", says: "null or { num: integer string, den: positive integer string }", test: orNull(isRational) },
-	by_derive: { cell: "written by derive (rational or null)", says: "null or { num: integer string, den: positive integer string }", test: orNull(isRational) },
+	by_derive: { cell: "reduced rational", says: "a reduced { num: integer string, den: positive integer string }", test: isRational },
+	context_by_derive: { cell: "written by derive (object or null)", says: "null or an object", test: orNull(isObject) },
+	derived_operation: { cell: "sum/diff/growth/share/ratio/expr", says: "a derived operation", test: (v) => Object.hasOwn(OPERATION_RULES, v) && OPERATION_RULES[v].kind === "derived" },
+	convert_operation: { cell: "`convert`", says: "convert", test: (v) => Object.hasOwn(OPERATION_RULES, v) && OPERATION_RULES[v].kind === "converted" },
+	precision: { cell: "`{ places, sigfigs }`", says: "nonnegative places and positive sigfigs", test: (v) => isObject(v) && Object.keys(v).sort().join(",") === "places,sigfigs" && Number.isSafeInteger(v.places) && v.places >= 0 && Number.isSafeInteger(v.sigfigs) && v.sigfigs >= 1 },
+	compatibility: { cell: "checked or unchecked (expr)", says: "the operation's compatibility", test: (v, r) => Object.hasOwn(OPERATION_RULES, r.operation) && v === OPERATION_RULES[r.operation].compatibility },
 })
 
 /** The grammar's per-kind columns, in table order; `exact` follows `value` because it reads it. */
-export const KIND_COLUMNS = Object.freeze(["fact_id", "value", "source_id", "locator", "snapshot_sha256", "rationale", "exact"])
+export const KIND_COLUMNS = Object.freeze(["fact_id", "value", "source_id", "locator", "snapshot_sha256", "rationale", "exact", "context", "operation", "precision", "compatibility"])
 
 const sourced = { source_id: "source", locator: "locator", snapshot_sha256: "sha256" }
 const unsourced = { source_id: "source_or_null", locator: "locator_or_null", snapshot_sha256: "sha256_or_null" }
-const row = (cells) => Object.freeze(Object.fromEntries(KIND_COLUMNS.map((column) => [column, cells[column]])))
+const row = (cells) => Object.freeze(Object.fromEntries(KIND_COLUMNS.map((column) => [column, cells[column] ?? (["precision", "compatibility"].includes(column) && cells.operation !== "null" ? column : "null")])))
 
 export const KIND_RULES = Object.freeze({
-	measured: row({ fact_id: "f_id", value: "decimal", ...sourced, rationale: "string_or_null", exact: "from_value" }),
-	quote: row({ fact_id: "q_id", value: "null", ...sourced, rationale: "string_or_null", exact: "null" }),
-	derived: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "string_or_null", exact: "by_derive" }),
-	assumed: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "non_empty", exact: "rational_or_null" }),
-	converted: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "string_or_null", exact: "by_derive" }),
+	measured: row({ fact_id: "f_id", value: "decimal", ...sourced, rationale: "string_or_null", exact: "from_value", context: "null", operation: "null" }),
+	quote: row({ fact_id: "q_id", value: "null", ...sourced, rationale: "string_or_null", exact: "null", context: "null", operation: "null" }),
+	derived: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "string_or_null", exact: "by_derive", context: "context_by_derive", operation: "derived_operation" }),
+	assumed: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "non_empty", exact: "rational_or_null", context: "null", operation: "null" }),
+	converted: row({ fact_id: "f_id", value: "decimal", ...unsourced, rationale: "string_or_null", exact: "by_derive", context: "context_by_derive", operation: "convert_operation" }),
 })
 
 /** The id prefix nextId assigns for a kind; an unknown kind gets F and fails validation later. */
@@ -100,15 +118,101 @@ export function exactComesFromValue(kind) {
 	return KIND_RULES[kind]?.exact === "from_value"
 }
 
+/** Columns added after schema version 1 shipped; a ledger written before them still loads (dimension 31). */
+export const LATER_COLUMNS = Object.freeze(["context", "precision", "compatibility"])
+const DERIVED_KINDS = Object.freeze(["derived", "converted"])
+const PRE_REGISTRY_COLUMNS = KIND_COLUMNS.slice(0, KIND_COLUMNS.indexOf("exact") + 1)
+
+/** A derived or converted row written before the later columns existed: loadable, never an operand. */
+export const isLegacyDerived = (row) => isObject(row) && DERIVED_KINDS.includes(row.kind) && LATER_COLUMNS.every((column) => !Object.hasOwn(row, column))
+/** A derived or converted row carrying this version's columns: the only rows checked strictly at read time. */
+export const isCurrentDerived = (row) => isObject(row) && DERIVED_KINDS.includes(row.kind) && !isLegacyDerived(row)
+
+/** An exact carrying keys beyond num and den: never normalized, never silently stripped, flagged on use. */
+export const hasUnknownExactKeys = (exact) => isObject(exact) && isString(exact.num) && isString(exact.den) && Object.keys(exact).sort().join(",") !== "den,num"
+/** Any integer pair dev's writer accepted, reduced: sign on num, no leading zeros, gcd 1; anything else as given. */
+function readExact(exact) {
+	if (!isObject(exact) || hasUnknownExactKeys(exact) || !isString(exact.num) || !isString(exact.den) || !INTEGER.test(exact.num) || !/^\d+$/.test(exact.den) || /^0+$/.test(exact.den)) return exact
+	const { num, den } = mul({ num: BigInt(exact.num), den: BigInt(exact.den) }, 1n)
+	return { num: String(num), den: String(den) }
+}
+
+const readDecimal = (value) => (isDecimal(value) ? value.replace(/^(-?)0+(?=\d)/, "$1") : value)
+
+/** A source unit read through the alias table; unknown, scaled, or unparseable text stays as given. */
+export function sourceUnit(unit) {
+	try {
+		const canonical = canonicalUnit(unit)
+		return isString(canonical) && !inputScale(unit) ? canonical : unit
+	} catch (error) {
+		if (error instanceof RuleError) return unit
+		throw error
+	}
+}
+
+/**
+ * A row as every reader sees it (dimension 31 wins at read time): rows not written by derive in this
+ * version get reduced exacts and decimals; source rows also get canonical units and null later columns.
+ * Current derived rows stay as stored so tampering is still detected. Idempotent.
+ */
+export function readRow(row) {
+	if (!isObject(row) || !Object.hasOwn(KIND_RULES, row.kind) || isCurrentDerived(row)) return row
+	const read = { ...row, value: readDecimal(row.value), exact: exactComesFromValue(row.kind) ? row.exact : readExact(row.exact) }
+	if (DERIVED_KINDS.includes(row.kind)) return read
+	read.unit = sourceUnit(row.unit)
+	for (const column of LATER_COLUMNS) if (!Object.hasOwn(read, column)) read[column] = null
+	return read
+}
+
+const carriesDerivation = (row) => row.operation !== null || row.operands?.length || row.derived_from?.length || row.formula !== null || row.factor !== null
+
+/** Dimension 32: a stored exact must equal the row's own value at the value's decimal places. */
+function valueExactDisagreement(row) {
+	const exact = readExact(row.exact)
+	const value = readDecimal(row.value)
+	if (!isRational(exact) || !isDecimal(value) || roundPlaces(toBig(exact), placesOf(value)) === value) return null
+	return `value ${value} disagrees with exact ${exact.num}/${exact.den} at ${placesOf(value)} decimal places`
+}
+function unitFlaw(unit) {
+	try {
+		return isString(unit) && (canonicalUnit(unit) !== unit || inputScale(unit)) ? `unit ${unit} has no canonical stored text` : null
+	} catch (error) {
+		if (error instanceof RuleError) return `unit ${unit}: ${error.message}`
+		throw error
+	}
+}
+
+/**
+ * What a loadable source row (read through readRow) still breaks under the strict dimensions: dev's
+ * writer accepted it, so it loads, but derive and recompute refuse it as an operand with this shape.
+ * addRow refuses a new row that has any. Empty for derived and converted rows.
+ */
+export function readFlaws(row) {
+	if (!isObject(row) || DERIVED_KINDS.includes(row.kind)) return []
+	const flaws = []
+	if (carriesDerivation(row)) flaws.push({ shape: "dimension 1", detail: "a source row carries an operation or derivation fields", expected: "null operation and empty derivation fields on measured, quote, and assumed rows" })
+	const unit = unitFlaw(row.unit)
+	if (unit) flaws.push({ shape: "dimension 6", detail: unit, expected: "a registered or canonical unit without scale" })
+	for (const detail of metadataErrors(row)) flaws.push({ shape: "dimension 18", detail, expected: "present, type-valid metadata; explicit null when unknown" })
+	const unknownKeys = hasUnknownExactKeys(row.exact)
+	const disagreement = unknownKeys ? null : valueExactDisagreement(row)
+	if (unknownKeys) flaws.push({ shape: "dimension 7", detail: `exact has unknown keys: ${Object.keys(row.exact).filter((key) => key !== "num" && key !== "den").join(", ")}`, expected: "an exact with exactly num and den (the stored row is kept as written)" })
+	if (disagreement) flaws.push({ shape: "dimension 32", detail: disagreement, expected: "an exact equal to value at the value's decimal places" })
+	return flaws
+}
+
 /** @returns {string[]} every per-kind column error of one row; empty for an unknown kind. */
 export function kindRuleErrors(candidate, at) {
 	const rules = KIND_RULES[candidate.kind]
 	if (rules === undefined) return []
 	const errors = []
-	for (const column of KIND_COLUMNS) {
-		const value = candidate[column]
-		if (value === undefined) continue
-		const rule = RULES[rules[column]]
+	const legacy = isLegacyDerived(candidate)
+	const columns = legacy ? PRE_REGISTRY_COLUMNS : isCurrentDerived(candidate) ? KIND_COLUMNS : KIND_COLUMNS.filter((column) => column !== "operation")
+	for (const column of columns) {
+		const value = candidate[column] === undefined && LATER_COLUMNS.includes(column) ? null : candidate[column]
+		const rule = RULES[legacy && column === "exact" ? "rational_or_null" : rules[column]]
+		// An exact with unknown keys loads as written (never stripped); readFlaws refuses it at use and at write.
+		if (column === "exact" && hasUnknownExactKeys(value) && !isCurrentDerived(candidate)) continue
 		if (rule.test(value, candidate)) continue
 		errors.push(`${at}.${column} must be ${rule.says} for a ${candidate.kind} row`)
 		if (rule.detail) errors.push(...rule.detail(value, `${at}.${column}`))
@@ -116,6 +220,16 @@ export function kindRuleErrors(candidate, at) {
 	return errors
 }
 
+/** At-rest rules of current derived rows; source rows answer to readFlaws at use and at write instead. */
+export function storedErrors(row) {
+	if (!isCurrentDerived(row)) return []
+	const errors = metadataErrors(row)
+	try {
+		if (canonicalUnit(row.unit) !== row.unit || inputScale(row.unit)) errors.push("unit must be canonical stored text")
+	} catch (error) { errors.push(error.message) }
+	return errors
+}
+
 export const KIND_TABLE_BEGIN = "<!-- kind-rules:begin (rendered from KIND_RULES; do not edit by hand) -->"
 export const KIND_TABLE_END = "<!-- kind-rules:end -->"