lazycodex-ai 5.1.44 → 5.1.46

package/packages/shared-skills/skills/ulw-research/scripts/gates-fidelity.mjs

@@ -0,0 +1,113 @@
+// gates-fidelity.mjs - the deterministic fidelity gate over a filled HTML report.
+//
+// checkFidelity refuses to judge a report whose brief has no locked `## Fidelity` block (one
+// fidelity_interview_incomplete defect). Otherwise it builds the static-gate document context
+// once and runs the value, binding, and lexicon checks over it, returning defects in the shape
+// `check` emits plus integrity, sorted blocker, major, minor, then line, selector, code. It never
+// calls a model or reads the network; the meaning verifier consumes its pairs separately.
+// Plain Node ESM, sibling imports only.
+
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { DEFECT_CODES } from "./contracts.mjs"
+import { bindingChecks, proseSentences } from "./fidelity-binding.mjs"
+import { isLocked, parseFidelityBlock } from "./fidelity-brief.mjs"
+import { lexiconChecks } from "./fidelity-lexicon.mjs"
+import { valueChecks } from "./fidelity-values.mjs"
+import { buildContext } from "./gates-static.mjs"
+import { parseHtml } from "./html-lite.mjs"
+
+/** Every code this gate can raise. */
+export const FIDELITY_GATE_CODES = Object.freeze([
+	"fidelity_interview_incomplete", "unresolved_placeholder", "value_mismatch", "unit_mismatch", "quote_not_verbatim",
+	"quote_unbound", "claim_status_invalid", "snapshot_hash_mismatch", "chart_value_untraceable", "untraceable_number",
+	"unsupported_verification", "conflicting_readings", "hedge_dropped", "direction_flipped", "causality_added",
+	"qualifier_dropped", "unbound_sentence", "snapshot_mismatch", "join_integrity_fail", "quote_stale_version",
+	"quote_fuzzy_match", "name_outside_quote", "scope_widened_warn", "claim_kind_mismatch", "verification_claim_unmarked",
+])
+const RANK = { blocker: 0, major: 1, minor: 2 }
+
+/** The gate and delivery barrier share registry policy; caller severities cannot demote it. */
+export function blocksDelivery(defect) {
+	if (!Object.hasOwn(DEFECT_CODES, defect?.code)) return true
+	const entry = DEFECT_CODES[defect.code]
+	return entry.integrity || entry.severity !== "minor"
+}
+
+export function deliveryBlockReason(defect) {
+	return Object.hasOwn(DEFECT_CODES, defect?.code) ? `fidelity_unresolved: ${defect.code}` : `unknown defect code ${defect?.code}`
+}
+
+function withIntegrity(defect) {
+	const { code, severity, selector, line, message, hint } = defect
+	return { code, severity, integrity: DEFECT_CODES[code].integrity, selector, line, message, hint }
+}
+
+function compareDefects(a, b) {
+	const text = (x, y) => (x < y ? -1 : x > y ? 1 : 0)
+	return RANK[a.severity] - RANK[b.severity] || (a.line ?? 0) - (b.line ?? 0) || text(a.selector ?? "", b.selector ?? "") || text(a.code, b.code)
+}
+
+function readAcronyms(sourcesDir, sourceId, cache) {
+	if (!sourcesDir || !sourceId) return {}
+	if (!cache.has(sourceId)) {
+		const path = join(sourcesDir, sourceId, "acronyms.json")
+		cache.set(sourceId, existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : {})
+	}
+	return cache.get(sourceId)
+}
+
+/** One (row quote, report sentence) pair per fact cited in a prose sentence outside sources and toc. */
+export function fidelityPairs(ctx, ledger, sourcesDir) {
+	const rows = new Map((ledger?.rows ?? []).map((row) => [row.fact_id, row]))
+	const cache = new Map()
+	const pairs = []
+	for (const block of proseSentences(ctx)) {
+		if (block.kind === "sources" || block.kind === "toc") continue
+		block.sentences.forEach((sentence, index) => {
+			for (const fact of sentence.facts) {
+				const row = rows.get(fact.id)
+				if (!row?.quote) continue
+				pairs.push({
+					pair_id: `${fact.id}@${block.owner.line}:${index + 1}`,
+					fact_id: fact.id,
+					quote: row.quote,
+					sentence: sentence.text,
+					span: [fact.start - sentence.start, fact.end - sentence.start],
+					qualifier: row.kind === "quote" ? null : row.qualifier,
+					acronyms: readAcronyms(sourcesDir, row.source_id, cache),
+					node: block.owner,
+				})
+			}
+		})
+	}
+	return pairs
+}
+
+function incomplete(parsed) {
+	const reasons = !parsed?.present ? "no ## Fidelity block" : [...(parsed.errors ?? []), ...parsed.missing.map((f) => `missing: ${f}`), ...parsed.pending.map((f) => `pending: ${f}`)].join("; ")
+	const entry = DEFECT_CODES.fidelity_interview_incomplete
+	return { code: "fidelity_interview_incomplete", severity: entry.severity, integrity: entry.integrity, selector: null, line: null, message: `fidelity interview incomplete (${reasons})`, hint: entry.hint }
+}
+
+/**
+ * @param {string} html the filled report
+ * @param {{ ledger: { rows: object[] }, sourcesDir?: string | null, claims?: object | null, fillMap?: object[] | null,
+ *   brief: string | object, baseDir?: string, recompute?: (ledger: object, row: object) => { ok: boolean } }} options
+ *   `brief` is the brief.md text or parseFidelityBlock output; `baseDir` resolves data-chart-source paths.
+ * @returns {{ defects: object[], gatesRun: string[] }}
+ */
+export function checkFidelity(html, { ledger, sourcesDir = null, claims = null, fillMap = null, brief, baseDir, recompute } = {}) {
+	const parsed = brief && typeof brief === "object" ? brief : parseFidelityBlock(String(brief ?? ""))
+	if (!isLocked(parsed)) return { defects: [incomplete(parsed)], gatesRun: ["fidelity_interview_incomplete"] }
+	const ctx = buildContext(parseHtml(html), { baseDir })
+	const defects = [
+		...valueChecks(ctx, ledger, fillMap, sourcesDir, { recompute, asOf: parsed.fields["as of"] }),
+		...bindingChecks(ctx, claims, ledger),
+	]
+	const pairs = fidelityPairs(ctx, ledger, sourcesDir)
+	const byId = new Map(pairs.map((pair) => [pair.pair_id, pair]))
+	for (const finding of lexiconChecks(pairs).findings) defects.push(ctx.defect(finding.code, byId.get(finding.pair_id).node, `${finding.pair_id}: ${finding.message}`))
+	return { defects: defects.map(withIntegrity).sort(compareDefects), gatesRun: [...FIDELITY_GATE_CODES] }
+}

package/packages/shared-skills/skills/ulw-research/scripts/gates-fidelity.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, test } from "bun:test"
+import { readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { indexClaims } from "./claims-index.mjs"
+import { DEFECT_CODES, FIDELITY_CODES } from "./contracts.mjs"
+import { checkFidelity } from "./gates-fidelity.mjs"
+
+type Defect = { code: string; severity: string; integrity: boolean; selector: string | null; line: number | null; message: string; hint: string }
+type Result = { defects: Defect[]; gatesRun: string[] }
+
+const FIXTURES = join(import.meta.dir, "tests", "fixtures")
+const FIX = join(FIXTURES, "fidelity")
+const read = (path: string) => readFileSync(path, "utf8")
+const OPTIONS = {
+	ledger: JSON.parse(read(join(FIX, "ledger-verify.json"))),
+	sourcesDir: join(FIX, "sources"),
+	claims: indexClaims(read(join(FIX, "claim-graph.md"))),
+	fillMap: null,
+	brief: read(join(FIXTURES, "fidelity-brief-locked.md")),
+	baseDir: FIX,
+}
+
+function run(html: string, extra: Record<string, unknown> = {}): Result {
+	return checkFidelity(html, { ...OPTIONS, ...extra }) as Result
+}
+
+const codes = (result: Result) => result.defects.map((defect) => defect.code)
+
+describe("checkFidelity", () => {
+	test("#given the filled good report with a locked brief #when checked #then zero defects", () => {
+		expect(run(read(join(FIX, "report-good.html"))).defects).toEqual([])
+	})
+
+	test("#given the bad report #when checked #then the exact defect multiset, blockers first", () => {
+		const result = run(read(join(FIX, "report-bad.html")))
+		expect(codes(result)).toEqual([
+			"unit_mismatch",
+			"value_mismatch",
+			"quote_not_verbatim",
+			"unresolved_placeholder",
+			"claim_status_invalid",
+			"untraceable_number",
+			"unbound_sentence",
+		])
+	})
+
+	test("#given any defect #when emitted #then it has the check shape with severity and integrity from the contract", () => {
+		for (const defect of run(read(join(FIX, "report-bad.html"))).defects) {
+			expect(Object.keys(defect).sort()).toEqual(["code", "hint", "integrity", "line", "message", "selector", "severity"])
+			expect(defect.severity).toBe(DEFECT_CODES[defect.code as keyof typeof DEFECT_CODES].severity)
+			expect(defect.integrity).toBe(DEFECT_CODES[defect.code as keyof typeof DEFECT_CODES].integrity)
+		}
+	})
+
+	test("#given an unlocked brief #when checked #then only fidelity_interview_incomplete", () => {
+		const result = run(read(join(FIX, "report-bad.html")), { brief: read(join(FIXTURES, "fidelity-brief-pending.md")) })
+		expect(codes(result)).toEqual(["fidelity_interview_incomplete"])
+		expect(result.defects[0]?.integrity).toBe(true)
+	})
+
+	test("#given a row qualifier dropped in the report #when checked #then qualifier_dropped, and the :q rendering is clean", () => {
+		const dropped = '<h2>Findings</h2><p>The span is <data class="fact" data-fact="F1" data-unit="m">320 m</data> long [S4].</p>'
+		expect(codes(run(dropped))).toEqual(["qualifier_dropped"])
+		expect(run(dropped.replace(">320 m<", ">up to 320 m<")).defects).toEqual([])
+	})
+
+	test("#given the gate #when it runs #then every code it reports is a fidelity code", () => {
+		const { gatesRun } = run(read(join(FIX, "report-good.html")))
+		expect(gatesRun.length).toBeGreaterThan(0)
+		expect(gatesRun.every((code) => FIDELITY_CODES.includes(code))).toBe(true)
+	})
+})

package/packages/shared-skills/skills/ulw-research/scripts/gates-static.mjs

@@ -18,14 +18,19 @@ import { PROSE_TAGS, proseText, selectorPath } from "./html-lite.mjs"
 const CITATION_MARKER = /\[S\d+\]|\[Source\s*\d+\]|\[\uCD9C\uCC98[^\]]*\]|\[\^\d+\]/i
 const ANCHOR_HREF = /^#(?:s|src|source|ref|fn)/i
 const ANCHOR_TEXT = /^\[(?:\uCD9C\uCC98|Source\s*\d+|S\d+)\]$/i
-/** Closing ("how this report was made") heading; Korean alternates cover the same register. */
-export const CLOSING_HEADING = /\b(?:method(?:ology)?|how)\b|\uB4B7\uBA74|\uBC29\uBC95|\uC5B4\uB5BB\uAC8C|\uB9CC\uB4E4\uC5B4/i
+/**
+ * Closing ("how this report was made") heading; Korean alternates cover the same register. A
+ * heading that is exactly "Closing" (or the Korean word for it) also counts; a heading that only
+ * starts with it ("Closing the gap") does not.
+ */
+export const CLOSING_HEADING = /\b(?:method(?:ology)?|how)\b|\uB4B7\uBA74|\uBC29\uBC95|\uC5B4\uB5BB\uAC8C|\uB9CC\uB4E4\uC5B4|^\s*(?:closing|\uB9C8\uBB34\uB9AC)\s*$/i
 export const SOURCES_HEADING = /references|sources|bibliography|\uCD9C\uCC98|\uCC38\uACE0\s*(?:\uBB38\uD5CC|\uC790\uB8CC)/i
-const OTHER_KINDS = [
+/** Section kinds after sources and closing, in match order; HTML and Markdown share this table. */
+export const OTHER_KINDS = Object.freeze([
 	["toc", /\b(?:contents|toc)\b|\uBAA9\uCC28/i],
 	["appendix", /appendix|\uBD80\uB85D/i],
 	["glossary", /glossary|\uC6A9\uC5B4/i],
-]
+])
 const TEXT_SKIP = new Set(["code", "pre", "kbd", "samp", "script", "style", "svg", "template", "noscript"])
 const HEADING_TAG = /^h[1-6]$/
 /** The one sanctioned severity override: section-level lineage demotes unsourced_number. */
@@ -75,9 +80,25 @@ function findBody(doc) {
 	return doc
 }
 
-function sectionKind(text, closingTest) {
+/**
+ * Closing-heading test for both editions: CLOSING_HEADING plus the spec's optional pattern.
+ * @param {RegExp|string|null} [pattern]
+ * @returns {(text: string) => boolean}
+ */
+export function closingTest(pattern) {
+	const extra = pattern ? toRegExp(pattern) : null
+	return (text) => CLOSING_HEADING.test(text) || Boolean(extra?.test(text))
+}
+
+/**
+ * Section kind of an h2 heading text: sources, closing, toc, appendix, glossary, or null.
+ * @param {string} text
+ * @param {(text: string) => boolean} isClosing
+ * @returns {string|null}
+ */
+export function sectionKind(text, isClosing) {
 	if (SOURCES_HEADING.test(text)) return "sources"
-	if (closingTest(text)) return "closing"
+	if (isClosing(text)) return "closing"
 	return OTHER_KINDS.find(([, re]) => re.test(text))?.[0] ?? null
 }
 
@@ -108,10 +129,9 @@ function walkBody(ctx, node, state) {
 	}
 }
 
-function buildContext(doc, options) {
+export function buildContext(doc, options) {
 	const lineageMode = options.lineageMode ?? "inline"
 	if (!LINEAGE_MODES.includes(lineageMode)) throw new TypeError(`lineageMode must be one of ${LINEAGE_MODES.join(", ")}`)
-	const closingPattern = options.closingPattern ? toRegExp(options.closingPattern) : null
 	const all = descendants(doc)
 	const styleBlocks = all.filter((node) => node.tag === "style").map((node) => ({ node, css: node.raw ?? "" }))
 	const styleAttrs = all.filter((node) => node.attrs.style).map((node) => ({ node, css: node.attrs.style }))
@@ -130,7 +150,7 @@ function buildContext(doc, options) {
 		requireSections: (options.requireSections ?? []).map(toRegExp),
 		baseDir: options.baseDir ?? null,
 		fs: options.fs ?? nodeFs,
-		isClosing: (text) => CLOSING_HEADING.test(text) || Boolean(closingPattern?.test(text)),
+		isClosing: closingTest(options.closingPattern),
 		util: { classTokens, closest, descendants, hasCitation, citationMarker: CITATION_MARKER },
 		kindOf: (scope) => (scope.closingH3 ? "closing" : (scope.section?.kind ?? null)),
 		defect(code, node, message) {

package/packages/shared-skills/skills/ulw-research/scripts/gates-structure.mjs

@@ -9,7 +9,7 @@ import { join } from "node:path"
 import { proseText } from "./html-lite.mjs"
 
 const MIN_SECTION_PROSE = 120
-const UNCHECKED_KINDS = new Set(["closing", "sources", "toc", "appendix", "glossary"])
+export const UNCHECKED_KINDS = new Set(["closing", "sources", "toc", "appendix", "glossary"])
 const DEFINITION_TAGS = new Set(["p", "li", "dd", "dt", "td"])
 const DEFINITION_LINE = /^\[(S|Source\s*)(\d+)\]/i
 const MIN_DEFINITIONS = 3

package/packages/shared-skills/skills/ulw-research/scripts/gates-text.mjs

@@ -4,6 +4,7 @@
 // Plain Node ESM, sibling imports only.
 
 import { proseText } from "./html-lite.mjs"
+import { CLAIMS, NOT_CLAIMS } from "./numerals.mjs"
 
 const HANGUL = /[\uAC00-\uD7A3]/g
 const KEEP_ALL = /word-break\s*:\s*keep-all/i
@@ -19,33 +20,6 @@ const MAX_HEADING_WORDS = 12
 const LINEAGE_TOKEN = /\b(?:MEASURED|ASSUMED|DERIVED)\b/
 const NUMBER_CONTAINERS = new Set(["figure", "table", "nav", "header", "footer"])
 const MAX_UNSOURCED = 50
-const MONTH = "(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\\.?"
-/** Stripped before claim detection: dates, versions, figure refs, times, bare years. */
-const NOT_CLAIMS = [
-	/\b\d{4}[-./]\d{1,2}(?:[-./]\d{1,2})?\b/g,
-	/\b\d{2}\.\d{2}\.\d{2}\b/g,
-	/\d{2,4}\s*\uB144\s*\d{1,2}\s*\uC6D4(?:\s*\d{1,2}\s*\uC77C)?/g,
-	/\d{1,2}\s*\uC6D4(?:\s*\d{1,2}\s*\uC77C)?/g,
-	/(?:19|20)\d{2}\s*\uB144/g,
-	new RegExp(`\\b${MONTH}\\s+\\d{1,2}(?:st|nd|rd|th)?(?:,?\\s+\\d{4})?`, "gi"),
-	new RegExp(`\\b\\d{1,2}(?:st|nd|rd|th)?\\s+${MONTH}(?:\\s+\\d{4})?`, "gi"),
-	/\bv\d+(?:\.\d+)*\b|\b\d+(?:\.\d+){2,}\b/gi,
-	/\b(?:version|ver\.?|release)\s*\d+(?:\.\d+)*/gi,
-	/\b(?:fig(?:ure)?\.?|table|chart|section|step|phase|wave|part)\s*\d+(?:\.\d+)*/gi,
-	/(?:\uADF8\uB9BC|\uB3C4\uD45C|\uD45C|\uBC84\uC804)\s*\d+(?:\.\d+)*/g,
-	/\b\d{1,2}:\d{2}\b/g,
-	/\b(?:19|20)\d{2}s?\b/g,
-]
-const NUM = "(?<![\\p{L}\\d.,])\\d+(?:[.,]\\d+)*"
-const UNIT =
-	"(?:%|\\u2030|\\u00D7|\\uD37C\\uC13C\\uD2B8|\\uB2EC\\uB7EC|\\uC720\\uB85C|\\uC5D4|\\uC6D0|\\uB9CC|\\uC5B5|\\uC870|\\uCC9C|\\uCD08|\\uBD84|\\uC2DC\\uAC04|\\uC77C|\\uC8FC|\\uAC1C\\uC6D4|\\uB144|\\uBA85|\\uAC74|\\uAC1C|\\uD68C|\\uBC88|\\uBC30" +
-	"|(?:percent|pp|bps?|x|k|m|b|bn|million|billion|thousand|trillion|usd|krw|eur|ms|s|secs?|seconds?|mins?|minutes?|h|hrs?|hours?|days?|weeks?|months?|years?|kb|mb|gb|tb|users?|people|employees|engineers|tokens?|requests?|stars?|downloads?|customers?)(?![\\p{L}\\d]))"
-const CLAIMS = [
-	new RegExp(`[$\\u20AC\\u00A3\\u00A5\\u20A9]\\s?${NUM}`, "u"),
-	new RegExp(`${NUM}\\s?${UNIT}`, "iu"),
-	/(?<![\d.,])\d{1,3}(?:,\d{3})+(?![\d,])/,
-	/(?<![\d.])\d+\.\d+(?![\d.])/,
-]
 
 function snippet(text, index, span = 24) {
 	return text.slice(Math.max(0, index - span), index + span).trim()

package/packages/shared-skills/skills/ulw-research/scripts/html-lite.mjs

@@ -54,8 +54,8 @@ function skipSpaces(source, at) {
 	return at
 }
 
-/** Read `<name attrs...>` starting just after `<`. */
-function readTag(source, start) {
+/** Read `<name attrs...>` starting just after `<`; the one attribute parser (any quoting). */
+export function readTag(source, start) {
 	const end = source.length
 	let at = start
 	while (at < end && !isSpace(source[at]) && source[at] !== ">" && source[at] !== "/") at++

package/packages/shared-skills/skills/ulw-research/scripts/html-markup.mjs

@@ -0,0 +1,182 @@
+// Byte-preserving HTML boundaries for polish. No CSS or visibility inference.
+import { readTag } from "./html-lite.mjs"
+
+const RAW = new Set(["script", "style"])
+const FROZEN = new Set([...RAW, "template"])
+const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"])
+const isLetter = (ch) => /[a-z]/i.test(ch ?? "")
+
+function scan(src, at, visit) {
+	while (at < src.length) {
+		const start = src.indexOf("<", at)
+		if (start === -1) return
+		const next = src[start + 1]
+		let kind
+		let end
+		let tag = null
+		if (src.startsWith("<!--", start)) {
+			// From start + 2 so the abrupt closes <!--> and <!---> end where the HTML tokenizer ends them.
+			kind = "skip"
+			const stop = src.indexOf("-->", start + 2)
+			end = stop === -1 ? src.length : stop + 3
+		} else if (next === "!" || next === "?" || (next === "/" && !isLetter(src[start + 2]))) {
+			// Declarations, <?...> and </ + non-letter (</> included) are bogus comments up to the next ">".
+			kind = "skip"
+			const stop = src.indexOf(">", start + 2)
+			end = stop === -1 ? src.length : stop + 1
+		} else if (isLetter(next) || (next === "/" && isLetter(src[start + 2]))) {
+			kind = next === "/" ? "close" : "open"
+			tag = readTag(src, start + (kind === "close" ? 2 : 1))
+			end = tag.end
+		} else {
+			at = start + 1
+			continue
+		}
+		const resume = visit(kind, start, end, tag)
+		if (resume === Infinity) return
+		at = resume ?? (kind === "open" && RAW.has(tag.name) ? closeOf(src, end, tag.name) : end)
+	}
+}
+
+// Script data with its escaped and double-escaped states: a </script> inside <!--<script>...-->
+// does not end the script. Returns the end of the closing tag, or the end of input.
+const SCRIPT_MARKS = { data: /<!--|<\/script(?=[\s/>])/gi, escaped: /-->|<\/?script(?=[\s/>])/gi, double: /-->|<\/script(?=[\s/>])/gi }
+function scriptEnd(src, from) {
+	let state = "data"
+	let at = from
+	for (;;) {
+		const marks = SCRIPT_MARKS[state]
+		marks.lastIndex = at
+		const match = marks.exec(src)
+		if (!match) return src.length
+		const token = match[0].toLowerCase()
+		if (token === "-->") [state, at] = ["data", match.index + 3]
+		else if (token === "<!--") [state, at] = ["escaped", match.index + 2]
+		else if (token === "<script") [state, at] = ["double", match.index + 7]
+		else if (state === "double") [state, at] = ["escaped", match.index + 8]
+		else return readTag(src, match.index + 2).end
+	}
+}
+
+function closeOf(src, from, name) {
+	if (name === "script") return scriptEnd(src, from)
+	if (RAW.has(name)) {
+		const close = new RegExp(`</${name}(?=[\\s/>])[^>]*>`, "gi")
+		close.lastIndex = from
+		const match = close.exec(src)
+		return match ? match.index + match[0].length : src.length
+	}
+	let depth = 0
+	let stop = src.length
+	scan(src, from, (kind, _start, end, tag) => {
+		if (tag?.name !== name) return undefined
+		if (kind === "open") depth++
+		else if (depth === 0) {
+			stop = end
+			return Infinity
+		} else depth--
+		return undefined
+	})
+	return stop
+}
+
+const endOf = (src, end, tag) => VOID.has(tag.name) ? end : closeOf(src, end, tag.name)
+
+/** Raw markup sequence, corresponding text slots, and markup with frozen elements omitted. */
+export function markupParts(html) {
+	const src = String(html)
+	const markup = []
+	const text = []
+	let source = ""
+	let kept = 0
+	scan(src, 0, (kind, start, end, tag) => {
+		const frozen = kind === "open" && FROZEN.has(tag.name)
+		const stop = frozen ? endOf(src, end, tag) : end
+		text.push(src.slice(kept, start))
+		markup.push(src.slice(start, stop))
+		source += src.slice(kept, start) + (frozen || kind === "skip" ? " " : src.slice(start, stop))
+		kept = stop
+		return stop
+	})
+	text.push(src.slice(kept))
+	source += src.slice(kept)
+	return { markup, text, source }
+}
+
+/** Whole elements accepted by the caller; nested matches stay within the outer match. */
+export function elementSpans(html, accept) {
+	const src = String(html)
+	const spans = []
+	scan(src, 0, (kind, start, end, tag) => {
+		if (kind !== "open" || !accept(tag.name, tag.attrs)) return undefined
+		const stop = endOf(src, end, tag)
+		spans.push({ start, end: stop, tag })
+		return stop
+	})
+	return spans
+}
+
+export function tagAttributes(html) {
+	const tags = []
+	scan(String(html), 0, (kind, _start, _end, tag) => {
+		if (kind === "open") tags.push({ name: tag.name, attrs: tag.attrs })
+		return undefined
+	})
+	return tags
+}
+
+/**
+ * Constructs this tokenizer does not model exactly; renderer output never contains them, so polish
+ * refuses a document holding any of them instead of trusting the markup comparison.
+ */
+export function ambiguousMarkup(html) {
+	const src = String(html)
+	const found = new Set()
+	if (/<\/(?![a-z])/i.test(src)) found.add("</ before a non-letter")
+	if (/<!\[CDATA\[/i.test(src)) found.add("<![CDATA[")
+	if (src.includes("<?")) found.add("<?")
+	if (src.includes("--!>")) found.add("--!>")
+	scan(src, 0, (kind, start, end, tag) => {
+		if (kind !== "skip" && src.slice(start + 1, end).includes("<")) found.add("< inside a tag or attribute value")
+		if (kind === "open" && RAW.has(tag.name) && src.slice(end, closeOf(src, end, tag.name)).includes("<!--")) found.add(`<!-- inside ${tag.name}`)
+		return undefined
+	})
+	return [...found]
+}
+
+/** One character reference token of a text node: & up to the first ; or non-name character. */
+export const CHARACTER_REFERENCE = /&[^;\s<&]*;?/g
+
+/**
+ * The skeleton of a document: every markup token (tags, comments, bogus comments and the frozen
+ * content of script/style/template) and every character reference of a text node, in document
+ * order, each with its byte offset. Polish may change text node characters only, and an edit can
+ * carry neither <, > nor & and may not touch a character reference, so an edited span holds no
+ * skeleton token: a polished document keeps this sequence byte-identical.
+ */
+export function skeletonTokens(html) {
+	const { markup, text } = markupParts(html)
+	const tokens = []
+	let at = 0
+	text.forEach((slot, index) => {
+		for (const match of slot.matchAll(CHARACTER_REFERENCE)) tokens.push({ kind: "reference", value: match[0], offset: at + match.index })
+		at += slot.length
+		if (index < markup.length) {
+			tokens.push({ kind: "markup", value: markup[index], offset: at })
+			at += markup[index].length
+		}
+	})
+	return tokens
+}
+
+const describeToken = (token) => (token ? `${token.kind} ${JSON.stringify(token.value)} at offset ${token.offset}` : "end of document")
+
+/** The first skeleton token two documents disagree on, as a message, or null when they agree. */
+export function skeletonChange(beforeHtml, afterHtml) {
+	const [before, after] = [skeletonTokens(beforeHtml), skeletonTokens(afterHtml)]
+	for (let index = 0; index < Math.max(before.length, after.length); index++) {
+		if (before[index]?.value === after[index]?.value) continue
+		return `skeleton changed at token ${index}: ${describeToken(before[index])} became ${describeToken(after[index])}`
+	}
+	return null
+}

package/packages/shared-skills/skills/ulw-research/scripts/lexicon.mjs

@@ -0,0 +1,47 @@
+// Bilingual fidelity lexicon: data only, read by the fidelity checks and the meaning-pair router.
+// Korean entries are \u escapes (this directory ships no Hangul); each list is frozen.
+
+// Hedge tokens: a source hedge the report drops is hedge_dropped.
+export const HEDGES = Object.freeze([
+	...["estimated", "about", "approximately", "may", "might", "expected", "forecast", "provisional", "not yet"],
+	...["\uCD94\uC815", "\uC57D", "\uAC00\uB2A5", "\uC804\uB9DD", "\uC608\uC0C1", "\uC7A0\uC815", "\uC544\uC9C1", "\uBBF8\uD655\uC815", "\uAC83\uC73C\uB85C \uBCF4\uC778\uB2E4"],
+])
+
+// Closed qualifier list for the ledger qualifier field (bounding or approximation words).
+export const QUALIFIERS = Object.freeze([
+	...["up to", "at least", "at most", "about", "approximately", "nearly", "over", "under", "more than", "less than"],
+	...["\uCD5C\uB300", "\uCD5C\uC18C", "\uC57D", "\uC774\uC0C1", "\uC774\uD558", "\uBBF8\uB9CC", "\uCD08\uACFC", "\uB0B4\uC678", "\uAE4C\uC9C0"],
+])
+
+// Direction classes: a source in one class and an assertion in the other is direction_flipped.
+export const DIRECTION = Object.freeze({
+	up: Object.freeze([...["increase", "rise", "grow", "up"], ...["\uC99D\uAC00", "\uC0C1\uC2B9", "\uB298"]]),
+	down: Object.freeze([...["decrease", "fall", "decline", "down"], ...["\uAC10\uC18C", "\uD558\uB77D", "\uC904"]]),
+})
+
+// Causal connectives: one in the assertion but in no supporting quote is causality_added.
+export const CAUSAL = Object.freeze([
+	...["because", "due to", "leads to", "results in", "causes", "therefore"],
+	...["\uB54C\uBB38\uC5D0", "\uB85C \uC778\uD574", "\uB530\uB77C\uC11C", "\uADF8\uB798\uC11C", "\uCD08\uB798"],
+])
+
+// Concessive connectives for the hedge-position routing heuristic.
+export const CONCESSIVE = Object.freeze([
+	...["although", "but", "however", "despite", "while"],
+	...["\uC73C\uB098", "\uC9C0\uB9CC", "\uC5D0\uB3C4 \uBD88\uAD6C\uD558\uACE0", "\uBC18\uBA74", "\uC640 \uB2EC\uB9AC"],
+])
+
+// Narrow-to-broad scope pairs [narrow, broad] for scope_widened_warn.
+export const SCOPE_NARROW_TO_BROAD = Object.freeze([
+	Object.freeze(["\uBD81\uBBF8", "\uC804 \uC138\uACC4"]),
+	Object.freeze(["pilot", "system"]),
+	Object.freeze(["one region", "all regions"]),
+	Object.freeze(["\uC77C\uBD80", "\uC804\uCCB4"]),
+	Object.freeze(["\uD55C \uC9C0\uC5ED", "\uC804\uAD6D"]),
+])
+
+// Negation tokens; negated direction cases route to a reader instead of the lexicon.
+export const NEGATION = Object.freeze([
+	...["not", "no", "never"],
+	...["\uC54A", "\uC5C6", "\uBABB"],
+])

package/packages/shared-skills/skills/ulw-research/scripts/lexicon.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, test } from "bun:test"
+
+import { CAUSAL, CONCESSIVE, DIRECTION, HEDGES, NEGATION, QUALIFIERS, SCOPE_NARROW_TO_BROAD } from "./lexicon.mjs"
+
+const LISTS = { HEDGES, QUALIFIERS, CAUSAL, CONCESSIVE, NEGATION, SCOPE_NARROW_TO_BROAD } as const
+
+describe("lexicon", () => {
+	test("given each exported list when read then it is non-empty and frozen", () => {
+		for (const [name, list] of Object.entries(LISTS)) {
+			expect({ name, length: list.length > 0 }).toEqual({ name, length: true })
+			expect({ name, frozen: Object.isFrozen(list) }).toEqual({ name, frozen: true })
+		}
+		expect(Object.isFrozen(DIRECTION)).toBe(true)
+		expect(Object.isFrozen(DIRECTION.up)).toBe(true)
+		expect(Object.isFrozen(DIRECTION.down)).toBe(true)
+		for (const pair of SCOPE_NARROW_TO_BROAD) expect(Object.isFrozen(pair)).toBe(true)
+	})
+
+	test("given HEDGES when read then it carries the named English and Korean hedges", () => {
+		const english = ["estimated", "about", "approximately", "may", "might", "expected", "forecast", "provisional", "not yet"]
+		const korean = ["추정", "약", "가능", "전망", "예상", "잠정", "아직", "미확정", "것으로 보인다"]
+		expect(HEDGES).toEqual(expect.arrayContaining([...english, ...korean]))
+	})
+
+	test("given QUALIFIERS when read then it is exactly the closed bilingual list", () => {
+		const korean = ["최대", "최소", "약", "이상", "이하", "미만", "초과", "내외", "까지"]
+		const english = ["up to", "at least", "at most", "about", "approximately", "nearly", "over", "under", "more than", "less than"]
+		expect([...QUALIFIERS].sort()).toEqual([...korean, ...english].sort())
+	})
+
+	test("given DIRECTION when read then up and down carry both languages", () => {
+		expect(DIRECTION.up).toEqual(expect.arrayContaining(["증가", "상승", "늘", "increase", "rise", "grow", "up"]))
+		expect(DIRECTION.down).toEqual(expect.arrayContaining(["감소", "하락", "줄", "decrease", "fall", "decline", "down"]))
+	})
+
+	test("given CAUSAL, CONCESSIVE and NEGATION when read then they carry the named connectives", () => {
+		expect(CAUSAL).toEqual(
+			expect.arrayContaining(["because", "due to", "leads to", "results in", "causes", "therefore", "때문에", "로 인해", "따라서", "그래서", "초래"]),
+		)
+		expect(CONCESSIVE).toEqual(
+			expect.arrayContaining(["although", "but", "however", "despite", "while", "으나", "지만", "에도 불구하고", "반면", "와 달리"]),
+		)
+		expect(NEGATION).toEqual(expect.arrayContaining(["not", "no", "never", "않", "없", "못"]))
+	})
+
+	test("given SCOPE_NARROW_TO_BROAD when read then it carries the named narrow-to-broad pairs", () => {
+		expect(SCOPE_NARROW_TO_BROAD).toEqual(
+			expect.arrayContaining([
+				["북미", "전 세계"],
+				["pilot", "system"],
+				["one region", "all regions"],
+				["일부", "전체"],
+				["한 지역", "전국"],
+			]),
+		)
+	})
+})