lazycodex-ai 5.1.44 → 5.1.46

package/packages/shared-skills/skills/ulw-research/scripts/fact-locate.mjs

@@ -0,0 +1,231 @@
+// fact-locate.mjs - deterministic fact locators over a source snapshot.
+//
+// Each locator returns { value, unit, raw, quote, value_span, sentence_id, qualifier, locator } or
+// { error: "fact_not_located" | "fact_ambiguous", candidates }; it never guesses. `quote` is the
+// whole containing sentence, cell, or line of text.txt, `value_span` is [start, end] inside
+// `quote` with quote.slice(...value_span) === raw, and locator.char_start/char_end place `quote`
+// in text.txt. Every call takes `sourceId` (S<n>) so sentence ids match sentences.jsonl.
+// Plain Node ESM, sibling imports only; no network, no OCR.
+
+import { proseText } from "./html-lite.mjs"
+import { QUALIFIERS } from "./lexicon.mjs"
+import { parseNumeral } from "./numerals.mjs"
+import { indexSentences, snapshotBlocks } from "./source-snapshot.mjs"
+import { findNormalized, normalizeText, tokenize } from "./text-normalize.mjs"
+import { similarity } from "./text-similarity.mjs"
+import { normalizeUnit } from "./units.mjs"
+
+const FIND_WINDOW = 80
+const CONTEXT = 60
+const FUZZY_FLOOR = 0.95
+const PHRASES = QUALIFIERS.map((phrase) => phrase.split(" "))
+const SINGLE_QUALIFIERS = new Set(QUALIFIERS.filter((phrase) => !phrase.includes(" ")))
+
+const notLocated = (candidates = []) => ({ error: "fact_not_located", candidates })
+const ambiguous = (candidates) => ({ error: "fact_ambiguous", candidates })
+const label = (text) => normalizeText(text).toLowerCase()
+const clean = (token) => token.replace(/^[^\p{L}\p{N}%]+|[^\p{L}\p{N}%]+$/gu, "").toLowerCase()
+const sentenceAt = (sentences, offset) => sentences.find((row) => row.start <= offset && offset < row.end) ?? null
+
+function requireSource(sourceId) {
+	if (!/^S[1-9]\d*$/.test(String(sourceId))) throw new TypeError(`sourceId must be S<n>, got ${sourceId}`)
+}
+
+/** Every normalized occurrence of `needle`, as raw [start, end) offsets. */
+function findAll(text, needle) {
+	const hits = []
+	for (let from = 0; from < text.length; ) {
+		const hit = findNormalized(text.slice(from), needle)
+		if (!hit) break
+		hits.push({ start: from + hit.start, end: from + hit.end })
+		from += hit.start + 1
+	}
+	return hits
+}
+
+function locatorFor(text, sourceId, start, end, extra = {}) {
+	const before = text.slice(0, start)
+	const pageStart = before.lastIndexOf("\f") + 1
+	const page = text.includes("\f") ? { page: before.split("\f").length } : {}
+	return { artifact: `${sourceId}/text.txt`, ...page, line: before.slice(pageStart).split("\n").length, char_start: start, char_end: end, ...extra }
+}
+
+function candidate(text, sentences, start, end) {
+	return { sentence_id: sentenceAt(sentences, start)?.sid ?? null, char_start: start, char_end: end, context: text.slice(Math.max(0, start - CONTEXT), end + CONTEXT) }
+}
+
+/** A QUALIFIERS token ending within the 2 tokens before the value, or the 1 token right after it. */
+function qualifierAround(quote, [start, end]) {
+	const before = quote.slice(0, start).split(/\s+/).filter(Boolean).map(clean)
+	for (let skip = 0; skip < 2; skip++) {
+		for (const words of PHRASES) {
+			const stop = before.length - skip
+			if (stop - words.length >= 0 && before.slice(stop - words.length, stop).join(" ") === words.join(" ")) return words.join(" ")
+		}
+	}
+	const next = /^\s*(\S+)/.exec(quote.slice(end))
+	return next && SINGLE_QUALIFIERS.has(clean(next[1])) ? clean(next[1]) : null
+}
+
+/** A located quantity: `holder` is the quote's { sid, start, end, text }, `numeral` is parsed from holder.text. */
+function quantityResult(text, sourceId, holder, numeral, extra = {}, unitFallback = null) {
+	const unit = numeral.unitRaw ? (normalizeUnit(numeral.unitRaw) ?? numeral.unitRaw) : unitFallback
+	return {
+		value: numeral.value,
+		unit,
+		raw: numeral.raw,
+		quote: holder.text,
+		value_span: [numeral.span[0], numeral.span[1]],
+		sentence_id: holder.sid,
+		qualifier: qualifierAround(holder.text, numeral.span),
+		locator: locatorFor(text, sourceId, holder.start, holder.end, extra),
+	}
+}
+
+/** --find: one anchor occurrence, then the first numeral starting within 80 chars after it in the same sentence. */
+export function locateFind(text, { find, unitExpected, sourceId }) {
+	requireSource(sourceId)
+	const sentences = indexSentences(text, sourceId)
+	const hits = findAll(text, find)
+	if (hits.length === 0) return notLocated()
+	if (hits.length > 1) return ambiguous(hits.map((hit) => candidate(text, sentences, hit.start, hit.end)))
+	const [hit] = hits
+	const sentence = sentenceAt(sentences, hit.start)
+	if (!sentence || hit.end > sentence.end) return notLocated()
+	const from = hit.end - sentence.start
+	const numeral = parseNumeral(sentence.text, from)
+	if (!numeral || numeral.span[0] - from > FIND_WINDOW) return notLocated([candidate(text, sentences, hit.start, hit.end)])
+	const result = quantityResult(text, sourceId, sentence, numeral)
+	if (unitExpected === undefined) return result
+	const expected = normalizeUnit(unitExpected) ?? unitExpected
+	return result.unit === expected ? result : { error: "unit_expected_mismatch", expected, found: result.unit, candidates: [result] }
+}
+
+function cellsOf(row) {
+	const cells = []
+	for (const cell of row.children.filter((child) => child.tag === "td" || child.tag === "th")) {
+		const span = Math.max(1, Number.parseInt(cell.attrs.colspan ?? "1", 10) || 1)
+		for (let k = 0; k < span; k++) cells.push(cell)
+	}
+	return cells
+}
+
+function rowsOf(table) {
+	const rows = []
+	const walk = (node) => {
+		for (const child of node.children) {
+			if (child.tag === "tr") rows.push(child)
+			else if (child.tag !== "table" && child.tag !== "#text") walk(child)
+		}
+	}
+	walk(table)
+	return rows
+}
+
+function tablesOf(root) {
+	const tables = []
+	const walk = (node) => {
+		for (const child of node.children) {
+			if (child.tag === "table") tables.push(child)
+			if (child.tag !== "#text") walk(child)
+		}
+	}
+	walk(root)
+	return tables.map((table) => {
+		const rows = rowsOf(table)
+		const header = rows.find((row) => row.children.some((child) => child.tag === "th")) ?? rows[0]
+		const caption = table.children.find((child) => child.tag === "caption")
+		return { table, header, rows: rows.filter((row) => row !== header), caption: caption ? label(proseText(caption)) : null }
+	})
+}
+
+const cellLabel = (cell) => label(proseText(cell))
+const indexesOf = (labels, wanted) => labels.flatMap((value, index) => (value === wanted ? [index] : []))
+
+/** --table/--row/--col: the table by id, caption, or a header label; row and column by normalized label, never by index. */
+export function locateTable(sourceHtml, { table, row, col, sourceId }) {
+	requireSource(sourceId)
+	const { root, text, blocks } = snapshotBlocks(sourceHtml)
+	const key = label(table)
+	const tables = tablesOf(root).filter((entry) => entry.table.attrs.id === table || entry.caption === key || (entry.header && cellsOf(entry.header).map(cellLabel).includes(key)))
+	if (tables.length !== 1) return tables.length === 0 ? notLocated() : ambiguous(tables.map((entry) => ({ table: entry.table.attrs.id ?? entry.caption })))
+	const [{ header, rows }] = tables
+	const headerCells = header ? cellsOf(header) : []
+	const columns = indexesOf(headerCells.map(cellLabel), label(col))
+	const matches = rows.filter((candidateRow) => cellsOf(candidateRow).length > 0 && cellLabel(cellsOf(candidateRow)[0]) === label(row))
+	if (columns.length > 1 || matches.length > 1) return ambiguous([...columns.map((index) => ({ col: index })), ...matches.map((match) => ({ row_line: match.line }))])
+	const cell = columns.length === 1 && matches.length === 1 ? cellsOf(matches[0])[columns[0]] : null
+	const block = cell ? blocks.find((entry) => entry.node === cell) : null
+	if (!block) return notLocated()
+	const holder = { sid: sentenceAt(indexSentences(text, sourceId), block.start)?.sid ?? null, start: block.start, end: block.end, text: text.slice(block.start, block.end) }
+	const numeral = parseNumeral(holder.text)
+	if (!numeral) return notLocated()
+	const headerUnit = /\(([^()]+)\)\s*$/.exec(cellLabel(headerCells[columns[0]]))
+	return quantityResult(text, sourceId, holder, numeral, { table, row_label: row, col_label: col }, headerUnit ? normalizeUnit(headerUnit[1]) : null)
+}
+
+/** --page/--line: exactly one quantity on that line of that page (pages split on a form feed). */
+export function locatePageLine(text, { page, line, sourceId }) {
+	requireSource(sourceId)
+	const pages = text.split("\f")
+	if (!Number.isInteger(page) || page < 1 || page > pages.length) return notLocated()
+	const lines = pages[page - 1].split("\n")
+	if (!Number.isInteger(line) || line < 1 || line > lines.length) return notLocated()
+	let start = pages.slice(0, page - 1).reduce((sum, body) => sum + body.length + 1, 0)
+	for (const body of lines.slice(0, line - 1)) start += body.length + 1
+	const holder = { start, end: start + lines[line - 1].length, text: lines[line - 1] }
+	const numerals = []
+	for (let numeral = parseNumeral(holder.text); numeral; numeral = parseNumeral(holder.text, numeral.span[1])) numerals.push(numeral)
+	const sentences = indexSentences(text, sourceId)
+	if (numerals.length !== 1) return numerals.length === 0 ? notLocated() : ambiguous(numerals.map((numeral) => candidate(text, sentences, start + numeral.span[0], start + numeral.span[1])))
+	holder.sid = sentenceAt(sentences, start + numerals[0].span[0])?.sid ?? null
+	return quantityResult(text, sourceId, holder, numerals[0])
+}
+
+/** Token-aligned windows of `sentence` whose similarity to `match` reaches the fuzzy floor; best first. */
+function fuzzyWindow(sentence, match) {
+	const target = normalizeText(match).length
+	const wanted = new Set(tokenize(match))
+	const present = new Set(tokenize(sentence.text))
+	if (wanted.size === 0 || [...wanted].filter((token) => present.has(token)).length / wanted.size < 0.5) return null
+	const bounds = [...sentence.text.matchAll(/[\p{L}\p{N}]+/gu)].map((m) => [m.index, m.index + m[0].length])
+	let best = null
+	for (let i = 0; i < bounds.length; i++) {
+		for (let j = i; j < bounds.length; j++) {
+			const window = sentence.text.slice(bounds[i][0], bounds[j][1])
+			if (window.length > target * 1.06 + 2) break
+			if (window.length < target * 0.94 - 2) continue
+			const score = similarity(window, match)
+			if (score >= FUZZY_FLOOR && (!best || score > best.score)) best = { score, start: sentence.start + bounds[i][0], end: sentence.start + bounds[j][1] }
+		}
+	}
+	return best
+}
+
+function quoteResult(text, sourceId, sentences, start, end, fuzzy) {
+	const first = sentenceAt(sentences, start)
+	const last = sentenceAt(sentences, end - 1) ?? first
+	return {
+		value: null,
+		unit: null,
+		raw: text.slice(start, end),
+		quote: text.slice(first.start, last.end),
+		value_span: null,
+		sentence_id: first.sid,
+		qualifier: null,
+		fuzzy,
+		locator: locatorFor(text, sourceId, first.start, last.end),
+	}
+}
+
+/** --match: a normalized exact match (fuzzy false), else the one window with similarity >= 0.95 (fuzzy true). */
+export function locateMatch(text, { match, sourceId }) {
+	requireSource(sourceId)
+	const sentences = indexSentences(text, sourceId)
+	const hits = findAll(text, match)
+	if (hits.length > 1) return ambiguous(hits.map((hit) => candidate(text, sentences, hit.start, hit.end)))
+	if (hits.length === 1) return quoteResult(text, sourceId, sentences, hits[0].start, hits[0].end, false)
+	const windows = sentences.map((sentence) => fuzzyWindow(sentence, match)).filter(Boolean)
+	if (windows.length > 1) return ambiguous(windows.map((window) => candidate(text, sentences, window.start, window.end)))
+	return windows.length === 1 ? quoteResult(text, sourceId, sentences, windows[0].start, windows[0].end, true) : notLocated()
+}

package/packages/shared-skills/skills/ulw-research/scripts/fact-locate.test.ts

@@ -0,0 +1,118 @@
+import { describe, expect, test } from "bun:test"
+import { readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { locateFind, locateMatch, locatePageLine, locateTable } from "./fact-locate.mjs"
+import { snapshotFromHtml } from "./source-snapshot.mjs"
+
+const FIXTURES = join(import.meta.dir, "tests", "fixtures", "fidelity", "sources")
+const S4_HTML = readFileSync(join(FIXTURES, "S4.html"), "utf8")
+const S4_TEXT = snapshotFromHtml(S4_HTML)
+const S9_TEXT = readFileSync(join(FIXTURES, "S9.txt"), "utf8")
+const MAX = "\uCD5C\uB300"
+
+type Located = {
+	value: string | null
+	unit: string | null
+	raw: string
+	quote: string
+	value_span: [number, number] | null
+	sentence_id: string
+	qualifier: string | null
+	fuzzy?: boolean
+	locator: { artifact: string; char_start: number; char_end: number; page?: number; line: number; table?: string; row_label?: string; col_label?: string }
+}
+
+function expectAnchored(text: string, result: Located): void {
+	expect(text.slice(result.locator.char_start, result.locator.char_end)).toBe(result.quote)
+	if (result.value_span) expect(result.quote.slice(result.value_span[0], result.value_span[1])).toBe(result.raw)
+}
+
+describe("locateFind", () => {
+	test("#given an anchor followed by a quantity #when located #then the value, unit, sentence and locator bind to the snapshot", () => {
+		const result = locateFind(S4_TEXT, { find: "Span length", sourceId: "S4" })
+		expect(result).toMatchObject({ value: "320", unit: "m", raw: "320 m", quote: "Span length: 320 m", qualifier: null })
+		expect(result.sentence_id).toMatch(/^S4:s\d+$/)
+		expect(result.locator.artifact).toBe("S4/text.txt")
+		expectAnchored(S4_TEXT, result)
+	})
+
+	test("#given an expected unit that differs #when located #then unit_expected_mismatch", () => {
+		expect(locateFind(S4_TEXT, { find: "Span length", unitExpected: "mm", sourceId: "S4" })).toMatchObject({ error: "unit_expected_mismatch" })
+		expect(locateFind(S4_TEXT, { find: "Span length", unitExpected: "meters", sourceId: "S4" })).toMatchObject({ value: "320" })
+	})
+
+	test("#given a Korean anchor before a scaled amount #when located #then the first numeral after the anchor is the value and the anchor is the qualifier", () => {
+		const result = locateFind(S9_TEXT, { find: MAX, sourceId: "S9" })
+		expect(result).toMatchObject({ value: "4200000000000", unit: "KRW", qualifier: MAX, sentence_id: "S9:s1" })
+		expectAnchored(S9_TEXT, result)
+	})
+
+	test("#given an English bounding phrase before the value #when located #then it is the qualifier", () => {
+		const text = "The contract costs up to 40 million USD in total."
+		expect(locateFind(text, { find: "costs", sourceId: "S3" })).toMatchObject({ value: "40000000", unit: "USD", qualifier: "up to" })
+	})
+
+	test("#given an anchor that occurs twice #when located #then fact_ambiguous lists both candidates", () => {
+		const result = locateFind(S4_TEXT, { find: "span cost", sourceId: "S4" })
+		expect(result.error).toBe("fact_ambiguous")
+		expect(result.candidates).toHaveLength(2)
+	})
+
+	test("#given a missing anchor, or no numeral within the anchor's sentence #when located #then fact_not_located", () => {
+		expect(locateFind(S4_TEXT, { find: "nonexistent", sourceId: "S4" })).toMatchObject({ error: "fact_not_located", candidates: [] })
+		expect(locateFind("Total reached.\nIt was 12 m.", { find: "Total", sourceId: "S1" })).toMatchObject({ error: "fact_not_located" })
+		expect(locateFind(`Delta ${"x".repeat(90)} 7 m`, { find: "Delta", sourceId: "S1" })).toMatchObject({ error: "fact_not_located" })
+	})
+})
+
+describe("locateTable", () => {
+	test("#given a table found by a header label #when the row and column labels match #then the cell value binds to its text.txt line", () => {
+		const result = locateTable(S4_HTML, { table: "Length", row: "Span", col: "Length", sourceId: "S4" })
+		expect(result).toMatchObject({ value: "320", unit: "m", quote: "320 m", locator: { table: "Length", row_label: "Span", col_label: "Length" } })
+		expect(result.sentence_id).toMatch(/^S4:s\d+$/)
+		expectAnchored(S4_TEXT, result)
+		expect(locateTable(S4_HTML, { table: "dimensions", row: "Approach", col: "Length", sourceId: "S4" })).toMatchObject({ value: "85" })
+	})
+
+	test("#given an unknown table, row or column, or a duplicated row label #when located #then it fails without guessing", () => {
+		expect(locateTable(S4_HTML, { table: "Width", row: "Span", col: "Length", sourceId: "S4" })).toMatchObject({ error: "fact_not_located" })
+		expect(locateTable(S4_HTML, { table: "Length", row: "Deck", col: "Length", sourceId: "S4" })).toMatchObject({ error: "fact_not_located" })
+		const twice = "<table><tr><th>Item</th><th>Cost</th></tr><tr><td>Span</td><td>3</td></tr><tr><td>Span</td><td>4</td></tr></table>"
+		expect(locateTable(twice, { table: "Cost", row: "Span", col: "Cost", sourceId: "S5" })).toMatchObject({ error: "fact_ambiguous" })
+	})
+})
+
+describe("locatePageLine", () => {
+	const text = "Intro line\nRevenue: 15 %\fCost: 3 m\nBetween 1 m and 2 m"
+
+	test("#given a page and line holding one quantity #when located #then it binds with its page and line", () => {
+		const result = locatePageLine(text, { page: 1, line: 2, sourceId: "S6" })
+		expect(result).toMatchObject({ value: "15", unit: "%", quote: "Revenue: 15 %", locator: { line: 2 } })
+		expectAnchored(text, result)
+		expect(locatePageLine(text, { page: 2, line: 1, sourceId: "S6" })).toMatchObject({ value: "3", locator: { page: 2, line: 1 } })
+	})
+
+	test("#given a line with two quantities or a missing page #when located #then ambiguous or not located", () => {
+		expect(locatePageLine(text, { page: 2, line: 2, sourceId: "S6" })).toMatchObject({ error: "fact_ambiguous" })
+		expect(locatePageLine(text, { page: 3, line: 1, sourceId: "S6" })).toMatchObject({ error: "fact_not_located" })
+	})
+})
+
+describe("locateMatch", () => {
+	test("#given an exact match #when located #then raw is the snapshot bytes, the quote is the whole sentence, and there is no value span", () => {
+		const result = locateMatch(S4_TEXT, { match: "no  earlier than 2027", sourceId: "S4" })
+		expect(result).toMatchObject({ raw: "no earlier than 2027", quote: "After the delay, we estimate completion no earlier than 2027.", value_span: null, value: null, fuzzy: false })
+		expectAnchored(S4_TEXT, result)
+	})
+
+	test("#given a near match at or above 0.95 similarity #when located #then the window returns with fuzzy true; below it, fact_not_located", () => {
+		const near = locateMatch(S4_TEXT, { match: "we estimate completion no earlier then 2027", sourceId: "S4" })
+		expect(near).toMatchObject({ raw: "we estimate completion no earlier than 2027", fuzzy: true })
+		expect(locateMatch(S4_TEXT, { match: "completion is certain by 2026", sourceId: "S4" })).toMatchObject({ error: "fact_not_located" })
+	})
+
+	test("#given a phrase that occurs twice #when located #then fact_ambiguous", () => {
+		expect(locateMatch(S4_TEXT, { match: "span cost", sourceId: "S4" })).toMatchObject({ error: "fact_ambiguous" })
+	})
+})

package/packages/shared-skills/skills/ulw-research/scripts/fact-pairs.mjs

@@ -0,0 +1,187 @@
+// fact-pairs.mjs - (source, report sentence) pairs for the meaning verifier.
+//
+// extractPairs walks the prose sentences of a filled HTML report and emits, in document order:
+// a quote pair per distinct ledger quote cited by a findings sentence (facts sharing one quote
+// share one pair), a claim pair per sentence bound only by data-claim (the statement comes from
+// the claim index), and a translation pair per data-translates span. force_strong routes a pair
+// to the strong reader when the lexicon raised name_outside_quote or scope_widened_warn, when the
+// hedge-position heuristic fired, or when the source hedges and the sentence's last six tokens do
+// not. No model, no IO: the readers run as separate lanes. Plain Node ESM; no Hangul here.
+
+import { isExemptKind, proseSentences } from "./fidelity-binding.mjs"
+import { isKoreanTerm, lexicalPattern, lexiconChecks } from "./fidelity-lexicon.mjs"
+import { sentenceKey } from "./fidelity-state.mjs"
+import { buildContext } from "./gates-static.mjs"
+import { parseHtml, proseText } from "./html-lite.mjs"
+import { HEDGES } from "./lexicon.mjs"
+import { normalizeText } from "./text-normalize.mjs"
+
+/** Every pair kind the producers below emit; consumers count and schema pairs by this list. */
+export const PAIR_KINDS = Object.freeze(["quote", "claim", "translation"])
+const TAIL = 6
+
+const lang = (text) => (isKoreanTerm(text) ? "ko" : "en")
+/** Sentence text with its fact spans cut out, so a Korean value does not make an English sentence Korean. */
+const outsideFacts = (sentence) => sentence.facts.reduceRight((text, fact) => text.slice(0, fact.start - sentence.start) + text.slice(fact.end - sentence.start), sentence.text)
+
+/** Latin hedges match whole words ("exact"); Korean hedges match as substrings, one syllable as a token prefix ("stem"). */
+const HEDGE_RES = HEDGES.map((term) => lexicalPattern(term, isKoreanTerm(term) ? "stem" : "exact"))
+
+/** Hedge terms that are also month names; capitalized and followed by a date number, they are months. */
+const MONTH_WORDS = new Set(["may"])
+
+/** A capitalized Latin match after the first word is a name or a month ("in May"), not a hedge. */
+const isHedgeAt = (text, match) => {
+	if (!/^\p{Lu}/u.test(match[0]) || isKoreanTerm(match[0])) return true
+	if (match.index !== 0) return false
+	// At the start of a text a capital is only sentence case, so the word still hedges - unless it is a
+	// month name followed by a date number ("May 2024 sales rose.").
+	return !(MONTH_WORDS.has(match[0].toLowerCase()) && /^\s+\p{Nd}/u.test(text.slice(match.index + match[0].length)))
+}
+
+export const hasHedge = (text) => {
+	const normal = normalizeText(text)
+	return HEDGE_RES.some((re) => [...normal.matchAll(re)].some((match) => isHedgeAt(normal, match)))
+}
+
+/** The source hedges and none of the sentence's last six tokens carries a hedge. */
+function tailDropsHedge(source, sentence) {
+	if (!hasHedge(source)) return false
+	return !hasHedge(normalizeText(sentence).split(/\s+/u).filter(Boolean).slice(-TAIL).join(" "))
+}
+
+function slugOf(scope) {
+	const heading = scope.section?.heading
+	if (!heading) return ""
+	return (heading.attrs.id ?? proseText(heading))
+		.toLowerCase()
+		.replace(/[^\p{L}\p{N}]+/gu, "-")
+		.replace(/^-+|-+$/gu, "")
+}
+
+function claimMap(claims) {
+	const list = Array.isArray(claims) ? claims : (claims?.claims ?? [])
+	return new Map(list.map((claim) => [claim.claim_id, claim]))
+}
+
+const idsOf = (attr) => String(attr ?? "").split(/[\s,]+/u).filter(Boolean)
+
+/** Claim ids that bind a sentence: on the block or an ancestor, or on an inner element whose text the sentence holds. */
+function claimIdsFor(ctx, block, sentence) {
+	const { closest, descendants } = ctx.util
+	const up = block.owner.attrs["data-claim"] !== undefined ? block.owner : closest(block.owner, (node) => node.attrs?.["data-claim"] !== undefined)
+	const ids = up ? idsOf(up.attrs["data-claim"]) : []
+	const text = normalizeText(sentence.text)
+	for (const node of descendants(block.owner)) {
+		if (node.attrs?.["data-claim"] === undefined) continue
+		const inner = normalizeText(proseText(node))
+		if (inner !== "" && text.includes(inner)) ids.push(...idsOf(node.attrs["data-claim"]))
+	}
+	return [...new Set(ids)]
+}
+
+function exempt(ctx, node) {
+	return isExemptKind(node.attrs["data-kind"]) || Boolean(ctx.util.closest(node, (cur) => isExemptKind(cur.attrs?.["data-kind"])))
+}
+
+/** Lexicon routing for every fact span of one quote group in one sentence. */
+function lexiconRouted(group, sentence) {
+	const probes = group.facts.map((fact, index) => ({
+		pair_id: String(index),
+		quote: group.quote,
+		sentence: sentence.text,
+		span: [fact.start - sentence.start, fact.end - sentence.start],
+		qualifier: group.row.kind === "quote" ? null : (group.rows.get(fact.id)?.qualifier ?? null),
+	}))
+	return lexiconChecks(probes).forceStrong.length > 0
+}
+
+function quotePairs(sentence, rows, filled, slug, reportLang) {
+	const groups = new Map()
+	for (const fact of sentence.facts) {
+		const row = rows.get(fact.id)
+		if (!row?.quote || (filled && !filled.has(fact.id))) continue
+		const group = groups.get(row.quote) ?? { quote: row.quote, row, rows, facts: [], ids: [] }
+		group.facts.push(fact)
+		if (!group.ids.includes(fact.id)) group.ids.push(fact.id)
+		groups.set(row.quote, group)
+	}
+	const allIds = [...new Set(sentence.facts.map((fact) => fact.id))]
+	return [...groups.values()].map((group) => ({
+		kind: "quote",
+		sentence_key: sentenceKey(slug, allIds),
+		fact_ids: [...group.ids].sort(),
+		quote: group.quote,
+		sentence: sentence.text,
+		source_lang: lang(group.quote),
+		report_lang: reportLang ?? lang(outsideFacts(sentence)),
+		force_strong: lexiconRouted(group, sentence) || tailDropsHedge(group.quote, sentence.text),
+	}))
+}
+
+function claimPairs(ctx, block, sentence, byClaim, slug, reportLang) {
+	if (sentence.facts.length > 0 || exempt(ctx, block.owner)) return []
+	return claimIdsFor(ctx, block, sentence)
+		.map((id) => byClaim.get(id))
+		.filter(Boolean)
+		.map((claim) => ({
+			kind: "claim",
+			sentence_key: sentenceKey(slug, [claim.claim_id]),
+			claim_id: claim.claim_id,
+			statement: claim.statement,
+			sentence: sentence.text,
+			source_lang: lang(claim.statement),
+			report_lang: reportLang ?? lang(sentence.text),
+			force_strong: lexiconChecks([{ pair_id: "c", quote: claim.statement, sentence: sentence.text }]).forceStrong.length > 0 || tailDropsHedge(claim.statement, sentence.text),
+		}))
+}
+
+function translationPair(node, rows, filled, slug, reportLang) {
+	const id = node.attrs["data-translates"]
+	const row = rows.get(id)
+	const text = proseText(node)
+	if (!row?.quote || text === "" || (filled && !filled.has(id))) return null
+	return {
+		kind: "translation",
+		sentence_key: sentenceKey(slug, [id]),
+		fact_ids: [id],
+		quote: row.quote,
+		sentence: text,
+		source_lang: lang(row.quote),
+		report_lang: reportLang ?? lang(text),
+		force_strong: tailDropsHedge(row.quote, text),
+	}
+}
+
+/**
+ * @param {string} html the filled report
+ * @param {{ rows: object[] }} ledger
+ * @param {{ claims: object[] } | object[] | null} claims indexClaims(...) output or its claims array
+ * @param {{ fact_id: string }[]} [fillMap] fact fill's map; when given, only facts it filled are paired
+ * @returns {object[]} pairs with pair_id P1, P2, ... in document order
+ */
+export function extractPairs(html, ledger, claims, fillMap) {
+	const ctx = buildContext(parseHtml(html), {})
+	const rows = new Map((ledger?.rows ?? []).map((row) => [row.fact_id, row]))
+	const filled = Array.isArray(fillMap) ? new Set(fillMap.map((entry) => entry.fact_id)) : null
+	const byClaim = claimMap(claims)
+	const htmlLang = ctx.all.find((node) => node.tag === "html")?.attrs.lang
+	const reportLang = htmlLang ? htmlLang.toLowerCase().split("-")[0] : null
+	const out = []
+	const seenSpans = new Set()
+	for (const block of proseSentences(ctx)) {
+		const findings = Boolean(block.scope.section) && block.kind === null
+		if (!findings) continue
+		const slug = slugOf(block.scope)
+		for (const sentence of block.sentences) {
+			out.push(...quotePairs(sentence, rows, filled, slug, reportLang), ...claimPairs(ctx, block, sentence, byClaim, slug, reportLang))
+		}
+		for (const node of ctx.util.descendants(block.owner)) {
+			if (node.attrs?.["data-translates"] === undefined || seenSpans.has(node)) continue
+			seenSpans.add(node)
+			const pair = translationPair(node, rows, filled, slug, reportLang)
+			if (pair) out.push(pair)
+		}
+	}
+	return out.map((pair, index) => ({ pair_id: `P${index + 1}`, ...pair }))
+}