lazycodex-ai 5.1.45 → 5.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +0 -5
- package/README.es.md +0 -5
- package/README.fr.md +0 -5
- package/README.it.md +0 -5
- package/README.ja.md +0 -5
- package/README.ko.md +0 -5
- package/README.md +4 -5
- package/README.pt-br.md +0 -5
- package/README.ru.md +0 -6
- package/README.zh-cn.md +0 -5
- package/dist/cli/index.js +76 -156
- package/dist/cli-node/index.js +76 -156
- package/package.json +1 -1
- package/packages/lsp-daemon/dist/cli.js +3 -3
- package/packages/lsp-daemon/dist/client.js +3 -3
- package/packages/lsp-daemon/dist/index.js +3 -3
- package/packages/lsp-tools-mcp/dist/cli.js +3 -3
- package/packages/lsp-tools-mcp/dist/mcp.js +3 -3
- package/packages/omo-codex/plugin/.codex-plugin/plugin.json +1 -1
- package/packages/omo-codex/plugin/components/bootstrap/dist/cli.js +22 -12
- package/packages/omo-codex/plugin/components/bootstrap/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/bootstrap/package.json +1 -1
- package/packages/omo-codex/plugin/components/comment-checker/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/comment-checker/package.json +1 -1
- package/packages/omo-codex/plugin/components/git-bash/hooks/hooks.json +2 -2
- package/packages/omo-codex/plugin/components/git-bash/package.json +1 -1
- package/packages/omo-codex/plugin/components/lazycodex-executor-verify/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/lazycodex-executor-verify/package.json +1 -1
- package/packages/omo-codex/plugin/components/lsp/dist/.omo-runtime-manifest.json +2 -2
- package/packages/omo-codex/plugin/components/lsp/hooks/hooks.json +2 -2
- package/packages/omo-codex/plugin/components/lsp/package.json +1 -1
- package/packages/omo-codex/plugin/components/rules/hooks/hooks.json +4 -4
- package/packages/omo-codex/plugin/components/rules/package.json +1 -1
- package/packages/omo-codex/plugin/components/teammode/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/teammode/package.json +1 -1
- package/packages/omo-codex/plugin/components/telemetry/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/telemetry/package.json +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/package.json +1 -1
- package/packages/omo-codex/plugin/components/ulw-execute-continuation/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/ulw-execute-continuation/package.json +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/hooks/hooks.json +5 -5
- package/packages/omo-codex/plugin/components/ulw-loop/package.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-compact-resetting-git-bash-mcp-reminder.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-compact-resetting-lsp-diagnostics-cache.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-compact-resetting-project-rule-cache.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-checking-comments.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-checking-lsp-diagnostics.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-checking-thread-title-hygiene.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-matching-project-rules.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-recording-spawn-admission.json +1 -1
- package/packages/omo-codex/plugin/hooks/pre-tool-use-enforcing-unlimited-goal-budget.json +1 -1
- package/packages/omo-codex/plugin/hooks/pre-tool-use-guarding-ulw-loop-spawns.json +1 -1
- package/packages/omo-codex/plugin/hooks/pre-tool-use-recommending-git-bash-mcp.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-checking-auto-update.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-checking-bootstrap-provisioning.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-loading-project-rules.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-recording-session-telemetry.json +1 -1
- package/packages/omo-codex/plugin/hooks/stop-checking-ulw-execute-continuation.json +1 -1
- package/packages/omo-codex/plugin/hooks/stop-checking-ulw-loop-resume.json +1 -1
- package/packages/omo-codex/plugin/hooks/subagent-stop-verifying-lazycodex-executor-evidence.json +1 -1
- package/packages/omo-codex/plugin/hooks/user-prompt-submit-checking-ultrawork-trigger.json +1 -1
- package/packages/omo-codex/plugin/hooks/user-prompt-submit-checking-ulw-loop-steering.json +1 -1
- package/packages/omo-codex/plugin/hooks/user-prompt-submit-loading-project-rules.json +1 -1
- package/packages/omo-codex/plugin/package-lock.json +12 -12
- package/packages/omo-codex/plugin/package.json +1 -1
- package/packages/omo-codex/plugin/scripts/sync-skills.mjs +8 -3
- package/packages/omo-codex/plugin/shared/test/config-loader.test.ts +0 -2
- package/packages/omo-codex/plugin/skills/ast-grep/AGENTS.md +1 -1
- package/packages/omo-codex/plugin/skills/browser/runtime/omowright/manifest.json +1 -1
- package/packages/omo-codex/plugin/skills/frontend/ATTRIBUTION.md +2 -2
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/AGENTS.md +1 -1
- package/packages/omo-codex/plugin/skills/ulw-research/AGENTS.md +14 -2
- package/packages/omo-codex/plugin/skills/ulw-research/SKILL.md +36 -11
- package/packages/omo-codex/plugin/skills/ulw-research/references/deliverable-phase.md +73 -13
- package/packages/omo-codex/plugin/skills/ulw-research/references/fact-ledger-grammar.md +19 -1
- package/packages/omo-codex/plugin/skills/ulw-research/references/fact-ledger.md +95 -0
- package/packages/omo-codex/plugin/skills/ulw-research/references/fidelity-claim-kinds.md +19 -0
- package/packages/omo-codex/plugin/skills/ulw-research/references/report-gates.md +21 -4
- package/packages/omo-codex/plugin/skills/ulw-research/references/report-visualization.md +2 -0
- package/packages/omo-codex/plugin/skills/ulw-research/references/unit-rules.md +157 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/claim-kinds.mjs +116 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/cli-support.mjs +19 -5
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/contracts.mjs +27 -6
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-commands-meaning.mjs +191 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-commands-verify.mjs +164 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-commands.mjs +239 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-derive.mjs +66 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-expr.mjs +108 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-fill.mjs +194 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-ledger-kinds.mjs +127 -13
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-ledger-schema.mjs +33 -16
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-ledger.mjs +29 -9
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-locate.mjs +231 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-pairs.mjs +187 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fact-recompute.mjs +89 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-binding.mjs +130 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-brief.mjs +5 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-claims.mjs +89 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-lexicon.mjs +145 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-records.mjs +163 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-state.mjs +2 -1
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-trace.mjs +85 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-values.mjs +229 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/gates-fidelity.mjs +113 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/gates-static.mjs +28 -8
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/html-lite.mjs +2 -2
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/html-markup.mjs +182 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/md-lite.mjs +193 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/meaning-check.mjs +196 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/outcome.mjs +6 -5
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/placeholder.mjs +184 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/polish-apply.mjs +115 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/polish-diff.mjs +31 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/polish-project.mjs +125 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/polish-rules.mjs +170 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/report-tools-commands.mjs +19 -8
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/report-tools.mjs +15 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-commands.mjs +140 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-snapshot.mjs +176 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/unit-algebra.mjs +101 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/unit-errors.mjs +23 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/unit-metadata.mjs +31 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/unit-pairs.mjs +72 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/unit-precision.mjs +44 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/unit-rules-shapes.mjs +64 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/unit-rules.mjs +173 -0
- package/packages/omo-codex/plugin/skills/ulw-research/scripts/units.mjs +4 -4
- package/packages/omo-codex/plugin/test/sync-skills-codex-compatibility.test.mjs +19 -0
- package/packages/omo-codex/plugin/test/sync-skills-test-support.mjs +7 -3
- package/packages/omo-codex/scripts/install-dist/install-local.mjs +26 -16
- package/packages/shared-skills/skills/ast-grep/AGENTS.md +1 -1
- package/packages/shared-skills/skills/browser/runtime/omowright/manifest.json +1 -1
- package/packages/shared-skills/skills/frontend/ATTRIBUTION.md +2 -2
- package/packages/shared-skills/skills/ultimate-browsing/engine/AGENTS.md +1 -1
- package/packages/shared-skills/skills/ulw-research/AGENTS.md +14 -2
- package/packages/shared-skills/skills/ulw-research/SKILL.md +37 -12
- package/packages/shared-skills/skills/ulw-research/references/deliverable-phase.md +73 -13
- package/packages/shared-skills/skills/ulw-research/references/fact-ledger-grammar.md +19 -1
- package/packages/shared-skills/skills/ulw-research/references/fact-ledger.md +95 -0
- package/packages/shared-skills/skills/ulw-research/references/fidelity-claim-kinds.md +19 -0
- package/packages/shared-skills/skills/ulw-research/references/report-gates.md +21 -4
- package/packages/shared-skills/skills/ulw-research/references/report-visualization.md +2 -0
- package/packages/shared-skills/skills/ulw-research/references/unit-rules.md +157 -0
- package/packages/shared-skills/skills/ulw-research/scripts/claim-kinds.mjs +116 -0
- package/packages/shared-skills/skills/ulw-research/scripts/claim-kinds.test.ts +193 -0
- package/packages/shared-skills/skills/ulw-research/scripts/cli-support.mjs +19 -5
- package/packages/shared-skills/skills/ulw-research/scripts/cli-support.test.ts +13 -2
- package/packages/shared-skills/skills/ulw-research/scripts/contracts.mjs +27 -6
- package/packages/shared-skills/skills/ulw-research/scripts/contracts.test.ts +3 -0
- package/packages/shared-skills/skills/ulw-research/scripts/copy-returns.test.ts +183 -0
- package/packages/shared-skills/skills/ulw-research/scripts/delivery-barrier.test.ts +245 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-commands-meaning.mjs +191 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-commands-verify.mjs +164 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-commands.mjs +239 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-derive.mjs +66 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-derive.test.ts +375 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-expr.mjs +108 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-fill.mjs +194 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-fill.test.ts +221 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger-grammar.test.ts +9 -2
- package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger-kinds.mjs +127 -13
- package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger-kinds.test.ts +18 -3
- package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger-legacy.test.ts +160 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger-schema.mjs +33 -16
- package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger.mjs +29 -9
- package/packages/shared-skills/skills/ulw-research/scripts/fact-ledger.test.ts +2 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-locate.mjs +231 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-locate.test.ts +118 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-pairs.mjs +187 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-pairs.test.ts +167 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-recompute.mjs +89 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-unit-dimensions.test.ts +409 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fact-unit-rules.test.ts +388 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-binding.mjs +130 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-binding.test.ts +81 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-brief.mjs +5 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-claims.mjs +89 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-lexicon.mjs +145 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-lexicon.test.ts +117 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-records.mjs +163 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-state.mjs +2 -1
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-trace.mjs +85 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-trace.test.ts +58 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-values.mjs +229 -0
- package/packages/shared-skills/skills/ulw-research/scripts/fidelity-values.test.ts +270 -0
- package/packages/shared-skills/skills/ulw-research/scripts/gates-fidelity.mjs +113 -0
- package/packages/shared-skills/skills/ulw-research/scripts/gates-fidelity.test.ts +73 -0
- package/packages/shared-skills/skills/ulw-research/scripts/gates-static.mjs +28 -8
- package/packages/shared-skills/skills/ulw-research/scripts/html-lite.mjs +2 -2
- package/packages/shared-skills/skills/ulw-research/scripts/html-markup.mjs +182 -0
- package/packages/shared-skills/skills/ulw-research/scripts/md-lite.mjs +193 -0
- package/packages/shared-skills/skills/ulw-research/scripts/md-lite.test.ts +271 -0
- package/packages/shared-skills/skills/ulw-research/scripts/meaning-check.mjs +196 -0
- package/packages/shared-skills/skills/ulw-research/scripts/meaning-check.test.ts +228 -0
- package/packages/shared-skills/skills/ulw-research/scripts/outcome.mjs +6 -5
- package/packages/shared-skills/skills/ulw-research/scripts/placeholder.mjs +184 -0
- package/packages/shared-skills/skills/ulw-research/scripts/placeholder.test.ts +77 -0
- package/packages/shared-skills/skills/ulw-research/scripts/polish-apply.mjs +115 -0
- package/packages/shared-skills/skills/ulw-research/scripts/polish-apply.test.ts +363 -0
- package/packages/shared-skills/skills/ulw-research/scripts/polish-diff.mjs +31 -0
- package/packages/shared-skills/skills/ulw-research/scripts/polish-diff.test.ts +168 -0
- package/packages/shared-skills/skills/ulw-research/scripts/polish-markup.test.ts +187 -0
- package/packages/shared-skills/skills/ulw-research/scripts/polish-project.mjs +125 -0
- package/packages/shared-skills/skills/ulw-research/scripts/polish-rules.mjs +170 -0
- package/packages/shared-skills/skills/ulw-research/scripts/report-tools-commands.mjs +19 -8
- package/packages/shared-skills/skills/ulw-research/scripts/report-tools.mjs +15 -0
- package/packages/shared-skills/skills/ulw-research/scripts/report-tools.test.ts +458 -2
- package/packages/shared-skills/skills/ulw-research/scripts/source-commands.mjs +140 -0
- package/packages/shared-skills/skills/ulw-research/scripts/source-snapshot.mjs +176 -0
- package/packages/shared-skills/skills/ulw-research/scripts/source-snapshot.test.ts +113 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/assets/share.csv +3 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/gen-legacy-dev.mjs +89 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/ledger-basic.json +223 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/ledger-legacy-dev.json +859 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/ledger-verify.json +420 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/report-bad.html +19 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/report-good.html +39 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4/acronyms.json +3 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4/meta.json +9 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4/sentences.jsonl +7 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4/text.txt +7 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4.html +19 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S5/acronyms.json +1 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S5/meta.json +9 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S5/sentences.jsonl +1 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S5/text.txt +1 -0
- package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S9.txt +1 -0
- package/packages/shared-skills/skills/ulw-research/scripts/unit-algebra.mjs +101 -0
- package/packages/shared-skills/skills/ulw-research/scripts/unit-errors.mjs +23 -0
- package/packages/shared-skills/skills/ulw-research/scripts/unit-metadata.mjs +31 -0
- package/packages/shared-skills/skills/ulw-research/scripts/unit-pairs.mjs +72 -0
- package/packages/shared-skills/skills/ulw-research/scripts/unit-precision.mjs +44 -0
- package/packages/shared-skills/skills/ulw-research/scripts/unit-rules-shapes.mjs +64 -0
- package/packages/shared-skills/skills/ulw-research/scripts/unit-rules.mjs +173 -0
- package/packages/shared-skills/skills/ulw-research/scripts/units.mjs +4 -4
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
// fact-locate.mjs - deterministic fact locators over a source snapshot.
|
|
2
|
+
//
|
|
3
|
+
// Each locator returns { value, unit, raw, quote, value_span, sentence_id, qualifier, locator } or
|
|
4
|
+
// { error: "fact_not_located" | "fact_ambiguous", candidates }; it never guesses. `quote` is the
|
|
5
|
+
// whole containing sentence, cell, or line of text.txt, `value_span` is [start, end] inside
|
|
6
|
+
// `quote` with quote.slice(...value_span) === raw, and locator.char_start/char_end place `quote`
|
|
7
|
+
// in text.txt. Every call takes `sourceId` (S<n>) so sentence ids match sentences.jsonl.
|
|
8
|
+
// Plain Node ESM, sibling imports only; no network, no OCR.
|
|
9
|
+
|
|
10
|
+
import { proseText } from "./html-lite.mjs"
|
|
11
|
+
import { QUALIFIERS } from "./lexicon.mjs"
|
|
12
|
+
import { parseNumeral } from "./numerals.mjs"
|
|
13
|
+
import { indexSentences, snapshotBlocks } from "./source-snapshot.mjs"
|
|
14
|
+
import { findNormalized, normalizeText, tokenize } from "./text-normalize.mjs"
|
|
15
|
+
import { similarity } from "./text-similarity.mjs"
|
|
16
|
+
import { normalizeUnit } from "./units.mjs"
|
|
17
|
+
|
|
18
|
+
const FIND_WINDOW = 80
|
|
19
|
+
const CONTEXT = 60
|
|
20
|
+
const FUZZY_FLOOR = 0.95
|
|
21
|
+
const PHRASES = QUALIFIERS.map((phrase) => phrase.split(" "))
|
|
22
|
+
const SINGLE_QUALIFIERS = new Set(QUALIFIERS.filter((phrase) => !phrase.includes(" ")))
|
|
23
|
+
|
|
24
|
+
const notLocated = (candidates = []) => ({ error: "fact_not_located", candidates })
|
|
25
|
+
const ambiguous = (candidates) => ({ error: "fact_ambiguous", candidates })
|
|
26
|
+
const label = (text) => normalizeText(text).toLowerCase()
|
|
27
|
+
const clean = (token) => token.replace(/^[^\p{L}\p{N}%]+|[^\p{L}\p{N}%]+$/gu, "").toLowerCase()
|
|
28
|
+
const sentenceAt = (sentences, offset) => sentences.find((row) => row.start <= offset && offset < row.end) ?? null
|
|
29
|
+
|
|
30
|
+
function requireSource(sourceId) {
|
|
31
|
+
if (!/^S[1-9]\d*$/.test(String(sourceId))) throw new TypeError(`sourceId must be S<n>, got ${sourceId}`)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Every normalized occurrence of `needle`, as raw [start, end) offsets. */
|
|
35
|
+
function findAll(text, needle) {
|
|
36
|
+
const hits = []
|
|
37
|
+
for (let from = 0; from < text.length; ) {
|
|
38
|
+
const hit = findNormalized(text.slice(from), needle)
|
|
39
|
+
if (!hit) break
|
|
40
|
+
hits.push({ start: from + hit.start, end: from + hit.end })
|
|
41
|
+
from += hit.start + 1
|
|
42
|
+
}
|
|
43
|
+
return hits
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function locatorFor(text, sourceId, start, end, extra = {}) {
|
|
47
|
+
const before = text.slice(0, start)
|
|
48
|
+
const pageStart = before.lastIndexOf("\f") + 1
|
|
49
|
+
const page = text.includes("\f") ? { page: before.split("\f").length } : {}
|
|
50
|
+
return { artifact: `${sourceId}/text.txt`, ...page, line: before.slice(pageStart).split("\n").length, char_start: start, char_end: end, ...extra }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function candidate(text, sentences, start, end) {
|
|
54
|
+
return { sentence_id: sentenceAt(sentences, start)?.sid ?? null, char_start: start, char_end: end, context: text.slice(Math.max(0, start - CONTEXT), end + CONTEXT) }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** A QUALIFIERS token ending within the 2 tokens before the value, or the 1 token right after it. */
|
|
58
|
+
function qualifierAround(quote, [start, end]) {
|
|
59
|
+
const before = quote.slice(0, start).split(/\s+/).filter(Boolean).map(clean)
|
|
60
|
+
for (let skip = 0; skip < 2; skip++) {
|
|
61
|
+
for (const words of PHRASES) {
|
|
62
|
+
const stop = before.length - skip
|
|
63
|
+
if (stop - words.length >= 0 && before.slice(stop - words.length, stop).join(" ") === words.join(" ")) return words.join(" ")
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
const next = /^\s*(\S+)/.exec(quote.slice(end))
|
|
67
|
+
return next && SINGLE_QUALIFIERS.has(clean(next[1])) ? clean(next[1]) : null
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** A located quantity: `holder` is the quote's { sid, start, end, text }, `numeral` is parsed from holder.text. */
|
|
71
|
+
function quantityResult(text, sourceId, holder, numeral, extra = {}, unitFallback = null) {
|
|
72
|
+
const unit = numeral.unitRaw ? (normalizeUnit(numeral.unitRaw) ?? numeral.unitRaw) : unitFallback
|
|
73
|
+
return {
|
|
74
|
+
value: numeral.value,
|
|
75
|
+
unit,
|
|
76
|
+
raw: numeral.raw,
|
|
77
|
+
quote: holder.text,
|
|
78
|
+
value_span: [numeral.span[0], numeral.span[1]],
|
|
79
|
+
sentence_id: holder.sid,
|
|
80
|
+
qualifier: qualifierAround(holder.text, numeral.span),
|
|
81
|
+
locator: locatorFor(text, sourceId, holder.start, holder.end, extra),
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** --find: one anchor occurrence, then the first numeral starting within 80 chars after it in the same sentence. */
|
|
86
|
+
export function locateFind(text, { find, unitExpected, sourceId }) {
|
|
87
|
+
requireSource(sourceId)
|
|
88
|
+
const sentences = indexSentences(text, sourceId)
|
|
89
|
+
const hits = findAll(text, find)
|
|
90
|
+
if (hits.length === 0) return notLocated()
|
|
91
|
+
if (hits.length > 1) return ambiguous(hits.map((hit) => candidate(text, sentences, hit.start, hit.end)))
|
|
92
|
+
const [hit] = hits
|
|
93
|
+
const sentence = sentenceAt(sentences, hit.start)
|
|
94
|
+
if (!sentence || hit.end > sentence.end) return notLocated()
|
|
95
|
+
const from = hit.end - sentence.start
|
|
96
|
+
const numeral = parseNumeral(sentence.text, from)
|
|
97
|
+
if (!numeral || numeral.span[0] - from > FIND_WINDOW) return notLocated([candidate(text, sentences, hit.start, hit.end)])
|
|
98
|
+
const result = quantityResult(text, sourceId, sentence, numeral)
|
|
99
|
+
if (unitExpected === undefined) return result
|
|
100
|
+
const expected = normalizeUnit(unitExpected) ?? unitExpected
|
|
101
|
+
return result.unit === expected ? result : { error: "unit_expected_mismatch", expected, found: result.unit, candidates: [result] }
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function cellsOf(row) {
|
|
105
|
+
const cells = []
|
|
106
|
+
for (const cell of row.children.filter((child) => child.tag === "td" || child.tag === "th")) {
|
|
107
|
+
const span = Math.max(1, Number.parseInt(cell.attrs.colspan ?? "1", 10) || 1)
|
|
108
|
+
for (let k = 0; k < span; k++) cells.push(cell)
|
|
109
|
+
}
|
|
110
|
+
return cells
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function rowsOf(table) {
|
|
114
|
+
const rows = []
|
|
115
|
+
const walk = (node) => {
|
|
116
|
+
for (const child of node.children) {
|
|
117
|
+
if (child.tag === "tr") rows.push(child)
|
|
118
|
+
else if (child.tag !== "table" && child.tag !== "#text") walk(child)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
walk(table)
|
|
122
|
+
return rows
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function tablesOf(root) {
|
|
126
|
+
const tables = []
|
|
127
|
+
const walk = (node) => {
|
|
128
|
+
for (const child of node.children) {
|
|
129
|
+
if (child.tag === "table") tables.push(child)
|
|
130
|
+
if (child.tag !== "#text") walk(child)
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
walk(root)
|
|
134
|
+
return tables.map((table) => {
|
|
135
|
+
const rows = rowsOf(table)
|
|
136
|
+
const header = rows.find((row) => row.children.some((child) => child.tag === "th")) ?? rows[0]
|
|
137
|
+
const caption = table.children.find((child) => child.tag === "caption")
|
|
138
|
+
return { table, header, rows: rows.filter((row) => row !== header), caption: caption ? label(proseText(caption)) : null }
|
|
139
|
+
})
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const cellLabel = (cell) => label(proseText(cell))
|
|
143
|
+
const indexesOf = (labels, wanted) => labels.flatMap((value, index) => (value === wanted ? [index] : []))
|
|
144
|
+
|
|
145
|
+
/** --table/--row/--col: the table by id, caption, or a header label; row and column by normalized label, never by index. */
|
|
146
|
+
export function locateTable(sourceHtml, { table, row, col, sourceId }) {
|
|
147
|
+
requireSource(sourceId)
|
|
148
|
+
const { root, text, blocks } = snapshotBlocks(sourceHtml)
|
|
149
|
+
const key = label(table)
|
|
150
|
+
const tables = tablesOf(root).filter((entry) => entry.table.attrs.id === table || entry.caption === key || (entry.header && cellsOf(entry.header).map(cellLabel).includes(key)))
|
|
151
|
+
if (tables.length !== 1) return tables.length === 0 ? notLocated() : ambiguous(tables.map((entry) => ({ table: entry.table.attrs.id ?? entry.caption })))
|
|
152
|
+
const [{ header, rows }] = tables
|
|
153
|
+
const headerCells = header ? cellsOf(header) : []
|
|
154
|
+
const columns = indexesOf(headerCells.map(cellLabel), label(col))
|
|
155
|
+
const matches = rows.filter((candidateRow) => cellsOf(candidateRow).length > 0 && cellLabel(cellsOf(candidateRow)[0]) === label(row))
|
|
156
|
+
if (columns.length > 1 || matches.length > 1) return ambiguous([...columns.map((index) => ({ col: index })), ...matches.map((match) => ({ row_line: match.line }))])
|
|
157
|
+
const cell = columns.length === 1 && matches.length === 1 ? cellsOf(matches[0])[columns[0]] : null
|
|
158
|
+
const block = cell ? blocks.find((entry) => entry.node === cell) : null
|
|
159
|
+
if (!block) return notLocated()
|
|
160
|
+
const holder = { sid: sentenceAt(indexSentences(text, sourceId), block.start)?.sid ?? null, start: block.start, end: block.end, text: text.slice(block.start, block.end) }
|
|
161
|
+
const numeral = parseNumeral(holder.text)
|
|
162
|
+
if (!numeral) return notLocated()
|
|
163
|
+
const headerUnit = /\(([^()]+)\)\s*$/.exec(cellLabel(headerCells[columns[0]]))
|
|
164
|
+
return quantityResult(text, sourceId, holder, numeral, { table, row_label: row, col_label: col }, headerUnit ? normalizeUnit(headerUnit[1]) : null)
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** --page/--line: exactly one quantity on that line of that page (pages split on a form feed). */
|
|
168
|
+
export function locatePageLine(text, { page, line, sourceId }) {
|
|
169
|
+
requireSource(sourceId)
|
|
170
|
+
const pages = text.split("\f")
|
|
171
|
+
if (!Number.isInteger(page) || page < 1 || page > pages.length) return notLocated()
|
|
172
|
+
const lines = pages[page - 1].split("\n")
|
|
173
|
+
if (!Number.isInteger(line) || line < 1 || line > lines.length) return notLocated()
|
|
174
|
+
let start = pages.slice(0, page - 1).reduce((sum, body) => sum + body.length + 1, 0)
|
|
175
|
+
for (const body of lines.slice(0, line - 1)) start += body.length + 1
|
|
176
|
+
const holder = { start, end: start + lines[line - 1].length, text: lines[line - 1] }
|
|
177
|
+
const numerals = []
|
|
178
|
+
for (let numeral = parseNumeral(holder.text); numeral; numeral = parseNumeral(holder.text, numeral.span[1])) numerals.push(numeral)
|
|
179
|
+
const sentences = indexSentences(text, sourceId)
|
|
180
|
+
if (numerals.length !== 1) return numerals.length === 0 ? notLocated() : ambiguous(numerals.map((numeral) => candidate(text, sentences, start + numeral.span[0], start + numeral.span[1])))
|
|
181
|
+
holder.sid = sentenceAt(sentences, start + numerals[0].span[0])?.sid ?? null
|
|
182
|
+
return quantityResult(text, sourceId, holder, numerals[0])
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Token-aligned windows of `sentence` whose similarity to `match` reaches the fuzzy floor; best first. */
|
|
186
|
+
function fuzzyWindow(sentence, match) {
|
|
187
|
+
const target = normalizeText(match).length
|
|
188
|
+
const wanted = new Set(tokenize(match))
|
|
189
|
+
const present = new Set(tokenize(sentence.text))
|
|
190
|
+
if (wanted.size === 0 || [...wanted].filter((token) => present.has(token)).length / wanted.size < 0.5) return null
|
|
191
|
+
const bounds = [...sentence.text.matchAll(/[\p{L}\p{N}]+/gu)].map((m) => [m.index, m.index + m[0].length])
|
|
192
|
+
let best = null
|
|
193
|
+
for (let i = 0; i < bounds.length; i++) {
|
|
194
|
+
for (let j = i; j < bounds.length; j++) {
|
|
195
|
+
const window = sentence.text.slice(bounds[i][0], bounds[j][1])
|
|
196
|
+
if (window.length > target * 1.06 + 2) break
|
|
197
|
+
if (window.length < target * 0.94 - 2) continue
|
|
198
|
+
const score = similarity(window, match)
|
|
199
|
+
if (score >= FUZZY_FLOOR && (!best || score > best.score)) best = { score, start: sentence.start + bounds[i][0], end: sentence.start + bounds[j][1] }
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return best
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function quoteResult(text, sourceId, sentences, start, end, fuzzy) {
|
|
206
|
+
const first = sentenceAt(sentences, start)
|
|
207
|
+
const last = sentenceAt(sentences, end - 1) ?? first
|
|
208
|
+
return {
|
|
209
|
+
value: null,
|
|
210
|
+
unit: null,
|
|
211
|
+
raw: text.slice(start, end),
|
|
212
|
+
quote: text.slice(first.start, last.end),
|
|
213
|
+
value_span: null,
|
|
214
|
+
sentence_id: first.sid,
|
|
215
|
+
qualifier: null,
|
|
216
|
+
fuzzy,
|
|
217
|
+
locator: locatorFor(text, sourceId, first.start, last.end),
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** --match: a normalized exact match (fuzzy false), else the one window with similarity >= 0.95 (fuzzy true). */
|
|
222
|
+
export function locateMatch(text, { match, sourceId }) {
|
|
223
|
+
requireSource(sourceId)
|
|
224
|
+
const sentences = indexSentences(text, sourceId)
|
|
225
|
+
const hits = findAll(text, match)
|
|
226
|
+
if (hits.length > 1) return ambiguous(hits.map((hit) => candidate(text, sentences, hit.start, hit.end)))
|
|
227
|
+
if (hits.length === 1) return quoteResult(text, sourceId, sentences, hits[0].start, hits[0].end, false)
|
|
228
|
+
const windows = sentences.map((sentence) => fuzzyWindow(sentence, match)).filter(Boolean)
|
|
229
|
+
if (windows.length > 1) return ambiguous(windows.map((window) => candidate(text, sentences, window.start, window.end)))
|
|
230
|
+
return windows.length === 1 ? quoteResult(text, sourceId, sentences, windows[0].start, windows[0].end, true) : notLocated()
|
|
231
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// fact-pairs.mjs - (source, report sentence) pairs for the meaning verifier.
|
|
2
|
+
//
|
|
3
|
+
// extractPairs walks the prose sentences of a filled HTML report and emits, in document order:
|
|
4
|
+
// a quote pair per distinct ledger quote cited by a findings sentence (facts sharing one quote
|
|
5
|
+
// share one pair), a claim pair per sentence bound only by data-claim (the statement comes from
|
|
6
|
+
// the claim index), and a translation pair per data-translates span. force_strong routes a pair
|
|
7
|
+
// to the strong reader when the lexicon raised name_outside_quote or scope_widened_warn, when the
|
|
8
|
+
// hedge-position heuristic fired, or when the source hedges and the sentence's last six tokens do
|
|
9
|
+
// not. No model, no IO: the readers run as separate lanes. Plain Node ESM; no Hangul here.
|
|
10
|
+
|
|
11
|
+
import { isExemptKind, proseSentences } from "./fidelity-binding.mjs"
|
|
12
|
+
import { isKoreanTerm, lexicalPattern, lexiconChecks } from "./fidelity-lexicon.mjs"
|
|
13
|
+
import { sentenceKey } from "./fidelity-state.mjs"
|
|
14
|
+
import { buildContext } from "./gates-static.mjs"
|
|
15
|
+
import { parseHtml, proseText } from "./html-lite.mjs"
|
|
16
|
+
import { HEDGES } from "./lexicon.mjs"
|
|
17
|
+
import { normalizeText } from "./text-normalize.mjs"
|
|
18
|
+
|
|
19
|
+
/** Every pair kind the producers below emit; consumers count and schema pairs by this list. */
|
|
20
|
+
export const PAIR_KINDS = Object.freeze(["quote", "claim", "translation"])
|
|
21
|
+
const TAIL = 6
|
|
22
|
+
|
|
23
|
+
const lang = (text) => (isKoreanTerm(text) ? "ko" : "en")
|
|
24
|
+
/** Sentence text with its fact spans cut out, so a Korean value does not make an English sentence Korean. */
|
|
25
|
+
const outsideFacts = (sentence) => sentence.facts.reduceRight((text, fact) => text.slice(0, fact.start - sentence.start) + text.slice(fact.end - sentence.start), sentence.text)
|
|
26
|
+
|
|
27
|
+
/** Latin hedges match whole words ("exact"); Korean hedges match as substrings, one syllable as a token prefix ("stem"). */
|
|
28
|
+
const HEDGE_RES = HEDGES.map((term) => lexicalPattern(term, isKoreanTerm(term) ? "stem" : "exact"))
|
|
29
|
+
|
|
30
|
+
/** Hedge terms that are also month names; capitalized and followed by a date number, they are months. */
|
|
31
|
+
const MONTH_WORDS = new Set(["may"])
|
|
32
|
+
|
|
33
|
+
/** A capitalized Latin match after the first word is a name or a month ("in May"), not a hedge. */
|
|
34
|
+
const isHedgeAt = (text, match) => {
|
|
35
|
+
if (!/^\p{Lu}/u.test(match[0]) || isKoreanTerm(match[0])) return true
|
|
36
|
+
if (match.index !== 0) return false
|
|
37
|
+
// At the start of a text a capital is only sentence case, so the word still hedges - unless it is a
|
|
38
|
+
// month name followed by a date number ("May 2024 sales rose.").
|
|
39
|
+
return !(MONTH_WORDS.has(match[0].toLowerCase()) && /^\s+\p{Nd}/u.test(text.slice(match.index + match[0].length)))
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export const hasHedge = (text) => {
|
|
43
|
+
const normal = normalizeText(text)
|
|
44
|
+
return HEDGE_RES.some((re) => [...normal.matchAll(re)].some((match) => isHedgeAt(normal, match)))
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The source hedges and none of the sentence's last six tokens carries a hedge. */
|
|
48
|
+
function tailDropsHedge(source, sentence) {
|
|
49
|
+
if (!hasHedge(source)) return false
|
|
50
|
+
return !hasHedge(normalizeText(sentence).split(/\s+/u).filter(Boolean).slice(-TAIL).join(" "))
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function slugOf(scope) {
|
|
54
|
+
const heading = scope.section?.heading
|
|
55
|
+
if (!heading) return ""
|
|
56
|
+
return (heading.attrs.id ?? proseText(heading))
|
|
57
|
+
.toLowerCase()
|
|
58
|
+
.replace(/[^\p{L}\p{N}]+/gu, "-")
|
|
59
|
+
.replace(/^-+|-+$/gu, "")
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function claimMap(claims) {
|
|
63
|
+
const list = Array.isArray(claims) ? claims : (claims?.claims ?? [])
|
|
64
|
+
return new Map(list.map((claim) => [claim.claim_id, claim]))
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const idsOf = (attr) => String(attr ?? "").split(/[\s,]+/u).filter(Boolean)
|
|
68
|
+
|
|
69
|
+
/** Claim ids that bind a sentence: on the block or an ancestor, or on an inner element whose text the sentence holds. */
|
|
70
|
+
function claimIdsFor(ctx, block, sentence) {
|
|
71
|
+
const { closest, descendants } = ctx.util
|
|
72
|
+
const up = block.owner.attrs["data-claim"] !== undefined ? block.owner : closest(block.owner, (node) => node.attrs?.["data-claim"] !== undefined)
|
|
73
|
+
const ids = up ? idsOf(up.attrs["data-claim"]) : []
|
|
74
|
+
const text = normalizeText(sentence.text)
|
|
75
|
+
for (const node of descendants(block.owner)) {
|
|
76
|
+
if (node.attrs?.["data-claim"] === undefined) continue
|
|
77
|
+
const inner = normalizeText(proseText(node))
|
|
78
|
+
if (inner !== "" && text.includes(inner)) ids.push(...idsOf(node.attrs["data-claim"]))
|
|
79
|
+
}
|
|
80
|
+
return [...new Set(ids)]
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function exempt(ctx, node) {
|
|
84
|
+
return isExemptKind(node.attrs["data-kind"]) || Boolean(ctx.util.closest(node, (cur) => isExemptKind(cur.attrs?.["data-kind"])))
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Lexicon routing for every fact span of one quote group in one sentence. */
|
|
88
|
+
function lexiconRouted(group, sentence) {
|
|
89
|
+
const probes = group.facts.map((fact, index) => ({
|
|
90
|
+
pair_id: String(index),
|
|
91
|
+
quote: group.quote,
|
|
92
|
+
sentence: sentence.text,
|
|
93
|
+
span: [fact.start - sentence.start, fact.end - sentence.start],
|
|
94
|
+
qualifier: group.row.kind === "quote" ? null : (group.rows.get(fact.id)?.qualifier ?? null),
|
|
95
|
+
}))
|
|
96
|
+
return lexiconChecks(probes).forceStrong.length > 0
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function quotePairs(sentence, rows, filled, slug, reportLang) {
|
|
100
|
+
const groups = new Map()
|
|
101
|
+
for (const fact of sentence.facts) {
|
|
102
|
+
const row = rows.get(fact.id)
|
|
103
|
+
if (!row?.quote || (filled && !filled.has(fact.id))) continue
|
|
104
|
+
const group = groups.get(row.quote) ?? { quote: row.quote, row, rows, facts: [], ids: [] }
|
|
105
|
+
group.facts.push(fact)
|
|
106
|
+
if (!group.ids.includes(fact.id)) group.ids.push(fact.id)
|
|
107
|
+
groups.set(row.quote, group)
|
|
108
|
+
}
|
|
109
|
+
const allIds = [...new Set(sentence.facts.map((fact) => fact.id))]
|
|
110
|
+
return [...groups.values()].map((group) => ({
|
|
111
|
+
kind: "quote",
|
|
112
|
+
sentence_key: sentenceKey(slug, allIds),
|
|
113
|
+
fact_ids: [...group.ids].sort(),
|
|
114
|
+
quote: group.quote,
|
|
115
|
+
sentence: sentence.text,
|
|
116
|
+
source_lang: lang(group.quote),
|
|
117
|
+
report_lang: reportLang ?? lang(outsideFacts(sentence)),
|
|
118
|
+
force_strong: lexiconRouted(group, sentence) || tailDropsHedge(group.quote, sentence.text),
|
|
119
|
+
}))
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function claimPairs(ctx, block, sentence, byClaim, slug, reportLang) {
|
|
123
|
+
if (sentence.facts.length > 0 || exempt(ctx, block.owner)) return []
|
|
124
|
+
return claimIdsFor(ctx, block, sentence)
|
|
125
|
+
.map((id) => byClaim.get(id))
|
|
126
|
+
.filter(Boolean)
|
|
127
|
+
.map((claim) => ({
|
|
128
|
+
kind: "claim",
|
|
129
|
+
sentence_key: sentenceKey(slug, [claim.claim_id]),
|
|
130
|
+
claim_id: claim.claim_id,
|
|
131
|
+
statement: claim.statement,
|
|
132
|
+
sentence: sentence.text,
|
|
133
|
+
source_lang: lang(claim.statement),
|
|
134
|
+
report_lang: reportLang ?? lang(sentence.text),
|
|
135
|
+
force_strong: lexiconChecks([{ pair_id: "c", quote: claim.statement, sentence: sentence.text }]).forceStrong.length > 0 || tailDropsHedge(claim.statement, sentence.text),
|
|
136
|
+
}))
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function translationPair(node, rows, filled, slug, reportLang) {
|
|
140
|
+
const id = node.attrs["data-translates"]
|
|
141
|
+
const row = rows.get(id)
|
|
142
|
+
const text = proseText(node)
|
|
143
|
+
if (!row?.quote || text === "" || (filled && !filled.has(id))) return null
|
|
144
|
+
return {
|
|
145
|
+
kind: "translation",
|
|
146
|
+
sentence_key: sentenceKey(slug, [id]),
|
|
147
|
+
fact_ids: [id],
|
|
148
|
+
quote: row.quote,
|
|
149
|
+
sentence: text,
|
|
150
|
+
source_lang: lang(row.quote),
|
|
151
|
+
report_lang: reportLang ?? lang(text),
|
|
152
|
+
force_strong: tailDropsHedge(row.quote, text),
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* @param {string} html the filled report
|
|
158
|
+
* @param {{ rows: object[] }} ledger
|
|
159
|
+
* @param {{ claims: object[] } | object[] | null} claims indexClaims(...) output or its claims array
|
|
160
|
+
* @param {{ fact_id: string }[]} [fillMap] fact fill's map; when given, only facts it filled are paired
|
|
161
|
+
* @returns {object[]} pairs with pair_id P1, P2, ... in document order
|
|
162
|
+
*/
|
|
163
|
+
export function extractPairs(html, ledger, claims, fillMap) {
|
|
164
|
+
const ctx = buildContext(parseHtml(html), {})
|
|
165
|
+
const rows = new Map((ledger?.rows ?? []).map((row) => [row.fact_id, row]))
|
|
166
|
+
const filled = Array.isArray(fillMap) ? new Set(fillMap.map((entry) => entry.fact_id)) : null
|
|
167
|
+
const byClaim = claimMap(claims)
|
|
168
|
+
const htmlLang = ctx.all.find((node) => node.tag === "html")?.attrs.lang
|
|
169
|
+
const reportLang = htmlLang ? htmlLang.toLowerCase().split("-")[0] : null
|
|
170
|
+
const out = []
|
|
171
|
+
const seenSpans = new Set()
|
|
172
|
+
for (const block of proseSentences(ctx)) {
|
|
173
|
+
const findings = Boolean(block.scope.section) && block.kind === null
|
|
174
|
+
if (!findings) continue
|
|
175
|
+
const slug = slugOf(block.scope)
|
|
176
|
+
for (const sentence of block.sentences) {
|
|
177
|
+
out.push(...quotePairs(sentence, rows, filled, slug, reportLang), ...claimPairs(ctx, block, sentence, byClaim, slug, reportLang))
|
|
178
|
+
}
|
|
179
|
+
for (const node of ctx.util.descendants(block.owner)) {
|
|
180
|
+
if (node.attrs?.["data-translates"] === undefined || seenSpans.has(node)) continue
|
|
181
|
+
seenSpans.add(node)
|
|
182
|
+
const pair = translationPair(node, rows, filled, slug, reportLang)
|
|
183
|
+
if (pair) out.push(pair)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return out.map((pair, index) => ({ pair_id: `P${index + 1}`, ...pair }))
|
|
187
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// Iterative postorder verifier: one snapshot and one arithmetic evaluation per reachable node.
|
|
2
|
+
import { compute } from "./fact-derive.mjs"
|
|
3
|
+
import { parseExpr } from "./fact-expr.mjs"
|
|
4
|
+
import { isLegacyDerived, kindRuleErrors, readFlaws, readRow } from "./fact-ledger-kinds.mjs"
|
|
5
|
+
import { rowErrors } from "./fact-ledger-schema.mjs"
|
|
6
|
+
import { refuse } from "./unit-algebra.mjs"
|
|
7
|
+
import { refusal } from "./unit-errors.mjs"
|
|
8
|
+
import { object, sameDeclared } from "./unit-metadata.mjs"
|
|
9
|
+
import { expectedUnit, malformed, ruleOf, unitMismatch } from "./unit-rules.mjs"
|
|
10
|
+
|
|
11
|
+
const derived = (row) => row.kind === "derived" || row.kind === "converted"
|
|
12
|
+
const fail = (detail, shape, expected) => ({ ok: false, detail, shape, expected })
|
|
13
|
+
function refuseLegacy(row) {
|
|
14
|
+
if (isLegacyDerived(row)) refuse(`${row.fact_id} was written before precision, compatibility, and context existed; regenerate it with derive`, undefined, "legacy derived row", "a derived row regenerated by derive")
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function verifyRow(row, rows) {
|
|
18
|
+
const rule = ruleOf(row.operation)
|
|
19
|
+
const ast = rule.expression ? parseExpr(row.formula) : null
|
|
20
|
+
const context = { ...row.context, rationale: row.rationale }
|
|
21
|
+
const unit = expectedUnit(row.operation, rows, { ast, context })
|
|
22
|
+
const expected = { unit, ...compute(row.operation, rows, unit, ast, context, row.formula), derived_from: rows.map((r) => r.fact_id) }
|
|
23
|
+
if (row.unit !== unit) return fail(`${row.fact_id}: ${unitMismatch(row.operation, rows, row.unit, unit)}`, "result_unit", expected)
|
|
24
|
+
for (const field of Object.keys(expected)) {
|
|
25
|
+
if (!sameDeclared(row[field], expected[field])) return fail(`${row.fact_id}: stored ${field} differs from the recomputed result`, `${field}_mismatch`, expected)
|
|
26
|
+
}
|
|
27
|
+
return { ok: true, expected }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function verifyGraph(ledger, roots, requireActive = false, override = null) {
|
|
31
|
+
try {
|
|
32
|
+
if (!object(ledger) || !Array.isArray(ledger.rows)) refuse("ledger rows must be an array")
|
|
33
|
+
const byId = new Map()
|
|
34
|
+
for (const r of ledger.rows) {
|
|
35
|
+
if (!object(r) || typeof r.fact_id !== "string" || byId.has(r.fact_id)) refuse("ledger rows need distinct fact ids", undefined, "identity", "distinct fact ids")
|
|
36
|
+
byId.set(r.fact_id, r)
|
|
37
|
+
}
|
|
38
|
+
if (override) byId.set(override.fact_id, override)
|
|
39
|
+
const states = new Map()
|
|
40
|
+
const snapshots = new Map()
|
|
41
|
+
const results = new Map()
|
|
42
|
+
const stack = roots.map((id) => ({ id, finish: false }))
|
|
43
|
+
while (stack.length) {
|
|
44
|
+
const { id, finish } = stack.pop()
|
|
45
|
+
if (states.get(id) === "done") continue
|
|
46
|
+
if (!finish) {
|
|
47
|
+
if (states.get(id) === "open") refuse(`${id}: operands form a cycle`, undefined, "cycle", "an acyclic dependency graph")
|
|
48
|
+
const original = byId.get(id)
|
|
49
|
+
if (!original) refuse(`no ledger row ${id}`, undefined, "missing_operand", "an existing fact id")
|
|
50
|
+
const row = { ...readRow(original) }
|
|
51
|
+
snapshots.set(id, row)
|
|
52
|
+
if (requireActive && row.status !== "active") refuse(`${id} is ${row.status}`, undefined, "inactive_dependency", "active dependencies at every depth")
|
|
53
|
+
const [flaw] = readFlaws(row)
|
|
54
|
+
if (flaw) refuse(`${id}: ${flaw.detail}`, undefined, flaw.shape, flaw.expected)
|
|
55
|
+
if (derived(row)) {
|
|
56
|
+
refuseLegacy(row)
|
|
57
|
+
const shape = malformed(row)
|
|
58
|
+
if (shape) refuse(`${id}: ${shape}`, undefined, "malformed_row", "the operation's registry row shape")
|
|
59
|
+
}
|
|
60
|
+
const errors = derived(row) ? kindRuleErrors(row, id) : rowErrors(row)
|
|
61
|
+
if (errors.length) refuse(errors.join("; "), undefined, "row_schema", "a canonical row matching its kind and operation")
|
|
62
|
+
if (row.exact === null || row.value === null) refuse(`${id} has no exact value to derive from`)
|
|
63
|
+
states.set(id, "open")
|
|
64
|
+
stack.push({ id, finish: true })
|
|
65
|
+
if (derived(row)) for (const dependency of row.operands) stack.push({ id: dependency, finish: false })
|
|
66
|
+
} else {
|
|
67
|
+
const row = snapshots.get(id)
|
|
68
|
+
if (derived(row)) {
|
|
69
|
+
const result = verifyRow(row, row.operands.map((ref) => snapshots.get(ref)))
|
|
70
|
+
if (!result.ok) return result
|
|
71
|
+
const errors = rowErrors(row)
|
|
72
|
+
if (errors.length) refuse(errors.join("; "), undefined, "row_schema", "a complete canonical row")
|
|
73
|
+
results.set(id, result)
|
|
74
|
+
}
|
|
75
|
+
states.set(id, "done")
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return results.get(roots[0]) ?? { ok: true }
|
|
79
|
+
} catch (error) { return { ok: false, ...refusal(error) } }
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function recompute(ledger, row) {
|
|
83
|
+
try {
|
|
84
|
+
refuseLegacy(row)
|
|
85
|
+
const shape = malformed(row)
|
|
86
|
+
if (shape) refuse(shape, undefined, "malformed_row", "a valid derived or converted row")
|
|
87
|
+
return verifyGraph(ledger, [row.fact_id], false, row)
|
|
88
|
+
} catch (error) { return { ok: false, ...refusal(error) } }
|
|
89
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// fidelity-binding.mjs - sentence binding and claim-status checks of the fidelity gate.
|
|
2
|
+
//
|
|
3
|
+
// proseSentences walks every leaf prose element (p, li) outside figures, tables and page chrome,
|
|
4
|
+
// keeps the offsets of each data-fact element inside its text, and splits it into sentences, so
|
|
5
|
+
// every check sees which ledger rows a sentence cites. bindingChecks raises unbound_sentence (a
|
|
6
|
+
// findings sentence with no fact, placeholder, citation, or data-claim), claim_status_invalid (a
|
|
7
|
+
// data-claim id that is unknown, refuted or unresolved outside a Gaps or Contradictions section,
|
|
8
|
+
// or high-risk and missing from verified-claims), and conflicting_readings (two active rows with
|
|
9
|
+
// one value and unit but different periods or qualifiers cited in one sentence). Plain Node ESM.
|
|
10
|
+
|
|
11
|
+
import { proseText } from "./html-lite.mjs"
|
|
12
|
+
import { fromDecimal, eq } from "./rational.mjs"
|
|
13
|
+
import { splitSentences } from "./sentence-split.mjs"
|
|
14
|
+
import { normalizeUnit } from "./units.mjs"
|
|
15
|
+
|
|
16
|
+
const SKIP = new Set(["code", "pre", "kbd", "samp", "script", "style", "svg", "template", "noscript"])
|
|
17
|
+
const CONTAINERS = new Set(["figure", "table", "nav", "header", "footer"])
|
|
18
|
+
/** data-kind values whose sentences need no binding (opinion and recommendation blocks). */
|
|
19
|
+
export const EXEMPT_KINDS = Object.freeze(["opinion", "recommendation"])
|
|
20
|
+
export const isExemptKind = (kind) => EXEMPT_KINDS.includes(kind)
|
|
21
|
+
const DECIMAL = /^-?\d+(?:\.\d+)?$/
|
|
22
|
+
/** Gaps and Contradictions headings (English, and the Korean gap/limit/contradiction/conflict words). */
|
|
23
|
+
const GAPS_HEADING = /\b(?:gaps?|contradictions?|unresolved)\b|\uACF5\uBC31|\uD55C\uACC4|\uBAA8\uC21C|\uC0C1\uCDA9/i
|
|
24
|
+
|
|
25
|
+
/** Canonical unit token, or the raw unit without spaces, lowercased; "" for none. */
|
|
26
|
+
export function unitKey(unit) {
|
|
27
|
+
if (unit === null || unit === undefined) return ""
|
|
28
|
+
return normalizeUnit(String(unit)) ?? String(unit).replace(/\s+/g, "").toLowerCase()
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Exact decimal equality; false when either side is not a plain decimal. */
|
|
32
|
+
export function sameValue(a, b) {
|
|
33
|
+
const [x, y] = [String(a ?? "").replace(/,/g, ""), String(b ?? "").replace(/,/g, "")]
|
|
34
|
+
return DECIMAL.test(x) && DECIMAL.test(y) && eq(fromDecimal(x), fromDecimal(y))
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function serialize(node, out) {
|
|
38
|
+
for (const child of node.children) {
|
|
39
|
+
if (child.tag === "#text") out.text += child.text
|
|
40
|
+
else if (child.tag === "br") out.text += " "
|
|
41
|
+
else if (SKIP.has(child.tag)) continue
|
|
42
|
+
else if (child.attrs["data-fact"] !== undefined) {
|
|
43
|
+
const text = proseText(child)
|
|
44
|
+
out.facts.push({ id: child.attrs["data-fact"], start: out.text.length, end: out.text.length + text.length, text, node: child })
|
|
45
|
+
out.text += text
|
|
46
|
+
} else serialize(child, out)
|
|
47
|
+
}
|
|
48
|
+
return out
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Leaf prose elements with their sentences and the facts each sentence cites.
|
|
53
|
+
* @returns {{ owner: object, scope: object, kind: string|null, sentences: { text: string, start: number, end: number, facts: object[] }[] }[]}
|
|
54
|
+
*/
|
|
55
|
+
export function proseSentences(ctx) {
|
|
56
|
+
const { closest, descendants } = ctx.util
|
|
57
|
+
const blocks = []
|
|
58
|
+
for (const { node, scope } of ctx.elements) {
|
|
59
|
+
if (node.tag !== "p" && node.tag !== "li") continue
|
|
60
|
+
if (node.tag === "li" && descendants(node).some((child) => child.tag === "p" || child.tag === "li")) continue
|
|
61
|
+
if (closest(node, (cur) => CONTAINERS.has(cur.tag))) continue
|
|
62
|
+
const { text, facts } = serialize(node, { text: "", facts: [] })
|
|
63
|
+
const sentences = splitSentences(text).map((s) => ({ ...s, facts: facts.filter((f) => f.start < s.end && f.end > s.start) }))
|
|
64
|
+
blocks.push({ owner: node, scope, kind: ctx.kindOf(scope), sentences })
|
|
65
|
+
}
|
|
66
|
+
return blocks
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function claimList(claims) {
|
|
70
|
+
const list = Array.isArray(claims) ? claims : (claims?.claims ?? [])
|
|
71
|
+
return new Map(list.map((claim) => [claim.claim_id, claim]))
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const selfOrUp = (ctx, node, test) => test(node) || Boolean(ctx.util.closest(node, test))
|
|
75
|
+
|
|
76
|
+
function claimDefects(ctx, claims) {
|
|
77
|
+
const byId = claimList(claims)
|
|
78
|
+
const out = []
|
|
79
|
+
for (const { node, scope } of ctx.elements) {
|
|
80
|
+
const attr = node.attrs["data-claim"]
|
|
81
|
+
if (attr === undefined) continue
|
|
82
|
+
const heading = scope.section?.heading
|
|
83
|
+
const inGaps = Boolean(heading) && GAPS_HEADING.test(proseText(heading))
|
|
84
|
+
for (const id of attr.split(/[\s,]+/).filter(Boolean)) {
|
|
85
|
+
const claim = byId.get(id)
|
|
86
|
+
let why = null
|
|
87
|
+
if (!claim) why = `data-claim ${id} is not in the claim index`
|
|
88
|
+
else if (/refuted|unresolved/i.test(claim.status) && !inGaps) why = `claim ${id} is ${claim.status} but asserted outside Gaps or Contradictions`
|
|
89
|
+
else if (!inGaps && /high/i.test(claim.risk) && !claim.in_verified) why = `high-risk claim ${id} is not in verified-claims`
|
|
90
|
+
if (why) out.push(ctx.defect("claim_status_invalid", node, why))
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return out
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function conflict(rows, facts) {
|
|
97
|
+
const cited = [...new Set(facts.map((f) => f.id))].map((id) => rows.get(id)).filter((row) => row && row.status === "active" && row.kind !== "quote")
|
|
98
|
+
for (let i = 0; i < cited.length; i++) {
|
|
99
|
+
for (let k = i + 1; k < cited.length; k++) {
|
|
100
|
+
const [a, b] = [cited[i], cited[k]]
|
|
101
|
+
if (!sameValue(a.value, b.value) || unitKey(a.unit) !== unitKey(b.unit)) continue
|
|
102
|
+
if (a.valid_at !== b.valid_at || a.qualifier !== b.qualifier) return [a, b]
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return null
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* @param {object} ctx buildContext(...) from gates-static.mjs
|
|
110
|
+
* @param {{ claims: object[] } | object[] | null} claims indexClaims(...) output or its claims array
|
|
111
|
+
* @param {{ rows: object[] } | null} ledger
|
|
112
|
+
*/
|
|
113
|
+
export function bindingChecks(ctx, claims, ledger) {
|
|
114
|
+
const rows = new Map((ledger?.rows ?? []).map((row) => [row.fact_id, row]))
|
|
115
|
+
const out = []
|
|
116
|
+
for (const block of proseSentences(ctx)) {
|
|
117
|
+
const findings = Boolean(block.scope.section) && block.kind === null
|
|
118
|
+
const exempt = selfOrUp(ctx, block.owner, (cur) => isExemptKind(cur.attrs["data-kind"]) || cur.attrs["data-claim"] !== undefined)
|
|
119
|
+
for (const sentence of block.sentences) {
|
|
120
|
+
const bound = sentence.facts.length > 0 || ctx.util.citationMarker.test(sentence.text) || sentence.text.includes("{{")
|
|
121
|
+
if (findings && !exempt && !bound) out.push(ctx.defect("unbound_sentence", block.owner, `no fact, citation, or data-claim binds: "${sentence.text.slice(0, 80)}"`))
|
|
122
|
+
const pair = conflict(rows, sentence.facts)
|
|
123
|
+
if (pair) {
|
|
124
|
+
const [a, b] = pair
|
|
125
|
+
out.push(ctx.defect("conflicting_readings", block.owner, `${a.fact_id} (${a.valid_at ?? "no period"}) and ${b.fact_id} (${b.valid_at ?? "no period"}) share ${a.value} ${a.unit ?? ""} in one sentence`))
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return [...out, ...claimDefects(ctx, claims)]
|
|
130
|
+
}
|
|
@@ -109,6 +109,11 @@ export function isLocked(parsed) {
|
|
|
109
109
|
return FIDELITY_FIELDS.every((field) => Object.hasOwn(parsed.fields, field) && LOCKED_ANSWERED_BY.has(parsed.answered_by[field]))
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
+
/** The brief's `meaning check` field turns the meaning verifier off only when it reads "off". */
|
|
113
|
+
export function meaningCheckEnabled(parsed) {
|
|
114
|
+
return !/^off\b/iu.test(parsed?.fields?.["meaning check"] ?? "")
|
|
115
|
+
}
|
|
116
|
+
|
|
112
117
|
function incompleteDefect(briefPath, parsed) {
|
|
113
118
|
const reasons = []
|
|
114
119
|
if (!parsed.present) reasons.push("no ## Fidelity block")
|