@claritylabs/cl-sdk 1.3.7 → 1.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.d.mts +185 -47
- package/dist/index.d.ts +185 -47
- package/dist/index.js +544 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +542 -55
- package/dist/index.mjs.map +1 -1
- package/dist/storage-sqlite.d.mts +89 -20
- package/dist/storage-sqlite.d.ts +89 -20
- package/dist/storage-sqlite.js +33 -2
- package/dist/storage-sqlite.js.map +1 -1
- package/dist/storage-sqlite.mjs +33 -2
- package/dist/storage-sqlite.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -623,6 +623,8 @@ var CoverageSchema = z4.object({
|
|
|
623
623
|
deductible: z4.string().optional(),
|
|
624
624
|
deductibleAmount: z4.number().optional(),
|
|
625
625
|
deductibleValueType: CoverageValueTypeSchema.optional(),
|
|
626
|
+
trigger: CoverageTriggerSchema.optional(),
|
|
627
|
+
retroactiveDate: z4.string().optional(),
|
|
626
628
|
formNumber: z4.string().optional(),
|
|
627
629
|
pageNumber: z4.number().optional(),
|
|
628
630
|
sectionRef: z4.string().optional(),
|
|
@@ -1851,6 +1853,15 @@ var SourceSpanKindSchema = z20.enum([
|
|
|
1851
1853
|
"plain_text",
|
|
1852
1854
|
"structured_field"
|
|
1853
1855
|
]);
|
|
1856
|
+
var SourceSpanUnitSchema = z20.enum([
|
|
1857
|
+
"page",
|
|
1858
|
+
"section",
|
|
1859
|
+
"table",
|
|
1860
|
+
"table_row",
|
|
1861
|
+
"table_cell",
|
|
1862
|
+
"key_value",
|
|
1863
|
+
"text"
|
|
1864
|
+
]);
|
|
1854
1865
|
var SourceKindSchema = z20.enum([
|
|
1855
1866
|
"policy_pdf",
|
|
1856
1867
|
"application_pdf",
|
|
@@ -1875,6 +1886,15 @@ var SourceSpanLocationSchema = z20.object({
|
|
|
1875
1886
|
lineEnd: z20.number().int().positive().optional(),
|
|
1876
1887
|
fieldPath: z20.string().optional()
|
|
1877
1888
|
});
|
|
1889
|
+
var SourceSpanTableLocationSchema = z20.object({
|
|
1890
|
+
tableId: z20.string().optional(),
|
|
1891
|
+
rowIndex: z20.number().int().nonnegative().optional(),
|
|
1892
|
+
columnIndex: z20.number().int().nonnegative().optional(),
|
|
1893
|
+
columnName: z20.string().optional(),
|
|
1894
|
+
rowSpanId: z20.string().optional(),
|
|
1895
|
+
tableSpanId: z20.string().optional(),
|
|
1896
|
+
isHeader: z20.boolean().optional()
|
|
1897
|
+
});
|
|
1878
1898
|
var SourceSpanSchema = z20.object({
|
|
1879
1899
|
id: z20.string().min(1),
|
|
1880
1900
|
documentId: z20.string().min(1),
|
|
@@ -1888,6 +1908,9 @@ var SourceSpanSchema = z20.object({
|
|
|
1888
1908
|
pageEnd: z20.number().int().positive().optional(),
|
|
1889
1909
|
sectionId: z20.string().optional(),
|
|
1890
1910
|
formNumber: z20.string().optional(),
|
|
1911
|
+
sourceUnit: SourceSpanUnitSchema.optional(),
|
|
1912
|
+
parentSpanId: z20.string().optional(),
|
|
1913
|
+
table: SourceSpanTableLocationSchema.optional(),
|
|
1891
1914
|
bbox: z20.array(SourceSpanBBoxSchema).optional(),
|
|
1892
1915
|
location: SourceSpanLocationSchema.optional(),
|
|
1893
1916
|
metadata: z20.record(z20.string(), z20.string()).optional()
|
|
@@ -2006,6 +2029,9 @@ function buildSourceSpan(input, localIndex = 0) {
|
|
|
2006
2029
|
pageEnd: input.pageEnd,
|
|
2007
2030
|
sectionId: input.sectionId,
|
|
2008
2031
|
formNumber: input.formNumber,
|
|
2032
|
+
sourceUnit: input.sourceUnit,
|
|
2033
|
+
parentSpanId: input.parentSpanId,
|
|
2034
|
+
table: input.table,
|
|
2009
2035
|
location: {
|
|
2010
2036
|
page: input.pageStart === input.pageEnd ? input.pageStart : void 0,
|
|
2011
2037
|
startPage: input.pageStart,
|
|
@@ -2026,7 +2052,11 @@ function buildPageSourceSpans(pages) {
|
|
|
2026
2052
|
pageEnd: page.pageNumber,
|
|
2027
2053
|
sectionId: page.sectionId,
|
|
2028
2054
|
formNumber: page.formNumber,
|
|
2029
|
-
|
|
2055
|
+
sourceUnit: "page",
|
|
2056
|
+
metadata: {
|
|
2057
|
+
...page.metadata ?? {},
|
|
2058
|
+
sourceUnit: page.metadata?.sourceUnit ?? "page"
|
|
2059
|
+
}
|
|
2030
2060
|
},
|
|
2031
2061
|
index
|
|
2032
2062
|
)
|
|
@@ -2048,6 +2078,7 @@ function buildSectionSourceSpans(pages, options = {}) {
|
|
|
2048
2078
|
pageEnd: page.pageNumber,
|
|
2049
2079
|
sectionId: section.title,
|
|
2050
2080
|
formNumber: inferFormNumber(section.text),
|
|
2081
|
+
sourceUnit: "section",
|
|
2051
2082
|
metadata: {
|
|
2052
2083
|
...page.metadata ?? {},
|
|
2053
2084
|
sourceUnit: "section_candidate"
|
|
@@ -2080,6 +2111,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2080
2111
|
const chunks = [];
|
|
2081
2112
|
let current = [];
|
|
2082
2113
|
let currentLength = 0;
|
|
2114
|
+
const spansForChunking = filterChunkableSourceSpans(spans);
|
|
2083
2115
|
const flush = () => {
|
|
2084
2116
|
if (current.length === 0) return;
|
|
2085
2117
|
const text = current.map((span) => span.text).join("\n\n");
|
|
@@ -2103,7 +2135,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2103
2135
|
current = [];
|
|
2104
2136
|
currentLength = 0;
|
|
2105
2137
|
};
|
|
2106
|
-
for (const span of
|
|
2138
|
+
for (const span of spansForChunking) {
|
|
2107
2139
|
const nextLength = currentLength + span.text.length + (current.length > 0 ? 2 : 0);
|
|
2108
2140
|
if (current.length > 0 && nextLength > maxChars) {
|
|
2109
2141
|
flush();
|
|
@@ -2114,6 +2146,20 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2114
2146
|
flush();
|
|
2115
2147
|
return chunks;
|
|
2116
2148
|
}
|
|
2149
|
+
function sourceUnit(span) {
|
|
2150
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
2151
|
+
}
|
|
2152
|
+
function filterChunkableSourceSpans(spans) {
|
|
2153
|
+
const rowIds = new Set(
|
|
2154
|
+
spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
|
|
2155
|
+
);
|
|
2156
|
+
if (rowIds.size === 0) return spans;
|
|
2157
|
+
return spans.filter((span) => {
|
|
2158
|
+
if (sourceUnit(span) !== "table_cell") return true;
|
|
2159
|
+
const rowId = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
2160
|
+
return !rowId || !rowIds.has(rowId);
|
|
2161
|
+
});
|
|
2162
|
+
}
|
|
2117
2163
|
function splitPageIntoSections(text, headingPattern, minSectionChars) {
|
|
2118
2164
|
const lines = text.split(/\r?\n/);
|
|
2119
2165
|
const sections = [];
|
|
@@ -2506,36 +2552,18 @@ function normalizeDoclingDocument(document, options) {
|
|
|
2506
2552
|
pageTexts.set(page, appendText(pageTexts.get(page), unit.text));
|
|
2507
2553
|
}
|
|
2508
2554
|
const fullText = Array.from({ length: pageCount }, (_, index) => {
|
|
2509
|
-
const
|
|
2510
|
-
const text = pageTexts.get(
|
|
2511
|
-
return text ? `Page ${
|
|
2555
|
+
const pageNumber2 = index + 1;
|
|
2556
|
+
const text = pageTexts.get(pageNumber2)?.trim();
|
|
2557
|
+
return text ? `Page ${pageNumber2}
|
|
2512
2558
|
${text}` : "";
|
|
2513
2559
|
}).filter(Boolean).join("\n\n");
|
|
2514
2560
|
const sourceKind = options.sourceKind ?? "policy_pdf";
|
|
2515
|
-
const sourceSpans = units.
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
pageStart: unit.pageStart,
|
|
2522
|
-
pageEnd: unit.pageEnd,
|
|
2523
|
-
sectionId: unit.label,
|
|
2524
|
-
metadata: {
|
|
2525
|
-
sourceSystem: "docling",
|
|
2526
|
-
sourceUnit: "docling_item",
|
|
2527
|
-
doclingRef: unit.ref,
|
|
2528
|
-
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2529
|
-
}
|
|
2530
|
-
},
|
|
2531
|
-
index
|
|
2532
|
-
);
|
|
2533
|
-
return {
|
|
2534
|
-
...span,
|
|
2535
|
-
kind: "plain_text",
|
|
2536
|
-
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2537
|
-
};
|
|
2538
|
-
});
|
|
2561
|
+
const sourceSpans = units.flatMap(
|
|
2562
|
+
(unit, index) => buildSourceSpansForUnit(unit, index, {
|
|
2563
|
+
documentId: options.documentId,
|
|
2564
|
+
sourceKind
|
|
2565
|
+
})
|
|
2566
|
+
);
|
|
2539
2567
|
return {
|
|
2540
2568
|
pageCount,
|
|
2541
2569
|
fullText,
|
|
@@ -2573,6 +2601,11 @@ function mergeSourceSpans(spans) {
|
|
|
2573
2601
|
span.pageStart ?? span.location?.startPage ?? span.location?.page ?? "na",
|
|
2574
2602
|
span.pageEnd ?? span.location?.endPage ?? span.pageStart ?? "na",
|
|
2575
2603
|
span.sectionId ?? span.location?.fieldPath ?? "na",
|
|
2604
|
+
span.sourceUnit ?? span.metadata?.sourceUnit ?? "na",
|
|
2605
|
+
span.parentSpanId ?? "na",
|
|
2606
|
+
span.table?.tableId ?? span.metadata?.tableId ?? "na",
|
|
2607
|
+
span.table?.rowIndex ?? span.metadata?.rowIndex ?? "na",
|
|
2608
|
+
span.table?.columnIndex ?? span.metadata?.columnIndex ?? "na",
|
|
2576
2609
|
span.textHash ?? sourceSpanTextHash(span.text)
|
|
2577
2610
|
].join(":");
|
|
2578
2611
|
if (seen.has(key)) continue;
|
|
@@ -2635,6 +2668,7 @@ function getOrderedBodyRefs(document, itemMap) {
|
|
|
2635
2668
|
function normalizeItem(ref, item) {
|
|
2636
2669
|
const text = getItemText(item).trim();
|
|
2637
2670
|
if (!text) return void 0;
|
|
2671
|
+
const table = getItemTable(item);
|
|
2638
2672
|
const pages = (item.prov ?? []).map((prov) => getPageNumber(prov)).filter((page) => typeof page === "number" && page > 0);
|
|
2639
2673
|
const pageStart = pages.length ? Math.min(...pages) : void 0;
|
|
2640
2674
|
const pageEnd = pages.length ? Math.max(...pages) : pageStart;
|
|
@@ -2645,17 +2679,143 @@ function normalizeItem(ref, item) {
|
|
|
2645
2679
|
text,
|
|
2646
2680
|
pageStart,
|
|
2647
2681
|
pageEnd,
|
|
2648
|
-
bboxes: bboxes.length ? bboxes : void 0
|
|
2682
|
+
bboxes: bboxes.length ? bboxes : void 0,
|
|
2683
|
+
table
|
|
2649
2684
|
};
|
|
2650
2685
|
}
|
|
2686
|
+
function buildSourceSpansForUnit(unit, index, options) {
|
|
2687
|
+
const baseMetadata = {
|
|
2688
|
+
sourceSystem: "docling",
|
|
2689
|
+
sourceUnit: unit.table ? "table" : "docling_item",
|
|
2690
|
+
doclingRef: unit.ref,
|
|
2691
|
+
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2692
|
+
};
|
|
2693
|
+
const tableId = unit.table ? `${unit.ref}:table` : void 0;
|
|
2694
|
+
const tableSpan = withDoclingKind(buildSourceSpan(
|
|
2695
|
+
{
|
|
2696
|
+
documentId: options.documentId,
|
|
2697
|
+
sourceKind: options.sourceKind,
|
|
2698
|
+
text: unit.text,
|
|
2699
|
+
pageStart: unit.pageStart,
|
|
2700
|
+
pageEnd: unit.pageEnd,
|
|
2701
|
+
sectionId: unit.label,
|
|
2702
|
+
sourceUnit: unit.table ? "table" : labelToSourceUnit(unit.label),
|
|
2703
|
+
table: tableId ? { tableId } : void 0,
|
|
2704
|
+
metadata: baseMetadata
|
|
2705
|
+
},
|
|
2706
|
+
index * 1e3
|
|
2707
|
+
), unit);
|
|
2708
|
+
if (!unit.table) return [tableSpan];
|
|
2709
|
+
const spans = [tableSpan];
|
|
2710
|
+
const table = unit.table;
|
|
2711
|
+
for (let rowIndex = 0; rowIndex < table.rows.length; rowIndex += 1) {
|
|
2712
|
+
const row = table.rows[rowIndex];
|
|
2713
|
+
const isHeader = rowIndex === 0 && table.headers.length > 0;
|
|
2714
|
+
const rowText = tableRowText(table, rowIndex);
|
|
2715
|
+
if (!rowText) continue;
|
|
2716
|
+
const rowSpan = withDoclingKind(buildSourceSpan(
|
|
2717
|
+
{
|
|
2718
|
+
documentId: options.documentId,
|
|
2719
|
+
sourceKind: options.sourceKind,
|
|
2720
|
+
text: rowText,
|
|
2721
|
+
pageStart: unit.pageStart,
|
|
2722
|
+
pageEnd: unit.pageEnd,
|
|
2723
|
+
sectionId: unit.label,
|
|
2724
|
+
sourceUnit: "table_row",
|
|
2725
|
+
parentSpanId: tableSpan.id,
|
|
2726
|
+
table: {
|
|
2727
|
+
tableId,
|
|
2728
|
+
tableSpanId: tableSpan.id,
|
|
2729
|
+
rowIndex,
|
|
2730
|
+
isHeader
|
|
2731
|
+
},
|
|
2732
|
+
metadata: {
|
|
2733
|
+
...baseMetadata,
|
|
2734
|
+
sourceUnit: "table_row",
|
|
2735
|
+
tableId: tableId ?? "",
|
|
2736
|
+
tableSpanId: tableSpan.id,
|
|
2737
|
+
rowIndex: String(rowIndex),
|
|
2738
|
+
isHeader: String(isHeader)
|
|
2739
|
+
}
|
|
2740
|
+
},
|
|
2741
|
+
index * 1e3 + 1 + rowIndex
|
|
2742
|
+
), unit);
|
|
2743
|
+
spans.push(rowSpan);
|
|
2744
|
+
for (let columnIndex = 0; columnIndex < row.length; columnIndex += 1) {
|
|
2745
|
+
const text = row[columnIndex]?.trim();
|
|
2746
|
+
if (!text) continue;
|
|
2747
|
+
const columnName = table.headers[columnIndex]?.trim() || void 0;
|
|
2748
|
+
const cellSpan = withDoclingKind(buildSourceSpan(
|
|
2749
|
+
{
|
|
2750
|
+
documentId: options.documentId,
|
|
2751
|
+
sourceKind: options.sourceKind,
|
|
2752
|
+
text,
|
|
2753
|
+
pageStart: unit.pageStart,
|
|
2754
|
+
pageEnd: unit.pageEnd,
|
|
2755
|
+
sectionId: unit.label,
|
|
2756
|
+
sourceUnit: "table_cell",
|
|
2757
|
+
parentSpanId: rowSpan.id,
|
|
2758
|
+
table: {
|
|
2759
|
+
tableId,
|
|
2760
|
+
tableSpanId: tableSpan.id,
|
|
2761
|
+
rowSpanId: rowSpan.id,
|
|
2762
|
+
rowIndex,
|
|
2763
|
+
columnIndex,
|
|
2764
|
+
columnName,
|
|
2765
|
+
isHeader
|
|
2766
|
+
},
|
|
2767
|
+
metadata: {
|
|
2768
|
+
...baseMetadata,
|
|
2769
|
+
sourceUnit: "table_cell",
|
|
2770
|
+
tableId: tableId ?? "",
|
|
2771
|
+
tableSpanId: tableSpan.id,
|
|
2772
|
+
rowSpanId: rowSpan.id,
|
|
2773
|
+
rowIndex: String(rowIndex),
|
|
2774
|
+
columnIndex: String(columnIndex),
|
|
2775
|
+
...columnName ? { columnName } : {},
|
|
2776
|
+
isHeader: String(isHeader)
|
|
2777
|
+
}
|
|
2778
|
+
},
|
|
2779
|
+
index * 1e3 + 100 + rowIndex * 100 + columnIndex
|
|
2780
|
+
), unit);
|
|
2781
|
+
spans.push(cellSpan);
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
return spans;
|
|
2785
|
+
}
|
|
2786
|
+
function withDoclingKind(span, unit) {
|
|
2787
|
+
return {
|
|
2788
|
+
...span,
|
|
2789
|
+
kind: "plain_text",
|
|
2790
|
+
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2791
|
+
};
|
|
2792
|
+
}
|
|
2793
|
+
function labelToSourceUnit(label) {
|
|
2794
|
+
const normalized = label?.toLowerCase() ?? "";
|
|
2795
|
+
if (normalized.includes("table")) return "table";
|
|
2796
|
+
if (normalized.includes("key")) return "key_value";
|
|
2797
|
+
if (normalized.includes("section") || normalized.includes("header")) return "section";
|
|
2798
|
+
return "text";
|
|
2799
|
+
}
|
|
2800
|
+
function tableRowText(table, rowIndex) {
|
|
2801
|
+
const row = table.rows[rowIndex] ?? [];
|
|
2802
|
+
const headers = table.headers;
|
|
2803
|
+
if (rowIndex === 0 && headers.length > 0) return row.filter(Boolean).join(" | ");
|
|
2804
|
+
return row.map((value, columnIndex) => {
|
|
2805
|
+
const trimmed = value.trim();
|
|
2806
|
+
if (!trimmed) return "";
|
|
2807
|
+
const header = headers[columnIndex]?.trim();
|
|
2808
|
+
return header ? `${header}: ${trimmed}` : trimmed;
|
|
2809
|
+
}).filter(Boolean).join(" | ");
|
|
2810
|
+
}
|
|
2651
2811
|
function getItemText(item) {
|
|
2652
2812
|
if (typeof item.text === "string" && item.text.trim()) return item.text;
|
|
2653
2813
|
if (typeof item.orig === "string" && item.orig.trim()) return item.orig;
|
|
2654
|
-
const table =
|
|
2655
|
-
if (table) return table;
|
|
2814
|
+
const table = parseTableData(item.data);
|
|
2815
|
+
if (table) return table.markdown;
|
|
2656
2816
|
return "";
|
|
2657
2817
|
}
|
|
2658
|
-
function
|
|
2818
|
+
function parseTableData(data) {
|
|
2659
2819
|
const record = asRecord(data);
|
|
2660
2820
|
const cells = Array.isArray(record?.table_cells) ? record.table_cells : Array.isArray(record?.tableCells) ? record.tableCells : void 0;
|
|
2661
2821
|
if (!cells) return void 0;
|
|
@@ -2671,10 +2831,20 @@ function tableToMarkdown(data) {
|
|
|
2671
2831
|
for (const cell of parsedCells) {
|
|
2672
2832
|
rows[cell.row][cell.col] = cell.text;
|
|
2673
2833
|
}
|
|
2674
|
-
if (rows.length === 1)
|
|
2834
|
+
if (rows.length === 1) {
|
|
2835
|
+
const markdown2 = rows[0].filter(Boolean).join(" | ");
|
|
2836
|
+
return { markdown: markdown2, headers: [], rows, cells: parsedCells };
|
|
2837
|
+
}
|
|
2675
2838
|
const header = rows[0];
|
|
2676
2839
|
const separator = header.map(() => "---");
|
|
2677
|
-
|
|
2840
|
+
const markdown = [header, separator, ...rows.slice(1)].map((row) => `| ${row.map((value) => value.trim()).join(" | ")} |`).join("\n");
|
|
2841
|
+
return { markdown, headers: header, rows, cells: parsedCells };
|
|
2842
|
+
}
|
|
2843
|
+
function getItemTable(item) {
|
|
2844
|
+
if (typeof item.text === "string" && item.text.trim() || typeof item.orig === "string" && item.orig.trim()) {
|
|
2845
|
+
return void 0;
|
|
2846
|
+
}
|
|
2847
|
+
return parseTableData(item.data);
|
|
2678
2848
|
}
|
|
2679
2849
|
function inferPageCount(document, units) {
|
|
2680
2850
|
const pages = document.pages;
|
|
@@ -2751,8 +2921,18 @@ function buildSourceContext(spans, maxChars = 12e3) {
|
|
|
2751
2921
|
if (spans.length === 0) return "";
|
|
2752
2922
|
const lines = [];
|
|
2753
2923
|
let length = 0;
|
|
2754
|
-
for (const span of spans) {
|
|
2755
|
-
const
|
|
2924
|
+
for (const span of orderSourceSpansForContext(spans)) {
|
|
2925
|
+
const unit = sourceUnit2(span);
|
|
2926
|
+
const table = span.table;
|
|
2927
|
+
const tableContext = [
|
|
2928
|
+
unit ? ` unit:${unit}` : "",
|
|
2929
|
+
table?.tableId ? ` table:${table.tableId}` : "",
|
|
2930
|
+
table?.rowIndex != null ? ` row:${table.rowIndex}` : "",
|
|
2931
|
+
table?.columnIndex != null ? ` col:${table.columnIndex}` : "",
|
|
2932
|
+
table?.columnName ? ` column:${table.columnName}` : "",
|
|
2933
|
+
span.parentSpanId ? ` parent:${span.parentSpanId}` : ""
|
|
2934
|
+
].join("");
|
|
2935
|
+
const header = `[sourceSpan:${span.id}${span.pageStart ? ` page:${span.pageStart}${span.pageEnd && span.pageEnd !== span.pageStart ? `-${span.pageEnd}` : ""}` : ""}${span.sectionId ? ` section:${span.sectionId}` : ""}${span.formNumber ? ` form:${span.formNumber}` : ""}${tableContext}]`;
|
|
2756
2936
|
const text = `${header}
|
|
2757
2937
|
${span.text}`;
|
|
2758
2938
|
if (length + text.length > maxChars && lines.length > 0) break;
|
|
@@ -2766,6 +2946,49 @@ ${lines.join("\n\n")}
|
|
|
2766
2946
|
|
|
2767
2947
|
Use sourceSpan IDs when grounding extracted contractual values.`;
|
|
2768
2948
|
}
|
|
2949
|
+
function sourceUnit2(span) {
|
|
2950
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
2951
|
+
}
|
|
2952
|
+
function sourceContextRank(span) {
|
|
2953
|
+
switch (sourceUnit2(span)) {
|
|
2954
|
+
case "table_row":
|
|
2955
|
+
return 0;
|
|
2956
|
+
case "table":
|
|
2957
|
+
return 1;
|
|
2958
|
+
case "section":
|
|
2959
|
+
return 2;
|
|
2960
|
+
case "page":
|
|
2961
|
+
return 3;
|
|
2962
|
+
case "key_value":
|
|
2963
|
+
return 4;
|
|
2964
|
+
case "table_cell":
|
|
2965
|
+
return 8;
|
|
2966
|
+
default:
|
|
2967
|
+
return 5;
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
function orderSourceSpansForContext(spans) {
|
|
2971
|
+
const parentRows = new Set(
|
|
2972
|
+
spans.filter((span) => sourceUnit2(span) === "table_row").map((span) => span.id)
|
|
2973
|
+
);
|
|
2974
|
+
const filtered = spans.filter((span) => {
|
|
2975
|
+
if (sourceUnit2(span) !== "table_cell") return true;
|
|
2976
|
+
const parent = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
2977
|
+
return !parent || !parentRows.has(parent);
|
|
2978
|
+
});
|
|
2979
|
+
return [...filtered].sort((left, right) => {
|
|
2980
|
+
const leftPage = left.pageStart ?? left.location?.startPage ?? left.location?.page ?? 0;
|
|
2981
|
+
const rightPage = right.pageStart ?? right.location?.startPage ?? right.location?.page ?? 0;
|
|
2982
|
+
if (leftPage !== rightPage) return leftPage - rightPage;
|
|
2983
|
+
const leftRank = sourceContextRank(left);
|
|
2984
|
+
const rightRank = sourceContextRank(right);
|
|
2985
|
+
if (leftRank !== rightRank) return leftRank - rightRank;
|
|
2986
|
+
const leftRow = left.table?.rowIndex ?? Number(left.metadata?.rowIndex ?? Number.NaN);
|
|
2987
|
+
const rightRow = right.table?.rowIndex ?? Number(right.metadata?.rowIndex ?? Number.NaN);
|
|
2988
|
+
if (Number.isFinite(leftRow) && Number.isFinite(rightRow) && leftRow !== rightRow) return leftRow - rightRow;
|
|
2989
|
+
return left.id.localeCompare(right.id);
|
|
2990
|
+
});
|
|
2991
|
+
}
|
|
2769
2992
|
async function runExtractor(params) {
|
|
2770
2993
|
const {
|
|
2771
2994
|
name,
|
|
@@ -4364,7 +4587,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4364
4587
|
const merged = mergeShallowPreferPresent(existing, incoming);
|
|
4365
4588
|
const existingCoverages = Array.isArray(existing.coverages) ? existing.coverages : [];
|
|
4366
4589
|
const incomingCoverages = Array.isArray(incoming.coverages) ? incoming.coverages : [];
|
|
4367
|
-
const
|
|
4590
|
+
const coverageKey2 = (coverage) => keyFromParts(
|
|
4368
4591
|
coverage.name,
|
|
4369
4592
|
coverage.limitType,
|
|
4370
4593
|
coverage.limit,
|
|
@@ -4373,7 +4596,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4373
4596
|
);
|
|
4374
4597
|
const byKey = /* @__PURE__ */ new Map();
|
|
4375
4598
|
for (const coverage of [...existingCoverages, ...incomingCoverages]) {
|
|
4376
|
-
const key =
|
|
4599
|
+
const key = coverageKey2(coverage);
|
|
4377
4600
|
const current = byKey.get(key);
|
|
4378
4601
|
byKey.set(key, current ? mergeShallowPreferPresent(current, coverage) : coverage);
|
|
4379
4602
|
}
|
|
@@ -6720,6 +6943,190 @@ async function resolveReferentialCoverages(params) {
|
|
|
6720
6943
|
};
|
|
6721
6944
|
}
|
|
6722
6945
|
|
|
6946
|
+
// src/extraction/coverage-schedule-recovery.ts
|
|
6947
|
+
function textValue(value) {
|
|
6948
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
6949
|
+
}
|
|
6950
|
+
function numberValue(value) {
|
|
6951
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
6952
|
+
}
|
|
6953
|
+
function normalize(value) {
|
|
6954
|
+
return value.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, " ").trim();
|
|
6955
|
+
}
|
|
6956
|
+
function sourceUnit3(span) {
|
|
6957
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
6958
|
+
}
|
|
6959
|
+
function pageNumber(span) {
|
|
6960
|
+
return span.pageStart ?? span.location?.page ?? span.location?.startPage;
|
|
6961
|
+
}
|
|
6962
|
+
function coveragePages(pageAssignments) {
|
|
6963
|
+
const pages = /* @__PURE__ */ new Set();
|
|
6964
|
+
for (const assignment of pageAssignments) {
|
|
6965
|
+
if (assignment.extractorNames.includes("coverage_limits") || assignment.hasScheduleValues) {
|
|
6966
|
+
pages.add(assignment.localPageNumber);
|
|
6967
|
+
}
|
|
6968
|
+
}
|
|
6969
|
+
return pages;
|
|
6970
|
+
}
|
|
6971
|
+
function parseCurrencyAmount(value) {
|
|
6972
|
+
const match = value.match(/(?:CAD|USD|US)?\s*\$?\s*([0-9][0-9,]*(?:\.\d+)?)/i);
|
|
6973
|
+
if (!match) return void 0;
|
|
6974
|
+
const amount = Number(match[1].replace(/,/g, ""));
|
|
6975
|
+
return Number.isFinite(amount) ? amount : void 0;
|
|
6976
|
+
}
|
|
6977
|
+
function limitTypeFrom(value) {
|
|
6978
|
+
const normalized = normalize(value);
|
|
6979
|
+
if (normalized.includes("/") || normalized.includes("each claim") && normalized.includes("aggregate")) return "scheduled";
|
|
6980
|
+
if (normalized.includes("each claim") || normalized.includes("per claim")) return "per_claim";
|
|
6981
|
+
if (normalized.includes("each occurrence") || normalized.includes("per occurrence")) return "per_occurrence";
|
|
6982
|
+
if (normalized.includes("aggregate")) return "aggregate";
|
|
6983
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage")) return "scheduled";
|
|
6984
|
+
if (normalized.includes("/")) return "scheduled";
|
|
6985
|
+
return void 0;
|
|
6986
|
+
}
|
|
6987
|
+
function limitValueTypeFrom(value) {
|
|
6988
|
+
const normalized = normalize(value);
|
|
6989
|
+
if (parseCurrencyAmount(value) !== void 0) return "numeric";
|
|
6990
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage") || normalized.includes("shown above")) {
|
|
6991
|
+
return "referential";
|
|
6992
|
+
}
|
|
6993
|
+
if (normalized.includes("included")) return "included";
|
|
6994
|
+
if (normalized.includes("not included") || normalized === "nil" || normalized === "none") return "not_included";
|
|
6995
|
+
if (normalized.includes("as stated")) return "as_stated";
|
|
6996
|
+
return "other";
|
|
6997
|
+
}
|
|
6998
|
+
function cleanName(value) {
|
|
6999
|
+
return value.replace(/\s*\([^)]*part of and not in addition to[^)]*\)\s*/gi, " ").replace(/\s+/g, " ").trim().replace(/[.:;-]+$/, "").trim();
|
|
7000
|
+
}
|
|
7001
|
+
function isDeductibleOnly(name, rowText) {
|
|
7002
|
+
const normalizedName = normalize(name);
|
|
7003
|
+
const normalizedRow = normalize(rowText);
|
|
7004
|
+
if (!/\b(deductible|retention|sir)\b/.test(`${normalizedName} ${normalizedRow}`)) return false;
|
|
7005
|
+
if (/\b(coverage|sub limit|sublimit|limit)\b/.test(normalizedName) && !/\b(enhanced|standard)\s+deductible\b/.test(normalizedName)) {
|
|
7006
|
+
return false;
|
|
7007
|
+
}
|
|
7008
|
+
return /\b(enhanced|standard)?\s*(deductible|retention|sir)\b/.test(normalizedName) || /^\s*(deductible|retention|sir)\b/.test(normalizedRow);
|
|
7009
|
+
}
|
|
7010
|
+
function splitRowFields(rowText) {
|
|
7011
|
+
return rowText.split(/\s+\|\s+/).map((part) => part.trim()).filter(Boolean).map((part) => {
|
|
7012
|
+
const match = part.match(/^([^:]{1,100}):\s*(.*)$/);
|
|
7013
|
+
if (!match) return { value: part };
|
|
7014
|
+
return { key: match[1].trim(), value: match[2].trim() };
|
|
7015
|
+
});
|
|
7016
|
+
}
|
|
7017
|
+
function firstField(fields, patterns) {
|
|
7018
|
+
for (const field of fields) {
|
|
7019
|
+
const target = `${field.key ?? ""} ${field.value}`;
|
|
7020
|
+
if (patterns.some((pattern) => pattern.test(target))) return field.value || field.key;
|
|
7021
|
+
}
|
|
7022
|
+
return void 0;
|
|
7023
|
+
}
|
|
7024
|
+
function coverageFromRow(span) {
|
|
7025
|
+
const rowText = span.text.trim();
|
|
7026
|
+
if (!rowText || sourceUnit3(span) !== "table_row") return void 0;
|
|
7027
|
+
if (span.table?.isHeader || span.metadata?.isHeader === "true") return void 0;
|
|
7028
|
+
const fields = splitRowFields(rowText);
|
|
7029
|
+
let name;
|
|
7030
|
+
let limit;
|
|
7031
|
+
for (const field of fields) {
|
|
7032
|
+
const key = field.key?.trim();
|
|
7033
|
+
const value = field.value.trim();
|
|
7034
|
+
if (!key || !value) continue;
|
|
7035
|
+
if (!name && /^coverage$/i.test(key)) name = cleanName(value);
|
|
7036
|
+
if (!limit && /\blimit\b/i.test(key)) limit = value;
|
|
7037
|
+
if (!name && /\b(sub[-\s]?limit|aggregate|each claim)\b/i.test(key) && parseCurrencyAmount(value) !== void 0) {
|
|
7038
|
+
name = cleanName(key);
|
|
7039
|
+
limit = limit ?? value;
|
|
7040
|
+
}
|
|
7041
|
+
}
|
|
7042
|
+
if (!name || !limit) {
|
|
7043
|
+
for (const field of fields) {
|
|
7044
|
+
const key = field.key?.trim();
|
|
7045
|
+
const value = field.value.trim();
|
|
7046
|
+
if (!key || !value) continue;
|
|
7047
|
+
if (!name && /\b(sub[-\s]?limit|coverage|aggregate|each claim)\b/i.test(key) && !parseCurrencyAmount(key)) {
|
|
7048
|
+
name = cleanName(key);
|
|
7049
|
+
limit = limit ?? value;
|
|
7050
|
+
}
|
|
7051
|
+
}
|
|
7052
|
+
}
|
|
7053
|
+
if (!name || !limit) return void 0;
|
|
7054
|
+
if (isDeductibleOnly(name, rowText)) return void 0;
|
|
7055
|
+
const normalizedLimit = normalize(limit);
|
|
7056
|
+
const hasUsableLimit = parseCurrencyAmount(limit) !== void 0 || /\b(shared within|within coverage|as stated|included|not included)\b/i.test(normalizedLimit);
|
|
7057
|
+
if (!hasUsableLimit) return void 0;
|
|
7058
|
+
const deductible = firstField(fields, [/\bdeductible\b/i, /\bretention\b/i, /\bsir\b/i]);
|
|
7059
|
+
const basis = firstField(fields, [/\bbasis\b/i]);
|
|
7060
|
+
const retroactiveDate = firstField(fields, [/\bretroactive date\b/i, /\bretro date\b/i]);
|
|
7061
|
+
const page = pageNumber(span);
|
|
7062
|
+
const limitAmount = parseCurrencyAmount(limit);
|
|
7063
|
+
const deductibleAmount = deductible ? parseCurrencyAmount(deductible) : void 0;
|
|
7064
|
+
return {
|
|
7065
|
+
name,
|
|
7066
|
+
limit,
|
|
7067
|
+
...limitAmount !== void 0 ? { limitAmount } : {},
|
|
7068
|
+
...limitTypeFrom(`${name} ${limit}`) ? { limitType: limitTypeFrom(`${name} ${limit}`) } : {},
|
|
7069
|
+
limitValueType: limitValueTypeFrom(limit),
|
|
7070
|
+
...deductible && !/^nil|none$/i.test(deductible.trim()) ? { deductible } : {},
|
|
7071
|
+
...deductibleAmount !== void 0 ? { deductibleAmount } : {},
|
|
7072
|
+
...deductible ? { deductibleValueType: limitValueTypeFrom(deductible) } : {},
|
|
7073
|
+
...basis && /claims[- ]made/i.test(basis) ? { trigger: "claims_made" } : {},
|
|
7074
|
+
...retroactiveDate ? { retroactiveDate } : {},
|
|
7075
|
+
...span.formNumber ? { formNumber: span.formNumber } : {},
|
|
7076
|
+
...page ? { pageNumber: page } : {},
|
|
7077
|
+
sectionRef: span.sectionId ?? "SCHEDULE",
|
|
7078
|
+
originalContent: rowText,
|
|
7079
|
+
sourceSpanIds: [span.id],
|
|
7080
|
+
sourceTextHash: span.textHash ?? span.hash
|
|
7081
|
+
};
|
|
7082
|
+
}
|
|
7083
|
+
function coverageKey(coverage) {
|
|
7084
|
+
return [
|
|
7085
|
+
textValue(coverage.name) ?? "",
|
|
7086
|
+
textValue(coverage.limit) ?? "",
|
|
7087
|
+
textValue(coverage.limitType) ?? "",
|
|
7088
|
+
numberValue(coverage.pageNumber) ?? ""
|
|
7089
|
+
].map((part) => normalize(String(part))).join("|");
|
|
7090
|
+
}
|
|
7091
|
+
function rowMatchesExisting(row, existing) {
|
|
7092
|
+
const rowKey = coverageKey(row);
|
|
7093
|
+
const rowName = normalize(textValue(row.name) ?? "");
|
|
7094
|
+
const rowLimit = normalize(textValue(row.limit) ?? "");
|
|
7095
|
+
return existing.some((coverage) => {
|
|
7096
|
+
if (coverageKey(coverage) === rowKey) return true;
|
|
7097
|
+
const name = normalize(textValue(coverage.name) ?? "");
|
|
7098
|
+
const limit = normalize(textValue(coverage.limit) ?? "");
|
|
7099
|
+
return Boolean(rowName && rowLimit && name === rowName && limit === rowLimit);
|
|
7100
|
+
});
|
|
7101
|
+
}
|
|
7102
|
+
function recoverCoverageScheduleRows(params) {
|
|
7103
|
+
const payload = params.memory.get("coverage_limits");
|
|
7104
|
+
const existing = Array.isArray(payload?.coverages) ? payload.coverages : [];
|
|
7105
|
+
const pages = coveragePages(params.pageAssignments);
|
|
7106
|
+
const candidates = params.sourceSpans.filter((span) => sourceUnit3(span) === "table_row").filter((span) => {
|
|
7107
|
+
const page = pageNumber(span);
|
|
7108
|
+
return page !== void 0 && pages.has(page);
|
|
7109
|
+
}).map(coverageFromRow).filter((coverage) => Boolean(coverage));
|
|
7110
|
+
const recovered = [];
|
|
7111
|
+
for (const coverage of candidates) {
|
|
7112
|
+
if (rowMatchesExisting(coverage, [...existing, ...recovered])) continue;
|
|
7113
|
+
recovered.push(coverage);
|
|
7114
|
+
}
|
|
7115
|
+
if (recovered.length > 0) {
|
|
7116
|
+
params.memory.set("coverage_limits", {
|
|
7117
|
+
...payload ?? {},
|
|
7118
|
+
coverages: [...existing, ...recovered]
|
|
7119
|
+
});
|
|
7120
|
+
}
|
|
7121
|
+
return {
|
|
7122
|
+
recovered,
|
|
7123
|
+
missingSourceRows: recovered.map((coverage) => {
|
|
7124
|
+
const id = Array.isArray(coverage.sourceSpanIds) ? coverage.sourceSpanIds[0] : void 0;
|
|
7125
|
+
return params.sourceSpans.find((span) => span.id === id);
|
|
7126
|
+
}).filter((span) => Boolean(span))
|
|
7127
|
+
};
|
|
7128
|
+
}
|
|
7129
|
+
|
|
6723
7130
|
// src/extraction/focused-dispatch.ts
|
|
6724
7131
|
async function runFocusedExtractorWithFallback(params) {
|
|
6725
7132
|
const {
|
|
@@ -6922,6 +7329,29 @@ function buildExtractionReviewReport(params) {
|
|
|
6922
7329
|
addMissingSourceGroundingIssues(deterministicIssues, "definitions", "definitions", definitions, "term");
|
|
6923
7330
|
addMissingSourceGroundingIssues(deterministicIssues, "covered_reasons", "coveredReasons", coveredReasons, "name");
|
|
6924
7331
|
}
|
|
7332
|
+
if (params.sourceSpans?.length) {
|
|
7333
|
+
const tempCoveragePayload = {
|
|
7334
|
+
...memory.get("coverage_limits") ?? {},
|
|
7335
|
+
coverages: coverages.map((coverage) => ({ ...coverage }))
|
|
7336
|
+
};
|
|
7337
|
+
const tempMemory = new Map(memory);
|
|
7338
|
+
tempMemory.set("coverage_limits", tempCoveragePayload);
|
|
7339
|
+
const scheduleRecovery = recoverCoverageScheduleRows({
|
|
7340
|
+
memory: tempMemory,
|
|
7341
|
+
sourceSpans: params.sourceSpans,
|
|
7342
|
+
pageAssignments: params.pageAssignments
|
|
7343
|
+
});
|
|
7344
|
+
for (const recovered of scheduleRecovery.recovered) {
|
|
7345
|
+
deterministicIssues.push({
|
|
7346
|
+
code: "coverage_schedule_row_missing",
|
|
7347
|
+
severity: "blocking",
|
|
7348
|
+
message: `Coverage schedule row "${String(recovered.name ?? "unknown")}" is present in source table evidence but missing from extracted coverages.`,
|
|
7349
|
+
extractorName: "coverage_limits",
|
|
7350
|
+
pageNumber: typeof recovered.pageNumber === "number" ? recovered.pageNumber : void 0,
|
|
7351
|
+
itemName: typeof recovered.name === "string" ? recovered.name : void 0
|
|
7352
|
+
});
|
|
7353
|
+
}
|
|
7354
|
+
}
|
|
6925
7355
|
if (mappedDefinitions && definitions.length === 0) {
|
|
6926
7356
|
deterministicIssues.push({
|
|
6927
7357
|
code: "definitions_mapped_but_empty",
|
|
@@ -7350,17 +7780,17 @@ var ARRAY_PATHS = [
|
|
|
7350
7780
|
{ memoryKey: "covered_reasons", arrayKeys: ["coveredReasons", "covered_reasons"] },
|
|
7351
7781
|
{ memoryKey: "declarations", arrayKeys: ["fields"] }
|
|
7352
7782
|
];
|
|
7353
|
-
function
|
|
7783
|
+
function normalize2(value) {
|
|
7354
7784
|
return value.replace(/\s+/g, " ").trim().toLowerCase();
|
|
7355
7785
|
}
|
|
7356
|
-
function
|
|
7786
|
+
function textValue2(record, ...keys) {
|
|
7357
7787
|
for (const key of keys) {
|
|
7358
7788
|
const value = record[key];
|
|
7359
7789
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
7360
7790
|
}
|
|
7361
7791
|
return void 0;
|
|
7362
7792
|
}
|
|
7363
|
-
function
|
|
7793
|
+
function numberValue2(record, ...keys) {
|
|
7364
7794
|
for (const key of keys) {
|
|
7365
7795
|
const value = record[key];
|
|
7366
7796
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
@@ -7377,37 +7807,73 @@ function pageOverlaps(recordStart, recordEnd, span) {
|
|
|
7377
7807
|
return start <= (spanEnd ?? spanStart) && end >= spanStart;
|
|
7378
7808
|
}
|
|
7379
7809
|
function formMatches(record, span) {
|
|
7380
|
-
const formNumber =
|
|
7810
|
+
const formNumber = textValue2(record, "formNumber");
|
|
7381
7811
|
if (!formNumber || !span.formNumber) return false;
|
|
7382
|
-
return
|
|
7812
|
+
return normalize2(formNumber) === normalize2(span.formNumber);
|
|
7383
7813
|
}
|
|
7384
7814
|
function textMatches(record, span) {
|
|
7385
|
-
const spanText =
|
|
7815
|
+
const spanText = normalize2(span.text);
|
|
7386
7816
|
const candidates = [
|
|
7387
|
-
|
|
7388
|
-
|
|
7389
|
-
|
|
7817
|
+
textValue2(record, "originalContent", "content", "definition", "value"),
|
|
7818
|
+
textValue2(record, "name", "title", "term", "field", "coverageName"),
|
|
7819
|
+
textValue2(record, "limit", "deductible", "premium")
|
|
7390
7820
|
].filter((value) => !!value && value.length >= 3);
|
|
7391
|
-
return candidates.some((candidate) => spanText.includes(
|
|
7821
|
+
return candidates.some((candidate) => spanText.includes(normalize2(candidate)));
|
|
7392
7822
|
}
|
|
7393
7823
|
function sourceHashFor(spans) {
|
|
7394
7824
|
return spans.map((span) => span.textHash ?? span.hash).filter(Boolean).join(":") || void 0;
|
|
7395
7825
|
}
|
|
7826
|
+
function sourceUnit4(span) {
|
|
7827
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
7828
|
+
}
|
|
7829
|
+
function hierarchyScore(span) {
|
|
7830
|
+
switch (sourceUnit4(span)) {
|
|
7831
|
+
case "table_row":
|
|
7832
|
+
return 5;
|
|
7833
|
+
case "table":
|
|
7834
|
+
return 3;
|
|
7835
|
+
case "section":
|
|
7836
|
+
case "page":
|
|
7837
|
+
return 2;
|
|
7838
|
+
case "table_cell":
|
|
7839
|
+
return -2;
|
|
7840
|
+
default:
|
|
7841
|
+
return 0;
|
|
7842
|
+
}
|
|
7843
|
+
}
|
|
7844
|
+
function parentRowId(span) {
|
|
7845
|
+
return span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
7846
|
+
}
|
|
7847
|
+
function preferParentRows(matches, sourceSpans) {
|
|
7848
|
+
if (matches.length === 0) return matches;
|
|
7849
|
+
const byId = new Map(sourceSpans.map((span) => [span.id, span]));
|
|
7850
|
+
const expanded = [];
|
|
7851
|
+
const seen = /* @__PURE__ */ new Set();
|
|
7852
|
+
for (const match of matches) {
|
|
7853
|
+
const parent = sourceUnit4(match) === "table_cell" ? byId.get(parentRowId(match) ?? "") : void 0;
|
|
7854
|
+
const preferred = parent && sourceUnit4(parent) === "table_row" ? parent : match;
|
|
7855
|
+
if (seen.has(preferred.id)) continue;
|
|
7856
|
+
seen.add(preferred.id);
|
|
7857
|
+
expanded.push(preferred);
|
|
7858
|
+
}
|
|
7859
|
+
return expanded;
|
|
7860
|
+
}
|
|
7396
7861
|
function findSourceSpansForRecord(record, sourceSpans) {
|
|
7397
7862
|
if (sourceSpans.length === 0) return [];
|
|
7398
|
-
const pageStart =
|
|
7399
|
-
const pageEnd =
|
|
7863
|
+
const pageStart = numberValue2(record, "pageNumber", "pageStart");
|
|
7864
|
+
const pageEnd = numberValue2(record, "pageNumber", "pageEnd");
|
|
7400
7865
|
const scored = sourceSpans.map((span) => {
|
|
7401
7866
|
let score = 0;
|
|
7402
7867
|
if (pageOverlaps(pageStart, pageEnd, span)) score += 4;
|
|
7403
7868
|
if (formMatches(record, span)) score += 3;
|
|
7404
7869
|
if (textMatches(record, span)) score += 2;
|
|
7870
|
+
if (score > 0) score += hierarchyScore(span);
|
|
7405
7871
|
return { span, score };
|
|
7406
7872
|
}).filter((item) => item.score >= 2).sort((left, right) => {
|
|
7407
7873
|
if (right.score !== left.score) return right.score - left.score;
|
|
7408
7874
|
return left.span.id.localeCompare(right.span.id);
|
|
7409
7875
|
});
|
|
7410
|
-
return scored.slice(0,
|
|
7876
|
+
return preferParentRows(scored.slice(0, 5).map((item) => item.span), sourceSpans).slice(0, 3);
|
|
7411
7877
|
}
|
|
7412
7878
|
function groundRecord(record, sourceSpans) {
|
|
7413
7879
|
if (Array.isArray(record.sourceSpanIds) && record.sourceSpanIds.length > 0 && record.sourceTextHash) {
|
|
@@ -8115,6 +8581,14 @@ ${pageText}`;
|
|
|
8115
8581
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8116
8582
|
}
|
|
8117
8583
|
}
|
|
8584
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
8585
|
+
memory,
|
|
8586
|
+
sourceSpans,
|
|
8587
|
+
pageAssignments
|
|
8588
|
+
});
|
|
8589
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
8590
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from table evidence`);
|
|
8591
|
+
}
|
|
8118
8592
|
const planIncludesSupplementary = tasks.some((task) => task.extractorName === "supplementary");
|
|
8119
8593
|
if (!planIncludesSupplementary && hasSupplementaryExtractionSignal(pageAssignments, formInventory, memory)) {
|
|
8120
8594
|
onProgress?.("Extracting supplementary retrieval facts...");
|
|
@@ -8211,7 +8685,8 @@ ${pageText}`;
|
|
|
8211
8685
|
memory,
|
|
8212
8686
|
pageAssignments,
|
|
8213
8687
|
reviewRounds,
|
|
8214
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
8688
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
8689
|
+
sourceSpans
|
|
8215
8690
|
});
|
|
8216
8691
|
if (shouldRunLlmReview(reviewMode, preReviewReport, sourceSpans.length > 0)) {
|
|
8217
8692
|
for (let round = 0; round < maxReviewRounds; round++) {
|
|
@@ -8280,6 +8755,14 @@ ${pageText}`;
|
|
|
8280
8755
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8281
8756
|
}
|
|
8282
8757
|
}
|
|
8758
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
8759
|
+
memory,
|
|
8760
|
+
sourceSpans,
|
|
8761
|
+
pageAssignments
|
|
8762
|
+
});
|
|
8763
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
8764
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from follow-up table evidence`);
|
|
8765
|
+
}
|
|
8283
8766
|
}
|
|
8284
8767
|
} else {
|
|
8285
8768
|
onProgress?.("Skipping LLM extraction review; deterministic checks passed.");
|
|
@@ -8289,7 +8772,8 @@ ${pageText}`;
|
|
|
8289
8772
|
memory,
|
|
8290
8773
|
pageAssignments,
|
|
8291
8774
|
reviewRounds,
|
|
8292
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
8775
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
8776
|
+
sourceSpans
|
|
8293
8777
|
});
|
|
8294
8778
|
if (reviewReport.issues.length > 0) {
|
|
8295
8779
|
await log?.(
|
|
@@ -8315,7 +8799,8 @@ ${pageText}`;
|
|
|
8315
8799
|
memory,
|
|
8316
8800
|
pageAssignments,
|
|
8317
8801
|
reviewRounds,
|
|
8318
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
8802
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
8803
|
+
sourceSpans
|
|
8319
8804
|
}));
|
|
8320
8805
|
onProgress?.("Assembling document...");
|
|
8321
8806
|
const document = assembleDocument(id, documentType, memory);
|
|
@@ -12237,6 +12722,8 @@ export {
|
|
|
12237
12722
|
SourceSpanLocationSchema,
|
|
12238
12723
|
SourceSpanRefSchema,
|
|
12239
12724
|
SourceSpanSchema,
|
|
12725
|
+
SourceSpanTableLocationSchema,
|
|
12726
|
+
SourceSpanUnitSchema,
|
|
12240
12727
|
SubAnswerSchema,
|
|
12241
12728
|
SubQuestionSchema,
|
|
12242
12729
|
SubjectivityCategorySchema,
|