@claritylabs/cl-sdk 1.3.6 → 1.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.d.mts +185 -47
- package/dist/index.d.ts +185 -47
- package/dist/index.js +569 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +567 -55
- package/dist/index.mjs.map +1 -1
- package/dist/storage-sqlite.d.mts +89 -20
- package/dist/storage-sqlite.d.ts +89 -20
- package/dist/storage-sqlite.js +33 -2
- package/dist/storage-sqlite.js.map +1 -1
- package/dist/storage-sqlite.mjs +33 -2
- package/dist/storage-sqlite.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -623,6 +623,8 @@ var CoverageSchema = z4.object({
|
|
|
623
623
|
deductible: z4.string().optional(),
|
|
624
624
|
deductibleAmount: z4.number().optional(),
|
|
625
625
|
deductibleValueType: CoverageValueTypeSchema.optional(),
|
|
626
|
+
trigger: CoverageTriggerSchema.optional(),
|
|
627
|
+
retroactiveDate: z4.string().optional(),
|
|
626
628
|
formNumber: z4.string().optional(),
|
|
627
629
|
pageNumber: z4.number().optional(),
|
|
628
630
|
sectionRef: z4.string().optional(),
|
|
@@ -1851,6 +1853,15 @@ var SourceSpanKindSchema = z20.enum([
|
|
|
1851
1853
|
"plain_text",
|
|
1852
1854
|
"structured_field"
|
|
1853
1855
|
]);
|
|
1856
|
+
var SourceSpanUnitSchema = z20.enum([
|
|
1857
|
+
"page",
|
|
1858
|
+
"section",
|
|
1859
|
+
"table",
|
|
1860
|
+
"table_row",
|
|
1861
|
+
"table_cell",
|
|
1862
|
+
"key_value",
|
|
1863
|
+
"text"
|
|
1864
|
+
]);
|
|
1854
1865
|
var SourceKindSchema = z20.enum([
|
|
1855
1866
|
"policy_pdf",
|
|
1856
1867
|
"application_pdf",
|
|
@@ -1875,6 +1886,15 @@ var SourceSpanLocationSchema = z20.object({
|
|
|
1875
1886
|
lineEnd: z20.number().int().positive().optional(),
|
|
1876
1887
|
fieldPath: z20.string().optional()
|
|
1877
1888
|
});
|
|
1889
|
+
var SourceSpanTableLocationSchema = z20.object({
|
|
1890
|
+
tableId: z20.string().optional(),
|
|
1891
|
+
rowIndex: z20.number().int().nonnegative().optional(),
|
|
1892
|
+
columnIndex: z20.number().int().nonnegative().optional(),
|
|
1893
|
+
columnName: z20.string().optional(),
|
|
1894
|
+
rowSpanId: z20.string().optional(),
|
|
1895
|
+
tableSpanId: z20.string().optional(),
|
|
1896
|
+
isHeader: z20.boolean().optional()
|
|
1897
|
+
});
|
|
1878
1898
|
var SourceSpanSchema = z20.object({
|
|
1879
1899
|
id: z20.string().min(1),
|
|
1880
1900
|
documentId: z20.string().min(1),
|
|
@@ -1888,6 +1908,9 @@ var SourceSpanSchema = z20.object({
|
|
|
1888
1908
|
pageEnd: z20.number().int().positive().optional(),
|
|
1889
1909
|
sectionId: z20.string().optional(),
|
|
1890
1910
|
formNumber: z20.string().optional(),
|
|
1911
|
+
sourceUnit: SourceSpanUnitSchema.optional(),
|
|
1912
|
+
parentSpanId: z20.string().optional(),
|
|
1913
|
+
table: SourceSpanTableLocationSchema.optional(),
|
|
1891
1914
|
bbox: z20.array(SourceSpanBBoxSchema).optional(),
|
|
1892
1915
|
location: SourceSpanLocationSchema.optional(),
|
|
1893
1916
|
metadata: z20.record(z20.string(), z20.string()).optional()
|
|
@@ -2006,6 +2029,9 @@ function buildSourceSpan(input, localIndex = 0) {
|
|
|
2006
2029
|
pageEnd: input.pageEnd,
|
|
2007
2030
|
sectionId: input.sectionId,
|
|
2008
2031
|
formNumber: input.formNumber,
|
|
2032
|
+
sourceUnit: input.sourceUnit,
|
|
2033
|
+
parentSpanId: input.parentSpanId,
|
|
2034
|
+
table: input.table,
|
|
2009
2035
|
location: {
|
|
2010
2036
|
page: input.pageStart === input.pageEnd ? input.pageStart : void 0,
|
|
2011
2037
|
startPage: input.pageStart,
|
|
@@ -2026,7 +2052,11 @@ function buildPageSourceSpans(pages) {
|
|
|
2026
2052
|
pageEnd: page.pageNumber,
|
|
2027
2053
|
sectionId: page.sectionId,
|
|
2028
2054
|
formNumber: page.formNumber,
|
|
2029
|
-
|
|
2055
|
+
sourceUnit: "page",
|
|
2056
|
+
metadata: {
|
|
2057
|
+
...page.metadata ?? {},
|
|
2058
|
+
sourceUnit: page.metadata?.sourceUnit ?? "page"
|
|
2059
|
+
}
|
|
2030
2060
|
},
|
|
2031
2061
|
index
|
|
2032
2062
|
)
|
|
@@ -2048,6 +2078,7 @@ function buildSectionSourceSpans(pages, options = {}) {
|
|
|
2048
2078
|
pageEnd: page.pageNumber,
|
|
2049
2079
|
sectionId: section.title,
|
|
2050
2080
|
formNumber: inferFormNumber(section.text),
|
|
2081
|
+
sourceUnit: "section",
|
|
2051
2082
|
metadata: {
|
|
2052
2083
|
...page.metadata ?? {},
|
|
2053
2084
|
sourceUnit: "section_candidate"
|
|
@@ -2080,6 +2111,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2080
2111
|
const chunks = [];
|
|
2081
2112
|
let current = [];
|
|
2082
2113
|
let currentLength = 0;
|
|
2114
|
+
const spansForChunking = filterChunkableSourceSpans(spans);
|
|
2083
2115
|
const flush = () => {
|
|
2084
2116
|
if (current.length === 0) return;
|
|
2085
2117
|
const text = current.map((span) => span.text).join("\n\n");
|
|
@@ -2103,7 +2135,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2103
2135
|
current = [];
|
|
2104
2136
|
currentLength = 0;
|
|
2105
2137
|
};
|
|
2106
|
-
for (const span of
|
|
2138
|
+
for (const span of spansForChunking) {
|
|
2107
2139
|
const nextLength = currentLength + span.text.length + (current.length > 0 ? 2 : 0);
|
|
2108
2140
|
if (current.length > 0 && nextLength > maxChars) {
|
|
2109
2141
|
flush();
|
|
@@ -2114,6 +2146,20 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2114
2146
|
flush();
|
|
2115
2147
|
return chunks;
|
|
2116
2148
|
}
|
|
2149
|
+
function sourceUnit(span) {
|
|
2150
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
2151
|
+
}
|
|
2152
|
+
function filterChunkableSourceSpans(spans) {
|
|
2153
|
+
const rowIds = new Set(
|
|
2154
|
+
spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
|
|
2155
|
+
);
|
|
2156
|
+
if (rowIds.size === 0) return spans;
|
|
2157
|
+
return spans.filter((span) => {
|
|
2158
|
+
if (sourceUnit(span) !== "table_cell") return true;
|
|
2159
|
+
const rowId = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
2160
|
+
return !rowId || !rowIds.has(rowId);
|
|
2161
|
+
});
|
|
2162
|
+
}
|
|
2117
2163
|
function splitPageIntoSections(text, headingPattern, minSectionChars) {
|
|
2118
2164
|
const lines = text.split(/\r?\n/);
|
|
2119
2165
|
const sections = [];
|
|
@@ -2506,36 +2552,18 @@ function normalizeDoclingDocument(document, options) {
|
|
|
2506
2552
|
pageTexts.set(page, appendText(pageTexts.get(page), unit.text));
|
|
2507
2553
|
}
|
|
2508
2554
|
const fullText = Array.from({ length: pageCount }, (_, index) => {
|
|
2509
|
-
const
|
|
2510
|
-
const text = pageTexts.get(
|
|
2511
|
-
return text ? `Page ${
|
|
2555
|
+
const pageNumber2 = index + 1;
|
|
2556
|
+
const text = pageTexts.get(pageNumber2)?.trim();
|
|
2557
|
+
return text ? `Page ${pageNumber2}
|
|
2512
2558
|
${text}` : "";
|
|
2513
2559
|
}).filter(Boolean).join("\n\n");
|
|
2514
2560
|
const sourceKind = options.sourceKind ?? "policy_pdf";
|
|
2515
|
-
const sourceSpans = units.
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
pageStart: unit.pageStart,
|
|
2522
|
-
pageEnd: unit.pageEnd,
|
|
2523
|
-
sectionId: unit.label,
|
|
2524
|
-
metadata: {
|
|
2525
|
-
sourceSystem: "docling",
|
|
2526
|
-
sourceUnit: "docling_item",
|
|
2527
|
-
doclingRef: unit.ref,
|
|
2528
|
-
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2529
|
-
}
|
|
2530
|
-
},
|
|
2531
|
-
index
|
|
2532
|
-
);
|
|
2533
|
-
return {
|
|
2534
|
-
...span,
|
|
2535
|
-
kind: "plain_text",
|
|
2536
|
-
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2537
|
-
};
|
|
2538
|
-
});
|
|
2561
|
+
const sourceSpans = units.flatMap(
|
|
2562
|
+
(unit, index) => buildSourceSpansForUnit(unit, index, {
|
|
2563
|
+
documentId: options.documentId,
|
|
2564
|
+
sourceKind
|
|
2565
|
+
})
|
|
2566
|
+
);
|
|
2539
2567
|
return {
|
|
2540
2568
|
pageCount,
|
|
2541
2569
|
fullText,
|
|
@@ -2573,6 +2601,11 @@ function mergeSourceSpans(spans) {
|
|
|
2573
2601
|
span.pageStart ?? span.location?.startPage ?? span.location?.page ?? "na",
|
|
2574
2602
|
span.pageEnd ?? span.location?.endPage ?? span.pageStart ?? "na",
|
|
2575
2603
|
span.sectionId ?? span.location?.fieldPath ?? "na",
|
|
2604
|
+
span.sourceUnit ?? span.metadata?.sourceUnit ?? "na",
|
|
2605
|
+
span.parentSpanId ?? "na",
|
|
2606
|
+
span.table?.tableId ?? span.metadata?.tableId ?? "na",
|
|
2607
|
+
span.table?.rowIndex ?? span.metadata?.rowIndex ?? "na",
|
|
2608
|
+
span.table?.columnIndex ?? span.metadata?.columnIndex ?? "na",
|
|
2576
2609
|
span.textHash ?? sourceSpanTextHash(span.text)
|
|
2577
2610
|
].join(":");
|
|
2578
2611
|
if (seen.has(key)) continue;
|
|
@@ -2635,6 +2668,7 @@ function getOrderedBodyRefs(document, itemMap) {
|
|
|
2635
2668
|
function normalizeItem(ref, item) {
|
|
2636
2669
|
const text = getItemText(item).trim();
|
|
2637
2670
|
if (!text) return void 0;
|
|
2671
|
+
const table = getItemTable(item);
|
|
2638
2672
|
const pages = (item.prov ?? []).map((prov) => getPageNumber(prov)).filter((page) => typeof page === "number" && page > 0);
|
|
2639
2673
|
const pageStart = pages.length ? Math.min(...pages) : void 0;
|
|
2640
2674
|
const pageEnd = pages.length ? Math.max(...pages) : pageStart;
|
|
@@ -2645,17 +2679,143 @@ function normalizeItem(ref, item) {
|
|
|
2645
2679
|
text,
|
|
2646
2680
|
pageStart,
|
|
2647
2681
|
pageEnd,
|
|
2648
|
-
bboxes: bboxes.length ? bboxes : void 0
|
|
2682
|
+
bboxes: bboxes.length ? bboxes : void 0,
|
|
2683
|
+
table
|
|
2684
|
+
};
|
|
2685
|
+
}
|
|
2686
|
+
function buildSourceSpansForUnit(unit, index, options) {
|
|
2687
|
+
const baseMetadata = {
|
|
2688
|
+
sourceSystem: "docling",
|
|
2689
|
+
sourceUnit: unit.table ? "table" : "docling_item",
|
|
2690
|
+
doclingRef: unit.ref,
|
|
2691
|
+
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2692
|
+
};
|
|
2693
|
+
const tableId = unit.table ? `${unit.ref}:table` : void 0;
|
|
2694
|
+
const tableSpan = withDoclingKind(buildSourceSpan(
|
|
2695
|
+
{
|
|
2696
|
+
documentId: options.documentId,
|
|
2697
|
+
sourceKind: options.sourceKind,
|
|
2698
|
+
text: unit.text,
|
|
2699
|
+
pageStart: unit.pageStart,
|
|
2700
|
+
pageEnd: unit.pageEnd,
|
|
2701
|
+
sectionId: unit.label,
|
|
2702
|
+
sourceUnit: unit.table ? "table" : labelToSourceUnit(unit.label),
|
|
2703
|
+
table: tableId ? { tableId } : void 0,
|
|
2704
|
+
metadata: baseMetadata
|
|
2705
|
+
},
|
|
2706
|
+
index * 1e3
|
|
2707
|
+
), unit);
|
|
2708
|
+
if (!unit.table) return [tableSpan];
|
|
2709
|
+
const spans = [tableSpan];
|
|
2710
|
+
const table = unit.table;
|
|
2711
|
+
for (let rowIndex = 0; rowIndex < table.rows.length; rowIndex += 1) {
|
|
2712
|
+
const row = table.rows[rowIndex];
|
|
2713
|
+
const isHeader = rowIndex === 0 && table.headers.length > 0;
|
|
2714
|
+
const rowText = tableRowText(table, rowIndex);
|
|
2715
|
+
if (!rowText) continue;
|
|
2716
|
+
const rowSpan = withDoclingKind(buildSourceSpan(
|
|
2717
|
+
{
|
|
2718
|
+
documentId: options.documentId,
|
|
2719
|
+
sourceKind: options.sourceKind,
|
|
2720
|
+
text: rowText,
|
|
2721
|
+
pageStart: unit.pageStart,
|
|
2722
|
+
pageEnd: unit.pageEnd,
|
|
2723
|
+
sectionId: unit.label,
|
|
2724
|
+
sourceUnit: "table_row",
|
|
2725
|
+
parentSpanId: tableSpan.id,
|
|
2726
|
+
table: {
|
|
2727
|
+
tableId,
|
|
2728
|
+
tableSpanId: tableSpan.id,
|
|
2729
|
+
rowIndex,
|
|
2730
|
+
isHeader
|
|
2731
|
+
},
|
|
2732
|
+
metadata: {
|
|
2733
|
+
...baseMetadata,
|
|
2734
|
+
sourceUnit: "table_row",
|
|
2735
|
+
tableId: tableId ?? "",
|
|
2736
|
+
tableSpanId: tableSpan.id,
|
|
2737
|
+
rowIndex: String(rowIndex),
|
|
2738
|
+
isHeader: String(isHeader)
|
|
2739
|
+
}
|
|
2740
|
+
},
|
|
2741
|
+
index * 1e3 + 1 + rowIndex
|
|
2742
|
+
), unit);
|
|
2743
|
+
spans.push(rowSpan);
|
|
2744
|
+
for (let columnIndex = 0; columnIndex < row.length; columnIndex += 1) {
|
|
2745
|
+
const text = row[columnIndex]?.trim();
|
|
2746
|
+
if (!text) continue;
|
|
2747
|
+
const columnName = table.headers[columnIndex]?.trim() || void 0;
|
|
2748
|
+
const cellSpan = withDoclingKind(buildSourceSpan(
|
|
2749
|
+
{
|
|
2750
|
+
documentId: options.documentId,
|
|
2751
|
+
sourceKind: options.sourceKind,
|
|
2752
|
+
text,
|
|
2753
|
+
pageStart: unit.pageStart,
|
|
2754
|
+
pageEnd: unit.pageEnd,
|
|
2755
|
+
sectionId: unit.label,
|
|
2756
|
+
sourceUnit: "table_cell",
|
|
2757
|
+
parentSpanId: rowSpan.id,
|
|
2758
|
+
table: {
|
|
2759
|
+
tableId,
|
|
2760
|
+
tableSpanId: tableSpan.id,
|
|
2761
|
+
rowSpanId: rowSpan.id,
|
|
2762
|
+
rowIndex,
|
|
2763
|
+
columnIndex,
|
|
2764
|
+
columnName,
|
|
2765
|
+
isHeader
|
|
2766
|
+
},
|
|
2767
|
+
metadata: {
|
|
2768
|
+
...baseMetadata,
|
|
2769
|
+
sourceUnit: "table_cell",
|
|
2770
|
+
tableId: tableId ?? "",
|
|
2771
|
+
tableSpanId: tableSpan.id,
|
|
2772
|
+
rowSpanId: rowSpan.id,
|
|
2773
|
+
rowIndex: String(rowIndex),
|
|
2774
|
+
columnIndex: String(columnIndex),
|
|
2775
|
+
...columnName ? { columnName } : {},
|
|
2776
|
+
isHeader: String(isHeader)
|
|
2777
|
+
}
|
|
2778
|
+
},
|
|
2779
|
+
index * 1e3 + 100 + rowIndex * 100 + columnIndex
|
|
2780
|
+
), unit);
|
|
2781
|
+
spans.push(cellSpan);
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
return spans;
|
|
2785
|
+
}
|
|
2786
|
+
function withDoclingKind(span, unit) {
|
|
2787
|
+
return {
|
|
2788
|
+
...span,
|
|
2789
|
+
kind: "plain_text",
|
|
2790
|
+
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2649
2791
|
};
|
|
2650
2792
|
}
|
|
2793
|
+
function labelToSourceUnit(label) {
|
|
2794
|
+
const normalized = label?.toLowerCase() ?? "";
|
|
2795
|
+
if (normalized.includes("table")) return "table";
|
|
2796
|
+
if (normalized.includes("key")) return "key_value";
|
|
2797
|
+
if (normalized.includes("section") || normalized.includes("header")) return "section";
|
|
2798
|
+
return "text";
|
|
2799
|
+
}
|
|
2800
|
+
function tableRowText(table, rowIndex) {
|
|
2801
|
+
const row = table.rows[rowIndex] ?? [];
|
|
2802
|
+
const headers = table.headers;
|
|
2803
|
+
if (rowIndex === 0 && headers.length > 0) return row.filter(Boolean).join(" | ");
|
|
2804
|
+
return row.map((value, columnIndex) => {
|
|
2805
|
+
const trimmed = value.trim();
|
|
2806
|
+
if (!trimmed) return "";
|
|
2807
|
+
const header = headers[columnIndex]?.trim();
|
|
2808
|
+
return header ? `${header}: ${trimmed}` : trimmed;
|
|
2809
|
+
}).filter(Boolean).join(" | ");
|
|
2810
|
+
}
|
|
2651
2811
|
function getItemText(item) {
|
|
2652
2812
|
if (typeof item.text === "string" && item.text.trim()) return item.text;
|
|
2653
2813
|
if (typeof item.orig === "string" && item.orig.trim()) return item.orig;
|
|
2654
|
-
const table =
|
|
2655
|
-
if (table) return table;
|
|
2814
|
+
const table = parseTableData(item.data);
|
|
2815
|
+
if (table) return table.markdown;
|
|
2656
2816
|
return "";
|
|
2657
2817
|
}
|
|
2658
|
-
function
|
|
2818
|
+
function parseTableData(data) {
|
|
2659
2819
|
const record = asRecord(data);
|
|
2660
2820
|
const cells = Array.isArray(record?.table_cells) ? record.table_cells : Array.isArray(record?.tableCells) ? record.tableCells : void 0;
|
|
2661
2821
|
if (!cells) return void 0;
|
|
@@ -2671,10 +2831,20 @@ function tableToMarkdown(data) {
|
|
|
2671
2831
|
for (const cell of parsedCells) {
|
|
2672
2832
|
rows[cell.row][cell.col] = cell.text;
|
|
2673
2833
|
}
|
|
2674
|
-
if (rows.length === 1)
|
|
2834
|
+
if (rows.length === 1) {
|
|
2835
|
+
const markdown2 = rows[0].filter(Boolean).join(" | ");
|
|
2836
|
+
return { markdown: markdown2, headers: [], rows, cells: parsedCells };
|
|
2837
|
+
}
|
|
2675
2838
|
const header = rows[0];
|
|
2676
2839
|
const separator = header.map(() => "---");
|
|
2677
|
-
|
|
2840
|
+
const markdown = [header, separator, ...rows.slice(1)].map((row) => `| ${row.map((value) => value.trim()).join(" | ")} |`).join("\n");
|
|
2841
|
+
return { markdown, headers: header, rows, cells: parsedCells };
|
|
2842
|
+
}
|
|
2843
|
+
function getItemTable(item) {
|
|
2844
|
+
if (typeof item.text === "string" && item.text.trim() || typeof item.orig === "string" && item.orig.trim()) {
|
|
2845
|
+
return void 0;
|
|
2846
|
+
}
|
|
2847
|
+
return parseTableData(item.data);
|
|
2678
2848
|
}
|
|
2679
2849
|
function inferPageCount(document, units) {
|
|
2680
2850
|
const pages = document.pages;
|
|
@@ -2751,8 +2921,18 @@ function buildSourceContext(spans, maxChars = 12e3) {
|
|
|
2751
2921
|
if (spans.length === 0) return "";
|
|
2752
2922
|
const lines = [];
|
|
2753
2923
|
let length = 0;
|
|
2754
|
-
for (const span of spans) {
|
|
2755
|
-
const
|
|
2924
|
+
for (const span of orderSourceSpansForContext(spans)) {
|
|
2925
|
+
const unit = sourceUnit2(span);
|
|
2926
|
+
const table = span.table;
|
|
2927
|
+
const tableContext = [
|
|
2928
|
+
unit ? ` unit:${unit}` : "",
|
|
2929
|
+
table?.tableId ? ` table:${table.tableId}` : "",
|
|
2930
|
+
table?.rowIndex != null ? ` row:${table.rowIndex}` : "",
|
|
2931
|
+
table?.columnIndex != null ? ` col:${table.columnIndex}` : "",
|
|
2932
|
+
table?.columnName ? ` column:${table.columnName}` : "",
|
|
2933
|
+
span.parentSpanId ? ` parent:${span.parentSpanId}` : ""
|
|
2934
|
+
].join("");
|
|
2935
|
+
const header = `[sourceSpan:${span.id}${span.pageStart ? ` page:${span.pageStart}${span.pageEnd && span.pageEnd !== span.pageStart ? `-${span.pageEnd}` : ""}` : ""}${span.sectionId ? ` section:${span.sectionId}` : ""}${span.formNumber ? ` form:${span.formNumber}` : ""}${tableContext}]`;
|
|
2756
2936
|
const text = `${header}
|
|
2757
2937
|
${span.text}`;
|
|
2758
2938
|
if (length + text.length > maxChars && lines.length > 0) break;
|
|
@@ -2766,6 +2946,49 @@ ${lines.join("\n\n")}
|
|
|
2766
2946
|
|
|
2767
2947
|
Use sourceSpan IDs when grounding extracted contractual values.`;
|
|
2768
2948
|
}
|
|
2949
|
+
function sourceUnit2(span) {
|
|
2950
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
2951
|
+
}
|
|
2952
|
+
function sourceContextRank(span) {
|
|
2953
|
+
switch (sourceUnit2(span)) {
|
|
2954
|
+
case "table_row":
|
|
2955
|
+
return 0;
|
|
2956
|
+
case "table":
|
|
2957
|
+
return 1;
|
|
2958
|
+
case "section":
|
|
2959
|
+
return 2;
|
|
2960
|
+
case "page":
|
|
2961
|
+
return 3;
|
|
2962
|
+
case "key_value":
|
|
2963
|
+
return 4;
|
|
2964
|
+
case "table_cell":
|
|
2965
|
+
return 8;
|
|
2966
|
+
default:
|
|
2967
|
+
return 5;
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
function orderSourceSpansForContext(spans) {
|
|
2971
|
+
const parentRows = new Set(
|
|
2972
|
+
spans.filter((span) => sourceUnit2(span) === "table_row").map((span) => span.id)
|
|
2973
|
+
);
|
|
2974
|
+
const filtered = spans.filter((span) => {
|
|
2975
|
+
if (sourceUnit2(span) !== "table_cell") return true;
|
|
2976
|
+
const parent = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
2977
|
+
return !parent || !parentRows.has(parent);
|
|
2978
|
+
});
|
|
2979
|
+
return [...filtered].sort((left, right) => {
|
|
2980
|
+
const leftPage = left.pageStart ?? left.location?.startPage ?? left.location?.page ?? 0;
|
|
2981
|
+
const rightPage = right.pageStart ?? right.location?.startPage ?? right.location?.page ?? 0;
|
|
2982
|
+
if (leftPage !== rightPage) return leftPage - rightPage;
|
|
2983
|
+
const leftRank = sourceContextRank(left);
|
|
2984
|
+
const rightRank = sourceContextRank(right);
|
|
2985
|
+
if (leftRank !== rightRank) return leftRank - rightRank;
|
|
2986
|
+
const leftRow = left.table?.rowIndex ?? Number(left.metadata?.rowIndex ?? Number.NaN);
|
|
2987
|
+
const rightRow = right.table?.rowIndex ?? Number(right.metadata?.rowIndex ?? Number.NaN);
|
|
2988
|
+
if (Number.isFinite(leftRow) && Number.isFinite(rightRow) && leftRow !== rightRow) return leftRow - rightRow;
|
|
2989
|
+
return left.id.localeCompare(right.id);
|
|
2990
|
+
});
|
|
2991
|
+
}
|
|
2769
2992
|
async function runExtractor(params) {
|
|
2770
2993
|
const {
|
|
2771
2994
|
name,
|
|
@@ -4364,7 +4587,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4364
4587
|
const merged = mergeShallowPreferPresent(existing, incoming);
|
|
4365
4588
|
const existingCoverages = Array.isArray(existing.coverages) ? existing.coverages : [];
|
|
4366
4589
|
const incomingCoverages = Array.isArray(incoming.coverages) ? incoming.coverages : [];
|
|
4367
|
-
const
|
|
4590
|
+
const coverageKey2 = (coverage) => keyFromParts(
|
|
4368
4591
|
coverage.name,
|
|
4369
4592
|
coverage.limitType,
|
|
4370
4593
|
coverage.limit,
|
|
@@ -4373,7 +4596,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4373
4596
|
);
|
|
4374
4597
|
const byKey = /* @__PURE__ */ new Map();
|
|
4375
4598
|
for (const coverage of [...existingCoverages, ...incomingCoverages]) {
|
|
4376
|
-
const key =
|
|
4599
|
+
const key = coverageKey2(coverage);
|
|
4377
4600
|
const current = byKey.get(key);
|
|
4378
4601
|
byKey.set(key, current ? mergeShallowPreferPresent(current, coverage) : coverage);
|
|
4379
4602
|
}
|
|
@@ -6720,6 +6943,190 @@ async function resolveReferentialCoverages(params) {
|
|
|
6720
6943
|
};
|
|
6721
6944
|
}
|
|
6722
6945
|
|
|
6946
|
+
// src/extraction/coverage-schedule-recovery.ts
|
|
6947
|
+
function textValue(value) {
|
|
6948
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
6949
|
+
}
|
|
6950
|
+
function numberValue(value) {
|
|
6951
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
6952
|
+
}
|
|
6953
|
+
function normalize(value) {
|
|
6954
|
+
return value.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, " ").trim();
|
|
6955
|
+
}
|
|
6956
|
+
function sourceUnit3(span) {
|
|
6957
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
6958
|
+
}
|
|
6959
|
+
function pageNumber(span) {
|
|
6960
|
+
return span.pageStart ?? span.location?.page ?? span.location?.startPage;
|
|
6961
|
+
}
|
|
6962
|
+
function coveragePages(pageAssignments) {
|
|
6963
|
+
const pages = /* @__PURE__ */ new Set();
|
|
6964
|
+
for (const assignment of pageAssignments) {
|
|
6965
|
+
if (assignment.extractorNames.includes("coverage_limits") || assignment.hasScheduleValues) {
|
|
6966
|
+
pages.add(assignment.localPageNumber);
|
|
6967
|
+
}
|
|
6968
|
+
}
|
|
6969
|
+
return pages;
|
|
6970
|
+
}
|
|
6971
|
+
function parseCurrencyAmount(value) {
|
|
6972
|
+
const match = value.match(/(?:CAD|USD|US)?\s*\$?\s*([0-9][0-9,]*(?:\.\d+)?)/i);
|
|
6973
|
+
if (!match) return void 0;
|
|
6974
|
+
const amount = Number(match[1].replace(/,/g, ""));
|
|
6975
|
+
return Number.isFinite(amount) ? amount : void 0;
|
|
6976
|
+
}
|
|
6977
|
+
function limitTypeFrom(value) {
|
|
6978
|
+
const normalized = normalize(value);
|
|
6979
|
+
if (normalized.includes("/") || normalized.includes("each claim") && normalized.includes("aggregate")) return "scheduled";
|
|
6980
|
+
if (normalized.includes("each claim") || normalized.includes("per claim")) return "per_claim";
|
|
6981
|
+
if (normalized.includes("each occurrence") || normalized.includes("per occurrence")) return "per_occurrence";
|
|
6982
|
+
if (normalized.includes("aggregate")) return "aggregate";
|
|
6983
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage")) return "scheduled";
|
|
6984
|
+
if (normalized.includes("/")) return "scheduled";
|
|
6985
|
+
return void 0;
|
|
6986
|
+
}
|
|
6987
|
+
function limitValueTypeFrom(value) {
|
|
6988
|
+
const normalized = normalize(value);
|
|
6989
|
+
if (parseCurrencyAmount(value) !== void 0) return "numeric";
|
|
6990
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage") || normalized.includes("shown above")) {
|
|
6991
|
+
return "referential";
|
|
6992
|
+
}
|
|
6993
|
+
if (normalized.includes("included")) return "included";
|
|
6994
|
+
if (normalized.includes("not included") || normalized === "nil" || normalized === "none") return "not_included";
|
|
6995
|
+
if (normalized.includes("as stated")) return "as_stated";
|
|
6996
|
+
return "other";
|
|
6997
|
+
}
|
|
6998
|
+
function cleanName(value) {
|
|
6999
|
+
return value.replace(/\s*\([^)]*part of and not in addition to[^)]*\)\s*/gi, " ").replace(/\s+/g, " ").trim().replace(/[.:;-]+$/, "").trim();
|
|
7000
|
+
}
|
|
7001
|
+
function isDeductibleOnly(name, rowText) {
|
|
7002
|
+
const normalizedName = normalize(name);
|
|
7003
|
+
const normalizedRow = normalize(rowText);
|
|
7004
|
+
if (!/\b(deductible|retention|sir)\b/.test(`${normalizedName} ${normalizedRow}`)) return false;
|
|
7005
|
+
if (/\b(coverage|sub limit|sublimit|limit)\b/.test(normalizedName) && !/\b(enhanced|standard)\s+deductible\b/.test(normalizedName)) {
|
|
7006
|
+
return false;
|
|
7007
|
+
}
|
|
7008
|
+
return /\b(enhanced|standard)?\s*(deductible|retention|sir)\b/.test(normalizedName) || /^\s*(deductible|retention|sir)\b/.test(normalizedRow);
|
|
7009
|
+
}
|
|
7010
|
+
function splitRowFields(rowText) {
|
|
7011
|
+
return rowText.split(/\s+\|\s+/).map((part) => part.trim()).filter(Boolean).map((part) => {
|
|
7012
|
+
const match = part.match(/^([^:]{1,100}):\s*(.*)$/);
|
|
7013
|
+
if (!match) return { value: part };
|
|
7014
|
+
return { key: match[1].trim(), value: match[2].trim() };
|
|
7015
|
+
});
|
|
7016
|
+
}
|
|
7017
|
+
function firstField(fields, patterns) {
|
|
7018
|
+
for (const field of fields) {
|
|
7019
|
+
const target = `${field.key ?? ""} ${field.value}`;
|
|
7020
|
+
if (patterns.some((pattern) => pattern.test(target))) return field.value || field.key;
|
|
7021
|
+
}
|
|
7022
|
+
return void 0;
|
|
7023
|
+
}
|
|
7024
|
+
function coverageFromRow(span) {
|
|
7025
|
+
const rowText = span.text.trim();
|
|
7026
|
+
if (!rowText || sourceUnit3(span) !== "table_row") return void 0;
|
|
7027
|
+
if (span.table?.isHeader || span.metadata?.isHeader === "true") return void 0;
|
|
7028
|
+
const fields = splitRowFields(rowText);
|
|
7029
|
+
let name;
|
|
7030
|
+
let limit;
|
|
7031
|
+
for (const field of fields) {
|
|
7032
|
+
const key = field.key?.trim();
|
|
7033
|
+
const value = field.value.trim();
|
|
7034
|
+
if (!key || !value) continue;
|
|
7035
|
+
if (!name && /^coverage$/i.test(key)) name = cleanName(value);
|
|
7036
|
+
if (!limit && /\blimit\b/i.test(key)) limit = value;
|
|
7037
|
+
if (!name && /\b(sub[-\s]?limit|aggregate|each claim)\b/i.test(key) && parseCurrencyAmount(value) !== void 0) {
|
|
7038
|
+
name = cleanName(key);
|
|
7039
|
+
limit = limit ?? value;
|
|
7040
|
+
}
|
|
7041
|
+
}
|
|
7042
|
+
if (!name || !limit) {
|
|
7043
|
+
for (const field of fields) {
|
|
7044
|
+
const key = field.key?.trim();
|
|
7045
|
+
const value = field.value.trim();
|
|
7046
|
+
if (!key || !value) continue;
|
|
7047
|
+
if (!name && /\b(sub[-\s]?limit|coverage|aggregate|each claim)\b/i.test(key) && !parseCurrencyAmount(key)) {
|
|
7048
|
+
name = cleanName(key);
|
|
7049
|
+
limit = limit ?? value;
|
|
7050
|
+
}
|
|
7051
|
+
}
|
|
7052
|
+
}
|
|
7053
|
+
if (!name || !limit) return void 0;
|
|
7054
|
+
if (isDeductibleOnly(name, rowText)) return void 0;
|
|
7055
|
+
const normalizedLimit = normalize(limit);
|
|
7056
|
+
const hasUsableLimit = parseCurrencyAmount(limit) !== void 0 || /\b(shared within|within coverage|as stated|included|not included)\b/i.test(normalizedLimit);
|
|
7057
|
+
if (!hasUsableLimit) return void 0;
|
|
7058
|
+
const deductible = firstField(fields, [/\bdeductible\b/i, /\bretention\b/i, /\bsir\b/i]);
|
|
7059
|
+
const basis = firstField(fields, [/\bbasis\b/i]);
|
|
7060
|
+
const retroactiveDate = firstField(fields, [/\bretroactive date\b/i, /\bretro date\b/i]);
|
|
7061
|
+
const page = pageNumber(span);
|
|
7062
|
+
const limitAmount = parseCurrencyAmount(limit);
|
|
7063
|
+
const deductibleAmount = deductible ? parseCurrencyAmount(deductible) : void 0;
|
|
7064
|
+
return {
|
|
7065
|
+
name,
|
|
7066
|
+
limit,
|
|
7067
|
+
...limitAmount !== void 0 ? { limitAmount } : {},
|
|
7068
|
+
...limitTypeFrom(`${name} ${limit}`) ? { limitType: limitTypeFrom(`${name} ${limit}`) } : {},
|
|
7069
|
+
limitValueType: limitValueTypeFrom(limit),
|
|
7070
|
+
...deductible && !/^nil|none$/i.test(deductible.trim()) ? { deductible } : {},
|
|
7071
|
+
...deductibleAmount !== void 0 ? { deductibleAmount } : {},
|
|
7072
|
+
...deductible ? { deductibleValueType: limitValueTypeFrom(deductible) } : {},
|
|
7073
|
+
...basis && /claims[- ]made/i.test(basis) ? { trigger: "claims_made" } : {},
|
|
7074
|
+
...retroactiveDate ? { retroactiveDate } : {},
|
|
7075
|
+
...span.formNumber ? { formNumber: span.formNumber } : {},
|
|
7076
|
+
...page ? { pageNumber: page } : {},
|
|
7077
|
+
sectionRef: span.sectionId ?? "SCHEDULE",
|
|
7078
|
+
originalContent: rowText,
|
|
7079
|
+
sourceSpanIds: [span.id],
|
|
7080
|
+
sourceTextHash: span.textHash ?? span.hash
|
|
7081
|
+
};
|
|
7082
|
+
}
|
|
7083
|
+
function coverageKey(coverage) {
|
|
7084
|
+
return [
|
|
7085
|
+
textValue(coverage.name) ?? "",
|
|
7086
|
+
textValue(coverage.limit) ?? "",
|
|
7087
|
+
textValue(coverage.limitType) ?? "",
|
|
7088
|
+
numberValue(coverage.pageNumber) ?? ""
|
|
7089
|
+
].map((part) => normalize(String(part))).join("|");
|
|
7090
|
+
}
|
|
7091
|
+
function rowMatchesExisting(row, existing) {
|
|
7092
|
+
const rowKey = coverageKey(row);
|
|
7093
|
+
const rowName = normalize(textValue(row.name) ?? "");
|
|
7094
|
+
const rowLimit = normalize(textValue(row.limit) ?? "");
|
|
7095
|
+
return existing.some((coverage) => {
|
|
7096
|
+
if (coverageKey(coverage) === rowKey) return true;
|
|
7097
|
+
const name = normalize(textValue(coverage.name) ?? "");
|
|
7098
|
+
const limit = normalize(textValue(coverage.limit) ?? "");
|
|
7099
|
+
return Boolean(rowName && rowLimit && name === rowName && limit === rowLimit);
|
|
7100
|
+
});
|
|
7101
|
+
}
|
|
7102
|
+
function recoverCoverageScheduleRows(params) {
|
|
7103
|
+
const payload = params.memory.get("coverage_limits");
|
|
7104
|
+
const existing = Array.isArray(payload?.coverages) ? payload.coverages : [];
|
|
7105
|
+
const pages = coveragePages(params.pageAssignments);
|
|
7106
|
+
const candidates = params.sourceSpans.filter((span) => sourceUnit3(span) === "table_row").filter((span) => {
|
|
7107
|
+
const page = pageNumber(span);
|
|
7108
|
+
return page !== void 0 && pages.has(page);
|
|
7109
|
+
}).map(coverageFromRow).filter((coverage) => Boolean(coverage));
|
|
7110
|
+
const recovered = [];
|
|
7111
|
+
for (const coverage of candidates) {
|
|
7112
|
+
if (rowMatchesExisting(coverage, [...existing, ...recovered])) continue;
|
|
7113
|
+
recovered.push(coverage);
|
|
7114
|
+
}
|
|
7115
|
+
if (recovered.length > 0) {
|
|
7116
|
+
params.memory.set("coverage_limits", {
|
|
7117
|
+
...payload ?? {},
|
|
7118
|
+
coverages: [...existing, ...recovered]
|
|
7119
|
+
});
|
|
7120
|
+
}
|
|
7121
|
+
return {
|
|
7122
|
+
recovered,
|
|
7123
|
+
missingSourceRows: recovered.map((coverage) => {
|
|
7124
|
+
const id = Array.isArray(coverage.sourceSpanIds) ? coverage.sourceSpanIds[0] : void 0;
|
|
7125
|
+
return params.sourceSpans.find((span) => span.id === id);
|
|
7126
|
+
}).filter((span) => Boolean(span))
|
|
7127
|
+
};
|
|
7128
|
+
}
|
|
7129
|
+
|
|
6723
7130
|
// src/extraction/focused-dispatch.ts
|
|
6724
7131
|
async function runFocusedExtractorWithFallback(params) {
|
|
6725
7132
|
const {
|
|
@@ -6883,6 +7290,20 @@ function sourcePrecedence(sectionRef) {
|
|
|
6883
7290
|
if (normalized.includes("coverage form") || normalized.includes("policy form")) return 1;
|
|
6884
7291
|
return 0;
|
|
6885
7292
|
}
|
|
7293
|
+
function looksDeductibleOnlyCoverageRow(coverage) {
|
|
7294
|
+
const name = typeof coverage.name === "string" ? coverage.name : "";
|
|
7295
|
+
const originalContent = typeof coverage.originalContent === "string" ? coverage.originalContent : "";
|
|
7296
|
+
const evidenceText = originalContent || name;
|
|
7297
|
+
const normalizedEvidence = evidenceText.toLowerCase();
|
|
7298
|
+
const normalizedName = name.toLowerCase();
|
|
7299
|
+
if (!/\b(deductible|retention|self[-\s]?insured retention|sir)\b/.test(`${normalizedName} ${normalizedEvidence}`)) return false;
|
|
7300
|
+
if (coverage.deductible || coverage.deductibleAmount !== void 0) return false;
|
|
7301
|
+
if (!coverage.limit && coverage.limitAmount === void 0) return false;
|
|
7302
|
+
if (originalContent && /\b(deductible|retention|self[-\s]?insured retention|sir)\b/.test(normalizedEvidence) && !/\b(?:sub[-\s]?limit|limit|coverage)\b/.test(normalizedEvidence)) {
|
|
7303
|
+
return true;
|
|
7304
|
+
}
|
|
7305
|
+
return /[-–—]\s*(?:enhanced\s+|standard\s+)?(?:deductible|retention|sir)\b/.test(normalizedName);
|
|
7306
|
+
}
|
|
6886
7307
|
function buildExtractionReviewReport(params) {
|
|
6887
7308
|
const { memory, reviewRounds } = params;
|
|
6888
7309
|
const deterministicIssues = [];
|
|
@@ -6908,6 +7329,29 @@ function buildExtractionReviewReport(params) {
|
|
|
6908
7329
|
addMissingSourceGroundingIssues(deterministicIssues, "definitions", "definitions", definitions, "term");
|
|
6909
7330
|
addMissingSourceGroundingIssues(deterministicIssues, "covered_reasons", "coveredReasons", coveredReasons, "name");
|
|
6910
7331
|
}
|
|
7332
|
+
if (params.sourceSpans?.length) {
|
|
7333
|
+
const tempCoveragePayload = {
|
|
7334
|
+
...memory.get("coverage_limits") ?? {},
|
|
7335
|
+
coverages: coverages.map((coverage) => ({ ...coverage }))
|
|
7336
|
+
};
|
|
7337
|
+
const tempMemory = new Map(memory);
|
|
7338
|
+
tempMemory.set("coverage_limits", tempCoveragePayload);
|
|
7339
|
+
const scheduleRecovery = recoverCoverageScheduleRows({
|
|
7340
|
+
memory: tempMemory,
|
|
7341
|
+
sourceSpans: params.sourceSpans,
|
|
7342
|
+
pageAssignments: params.pageAssignments
|
|
7343
|
+
});
|
|
7344
|
+
for (const recovered of scheduleRecovery.recovered) {
|
|
7345
|
+
deterministicIssues.push({
|
|
7346
|
+
code: "coverage_schedule_row_missing",
|
|
7347
|
+
severity: "blocking",
|
|
7348
|
+
message: `Coverage schedule row "${String(recovered.name ?? "unknown")}" is present in source table evidence but missing from extracted coverages.`,
|
|
7349
|
+
extractorName: "coverage_limits",
|
|
7350
|
+
pageNumber: typeof recovered.pageNumber === "number" ? recovered.pageNumber : void 0,
|
|
7351
|
+
itemName: typeof recovered.name === "string" ? recovered.name : void 0
|
|
7352
|
+
});
|
|
7353
|
+
}
|
|
7354
|
+
}
|
|
6911
7355
|
if (mappedDefinitions && definitions.length === 0) {
|
|
6912
7356
|
deterministicIssues.push({
|
|
6913
7357
|
code: "definitions_mapped_but_empty",
|
|
@@ -6988,6 +7432,17 @@ function buildExtractionReviewReport(params) {
|
|
|
6988
7432
|
itemName: coverage.name
|
|
6989
7433
|
});
|
|
6990
7434
|
}
|
|
7435
|
+
if (looksDeductibleOnlyCoverageRow(coverage)) {
|
|
7436
|
+
deterministicIssues.push({
|
|
7437
|
+
code: "deductible_row_as_coverage_limit",
|
|
7438
|
+
severity: "blocking",
|
|
7439
|
+
message: `Coverage "${String(coverage.name ?? "unknown")}" appears to store a deductible or retention value as a coverage limit.`,
|
|
7440
|
+
extractorName: "coverage_limits",
|
|
7441
|
+
formNumber,
|
|
7442
|
+
pageNumber: typeof coverage.pageNumber === "number" ? coverage.pageNumber : void 0,
|
|
7443
|
+
itemName: typeof coverage.name === "string" ? coverage.name : void 0
|
|
7444
|
+
});
|
|
7445
|
+
}
|
|
6991
7446
|
if (typeof coverage.pageNumber !== "number") {
|
|
6992
7447
|
deterministicIssues.push({
|
|
6993
7448
|
code: "coverage_missing_page_number",
|
|
@@ -7325,17 +7780,17 @@ var ARRAY_PATHS = [
|
|
|
7325
7780
|
{ memoryKey: "covered_reasons", arrayKeys: ["coveredReasons", "covered_reasons"] },
|
|
7326
7781
|
{ memoryKey: "declarations", arrayKeys: ["fields"] }
|
|
7327
7782
|
];
|
|
7328
|
-
function
|
|
7783
|
+
function normalize2(value) {
|
|
7329
7784
|
return value.replace(/\s+/g, " ").trim().toLowerCase();
|
|
7330
7785
|
}
|
|
7331
|
-
function
|
|
7786
|
+
function textValue2(record, ...keys) {
|
|
7332
7787
|
for (const key of keys) {
|
|
7333
7788
|
const value = record[key];
|
|
7334
7789
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
7335
7790
|
}
|
|
7336
7791
|
return void 0;
|
|
7337
7792
|
}
|
|
7338
|
-
function
|
|
7793
|
+
function numberValue2(record, ...keys) {
|
|
7339
7794
|
for (const key of keys) {
|
|
7340
7795
|
const value = record[key];
|
|
7341
7796
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
@@ -7352,37 +7807,73 @@ function pageOverlaps(recordStart, recordEnd, span) {
|
|
|
7352
7807
|
return start <= (spanEnd ?? spanStart) && end >= spanStart;
|
|
7353
7808
|
}
|
|
7354
7809
|
function formMatches(record, span) {
|
|
7355
|
-
const formNumber =
|
|
7810
|
+
const formNumber = textValue2(record, "formNumber");
|
|
7356
7811
|
if (!formNumber || !span.formNumber) return false;
|
|
7357
|
-
return
|
|
7812
|
+
return normalize2(formNumber) === normalize2(span.formNumber);
|
|
7358
7813
|
}
|
|
7359
7814
|
function textMatches(record, span) {
|
|
7360
|
-
const spanText =
|
|
7815
|
+
const spanText = normalize2(span.text);
|
|
7361
7816
|
const candidates = [
|
|
7362
|
-
|
|
7363
|
-
|
|
7364
|
-
|
|
7817
|
+
textValue2(record, "originalContent", "content", "definition", "value"),
|
|
7818
|
+
textValue2(record, "name", "title", "term", "field", "coverageName"),
|
|
7819
|
+
textValue2(record, "limit", "deductible", "premium")
|
|
7365
7820
|
].filter((value) => !!value && value.length >= 3);
|
|
7366
|
-
return candidates.some((candidate) => spanText.includes(
|
|
7821
|
+
return candidates.some((candidate) => spanText.includes(normalize2(candidate)));
|
|
7367
7822
|
}
|
|
7368
7823
|
function sourceHashFor(spans) {
|
|
7369
7824
|
return spans.map((span) => span.textHash ?? span.hash).filter(Boolean).join(":") || void 0;
|
|
7370
7825
|
}
|
|
7826
|
+
function sourceUnit4(span) {
|
|
7827
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
7828
|
+
}
|
|
7829
|
+
function hierarchyScore(span) {
|
|
7830
|
+
switch (sourceUnit4(span)) {
|
|
7831
|
+
case "table_row":
|
|
7832
|
+
return 5;
|
|
7833
|
+
case "table":
|
|
7834
|
+
return 3;
|
|
7835
|
+
case "section":
|
|
7836
|
+
case "page":
|
|
7837
|
+
return 2;
|
|
7838
|
+
case "table_cell":
|
|
7839
|
+
return -2;
|
|
7840
|
+
default:
|
|
7841
|
+
return 0;
|
|
7842
|
+
}
|
|
7843
|
+
}
|
|
7844
|
+
function parentRowId(span) {
|
|
7845
|
+
return span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
7846
|
+
}
|
|
7847
|
+
function preferParentRows(matches, sourceSpans) {
|
|
7848
|
+
if (matches.length === 0) return matches;
|
|
7849
|
+
const byId = new Map(sourceSpans.map((span) => [span.id, span]));
|
|
7850
|
+
const expanded = [];
|
|
7851
|
+
const seen = /* @__PURE__ */ new Set();
|
|
7852
|
+
for (const match of matches) {
|
|
7853
|
+
const parent = sourceUnit4(match) === "table_cell" ? byId.get(parentRowId(match) ?? "") : void 0;
|
|
7854
|
+
const preferred = parent && sourceUnit4(parent) === "table_row" ? parent : match;
|
|
7855
|
+
if (seen.has(preferred.id)) continue;
|
|
7856
|
+
seen.add(preferred.id);
|
|
7857
|
+
expanded.push(preferred);
|
|
7858
|
+
}
|
|
7859
|
+
return expanded;
|
|
7860
|
+
}
|
|
7371
7861
|
function findSourceSpansForRecord(record, sourceSpans) {
|
|
7372
7862
|
if (sourceSpans.length === 0) return [];
|
|
7373
|
-
const pageStart =
|
|
7374
|
-
const pageEnd =
|
|
7863
|
+
const pageStart = numberValue2(record, "pageNumber", "pageStart");
|
|
7864
|
+
const pageEnd = numberValue2(record, "pageNumber", "pageEnd");
|
|
7375
7865
|
const scored = sourceSpans.map((span) => {
|
|
7376
7866
|
let score = 0;
|
|
7377
7867
|
if (pageOverlaps(pageStart, pageEnd, span)) score += 4;
|
|
7378
7868
|
if (formMatches(record, span)) score += 3;
|
|
7379
7869
|
if (textMatches(record, span)) score += 2;
|
|
7870
|
+
if (score > 0) score += hierarchyScore(span);
|
|
7380
7871
|
return { span, score };
|
|
7381
7872
|
}).filter((item) => item.score >= 2).sort((left, right) => {
|
|
7382
7873
|
if (right.score !== left.score) return right.score - left.score;
|
|
7383
7874
|
return left.span.id.localeCompare(right.span.id);
|
|
7384
7875
|
});
|
|
7385
|
-
return scored.slice(0,
|
|
7876
|
+
return preferParentRows(scored.slice(0, 5).map((item) => item.span), sourceSpans).slice(0, 3);
|
|
7386
7877
|
}
|
|
7387
7878
|
function groundRecord(record, sourceSpans) {
|
|
7388
7879
|
if (Array.isArray(record.sourceSpanIds) && record.sourceSpanIds.length > 0 && record.sourceTextHash) {
|
|
@@ -8090,6 +8581,14 @@ ${pageText}`;
|
|
|
8090
8581
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8091
8582
|
}
|
|
8092
8583
|
}
|
|
8584
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
8585
|
+
memory,
|
|
8586
|
+
sourceSpans,
|
|
8587
|
+
pageAssignments
|
|
8588
|
+
});
|
|
8589
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
8590
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from table evidence`);
|
|
8591
|
+
}
|
|
8093
8592
|
const planIncludesSupplementary = tasks.some((task) => task.extractorName === "supplementary");
|
|
8094
8593
|
if (!planIncludesSupplementary && hasSupplementaryExtractionSignal(pageAssignments, formInventory, memory)) {
|
|
8095
8594
|
onProgress?.("Extracting supplementary retrieval facts...");
|
|
@@ -8186,7 +8685,8 @@ ${pageText}`;
|
|
|
8186
8685
|
memory,
|
|
8187
8686
|
pageAssignments,
|
|
8188
8687
|
reviewRounds,
|
|
8189
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
8688
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
8689
|
+
sourceSpans
|
|
8190
8690
|
});
|
|
8191
8691
|
if (shouldRunLlmReview(reviewMode, preReviewReport, sourceSpans.length > 0)) {
|
|
8192
8692
|
for (let round = 0; round < maxReviewRounds; round++) {
|
|
@@ -8255,6 +8755,14 @@ ${pageText}`;
|
|
|
8255
8755
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8256
8756
|
}
|
|
8257
8757
|
}
|
|
8758
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
8759
|
+
memory,
|
|
8760
|
+
sourceSpans,
|
|
8761
|
+
pageAssignments
|
|
8762
|
+
});
|
|
8763
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
8764
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from follow-up table evidence`);
|
|
8765
|
+
}
|
|
8258
8766
|
}
|
|
8259
8767
|
} else {
|
|
8260
8768
|
onProgress?.("Skipping LLM extraction review; deterministic checks passed.");
|
|
@@ -8264,7 +8772,8 @@ ${pageText}`;
|
|
|
8264
8772
|
memory,
|
|
8265
8773
|
pageAssignments,
|
|
8266
8774
|
reviewRounds,
|
|
8267
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
8775
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
8776
|
+
sourceSpans
|
|
8268
8777
|
});
|
|
8269
8778
|
if (reviewReport.issues.length > 0) {
|
|
8270
8779
|
await log?.(
|
|
@@ -8290,7 +8799,8 @@ ${pageText}`;
|
|
|
8290
8799
|
memory,
|
|
8291
8800
|
pageAssignments,
|
|
8292
8801
|
reviewRounds,
|
|
8293
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
8802
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
8803
|
+
sourceSpans
|
|
8294
8804
|
}));
|
|
8295
8805
|
onProgress?.("Assembling document...");
|
|
8296
8806
|
const document = assembleDocument(id, documentType, memory);
|
|
@@ -12212,6 +12722,8 @@ export {
|
|
|
12212
12722
|
SourceSpanLocationSchema,
|
|
12213
12723
|
SourceSpanRefSchema,
|
|
12214
12724
|
SourceSpanSchema,
|
|
12725
|
+
SourceSpanTableLocationSchema,
|
|
12726
|
+
SourceSpanUnitSchema,
|
|
12215
12727
|
SubAnswerSchema,
|
|
12216
12728
|
SubQuestionSchema,
|
|
12217
12729
|
SubjectivityCategorySchema,
|