@claritylabs/cl-sdk 1.3.7 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/index.d.mts +2303 -467
- package/dist/index.d.ts +2303 -467
- package/dist/index.js +919 -60
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +912 -60
- package/dist/index.mjs.map +1 -1
- package/dist/storage-sqlite.d.mts +947 -178
- package/dist/storage-sqlite.d.ts +947 -178
- package/dist/storage-sqlite.js +33 -2
- package/dist/storage-sqlite.js.map +1 -1
- package/dist/storage-sqlite.mjs +33 -2
- package/dist/storage-sqlite.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -567,14 +567,20 @@ var FormReferenceSchema = z3.object({
|
|
|
567
567
|
title: z3.string().optional(),
|
|
568
568
|
formType: z3.enum(["coverage", "endorsement", "declarations", "application", "notice", "other"]),
|
|
569
569
|
pageStart: z3.number().optional(),
|
|
570
|
-
pageEnd: z3.number().optional()
|
|
570
|
+
pageEnd: z3.number().optional(),
|
|
571
|
+
documentNodeId: z3.string().optional(),
|
|
572
|
+
sourceSpanIds: z3.array(z3.string()).optional(),
|
|
573
|
+
sourceTextHash: z3.string().optional()
|
|
571
574
|
});
|
|
572
575
|
var TaxFeeItemSchema = z3.object({
|
|
573
576
|
name: z3.string(),
|
|
574
577
|
amount: z3.string(),
|
|
575
578
|
amountValue: z3.number().optional(),
|
|
576
579
|
type: z3.enum(["tax", "fee", "surcharge", "assessment"]).optional(),
|
|
577
|
-
description: z3.string().optional()
|
|
580
|
+
description: z3.string().optional(),
|
|
581
|
+
documentNodeId: z3.string().optional(),
|
|
582
|
+
sourceSpanIds: z3.array(z3.string()).optional(),
|
|
583
|
+
sourceTextHash: z3.string().optional()
|
|
578
584
|
});
|
|
579
585
|
var RatingBasisSchema = z3.object({
|
|
580
586
|
type: RatingBasisTypeSchema,
|
|
@@ -623,11 +629,14 @@ var CoverageSchema = z4.object({
|
|
|
623
629
|
deductible: z4.string().optional(),
|
|
624
630
|
deductibleAmount: z4.number().optional(),
|
|
625
631
|
deductibleValueType: CoverageValueTypeSchema.optional(),
|
|
632
|
+
trigger: CoverageTriggerSchema.optional(),
|
|
633
|
+
retroactiveDate: z4.string().optional(),
|
|
626
634
|
formNumber: z4.string().optional(),
|
|
627
635
|
pageNumber: z4.number().optional(),
|
|
628
636
|
sectionRef: z4.string().optional(),
|
|
629
637
|
originalContent: z4.string().optional(),
|
|
630
638
|
recordId: z4.string().optional(),
|
|
639
|
+
documentNodeId: z4.string().optional(),
|
|
631
640
|
sourceSpanIds: z4.array(z4.string()).optional(),
|
|
632
641
|
sourceTextHash: z4.string().optional()
|
|
633
642
|
});
|
|
@@ -657,6 +666,7 @@ var EnrichedCoverageSchema = z4.object({
|
|
|
657
666
|
sectionRef: z4.string().optional(),
|
|
658
667
|
originalContent: z4.string().optional(),
|
|
659
668
|
recordId: z4.string().optional(),
|
|
669
|
+
documentNodeId: z4.string().optional(),
|
|
660
670
|
sourceSpanIds: z4.array(z4.string()).optional(),
|
|
661
671
|
sourceTextHash: z4.string().optional()
|
|
662
672
|
});
|
|
@@ -685,6 +695,7 @@ var EndorsementSchema = z5.object({
|
|
|
685
695
|
pageStart: z5.number(),
|
|
686
696
|
pageEnd: z5.number().optional(),
|
|
687
697
|
recordId: z5.string().optional(),
|
|
698
|
+
documentNodeId: z5.string().optional(),
|
|
688
699
|
sourceSpanIds: z5.array(z5.string()).optional(),
|
|
689
700
|
sourceTextHash: z5.string().optional()
|
|
690
701
|
});
|
|
@@ -703,6 +714,7 @@ var ExclusionSchema = z6.object({
|
|
|
703
714
|
content: z6.string(),
|
|
704
715
|
pageNumber: z6.number().optional(),
|
|
705
716
|
recordId: z6.string().optional(),
|
|
717
|
+
documentNodeId: z6.string().optional(),
|
|
706
718
|
sourceSpanIds: z6.array(z6.string()).optional(),
|
|
707
719
|
sourceTextHash: z6.string().optional()
|
|
708
720
|
});
|
|
@@ -720,6 +732,7 @@ var PolicyConditionSchema = z7.object({
|
|
|
720
732
|
keyValues: z7.array(ConditionKeyValueSchema).optional(),
|
|
721
733
|
pageNumber: z7.number().optional(),
|
|
722
734
|
recordId: z7.string().optional(),
|
|
735
|
+
documentNodeId: z7.string().optional(),
|
|
723
736
|
sourceSpanIds: z7.array(z7.string()).optional(),
|
|
724
737
|
sourceTextHash: z7.string().optional()
|
|
725
738
|
});
|
|
@@ -1296,6 +1309,7 @@ var SubsectionSchema = z16.object({
|
|
|
1296
1309
|
pageNumber: z16.number().optional(),
|
|
1297
1310
|
excerpt: z16.string().optional(),
|
|
1298
1311
|
content: z16.string().optional(),
|
|
1312
|
+
documentNodeId: z16.string().optional(),
|
|
1299
1313
|
sourceSpanIds: z16.array(z16.string()).optional(),
|
|
1300
1314
|
sourceTextHash: z16.string().optional()
|
|
1301
1315
|
});
|
|
@@ -1310,6 +1324,7 @@ var SectionSchema = z16.object({
|
|
|
1310
1324
|
content: z16.string().optional(),
|
|
1311
1325
|
subsections: z16.array(SubsectionSchema).optional(),
|
|
1312
1326
|
recordId: z16.string().optional(),
|
|
1327
|
+
documentNodeId: z16.string().optional(),
|
|
1313
1328
|
sourceSpanIds: z16.array(z16.string()).optional(),
|
|
1314
1329
|
sourceTextHash: z16.string().optional()
|
|
1315
1330
|
});
|
|
@@ -1323,13 +1338,19 @@ var UnderwritingConditionSchema = z16.object({
|
|
|
1323
1338
|
var PremiumLineSchema = z16.object({
|
|
1324
1339
|
line: z16.string(),
|
|
1325
1340
|
amount: z16.string(),
|
|
1326
|
-
amountValue: z16.number().optional()
|
|
1341
|
+
amountValue: z16.number().optional(),
|
|
1342
|
+
documentNodeId: z16.string().optional(),
|
|
1343
|
+
sourceSpanIds: z16.array(z16.string()).optional(),
|
|
1344
|
+
sourceTextHash: z16.string().optional()
|
|
1327
1345
|
});
|
|
1328
1346
|
var AuxiliaryFactSchema = z16.object({
|
|
1329
1347
|
key: z16.string(),
|
|
1330
1348
|
value: z16.string(),
|
|
1331
1349
|
subject: z16.string().optional(),
|
|
1332
|
-
context: z16.string().optional()
|
|
1350
|
+
context: z16.string().optional(),
|
|
1351
|
+
documentNodeId: z16.string().optional(),
|
|
1352
|
+
sourceSpanIds: z16.array(z16.string()).optional(),
|
|
1353
|
+
sourceTextHash: z16.string().optional()
|
|
1333
1354
|
});
|
|
1334
1355
|
var DefinitionSchema = z16.object({
|
|
1335
1356
|
term: z16.string(),
|
|
@@ -1340,6 +1361,7 @@ var DefinitionSchema = z16.object({
|
|
|
1340
1361
|
sectionRef: z16.string().optional(),
|
|
1341
1362
|
originalContent: z16.string().optional(),
|
|
1342
1363
|
recordId: z16.string().optional(),
|
|
1364
|
+
documentNodeId: z16.string().optional(),
|
|
1343
1365
|
sourceSpanIds: z16.array(z16.string()).optional(),
|
|
1344
1366
|
sourceTextHash: z16.string().optional()
|
|
1345
1367
|
});
|
|
@@ -1357,9 +1379,60 @@ var CoveredReasonSchema = z16.object({
|
|
|
1357
1379
|
sectionRef: z16.string().optional(),
|
|
1358
1380
|
originalContent: z16.string().optional(),
|
|
1359
1381
|
recordId: z16.string().optional(),
|
|
1382
|
+
documentNodeId: z16.string().optional(),
|
|
1360
1383
|
sourceSpanIds: z16.array(z16.string()).optional(),
|
|
1361
1384
|
sourceTextHash: z16.string().optional()
|
|
1362
1385
|
});
|
|
1386
|
+
var DocumentTableOfContentsEntrySchema = z16.object({
|
|
1387
|
+
title: z16.string(),
|
|
1388
|
+
level: z16.number().int().positive().optional(),
|
|
1389
|
+
pageStart: z16.number().optional(),
|
|
1390
|
+
pageEnd: z16.number().optional(),
|
|
1391
|
+
documentNodeId: z16.string().optional(),
|
|
1392
|
+
sourceSpanIds: z16.array(z16.string()).optional()
|
|
1393
|
+
});
|
|
1394
|
+
var DocumentPageMapEntrySchema = z16.object({
|
|
1395
|
+
page: z16.number().int().positive(),
|
|
1396
|
+
label: z16.string().optional(),
|
|
1397
|
+
formNumber: z16.string().optional(),
|
|
1398
|
+
formTitle: z16.string().optional(),
|
|
1399
|
+
sectionTitle: z16.string().optional(),
|
|
1400
|
+
extractorNames: z16.array(z16.string()).optional(),
|
|
1401
|
+
sourceSpanIds: z16.array(z16.string()).optional()
|
|
1402
|
+
});
|
|
1403
|
+
var DocumentAgentGuidanceSchema = z16.object({
|
|
1404
|
+
kind: z16.string(),
|
|
1405
|
+
title: z16.string(),
|
|
1406
|
+
detail: z16.string(),
|
|
1407
|
+
sourceSpanIds: z16.array(z16.string()).optional()
|
|
1408
|
+
});
|
|
1409
|
+
var DocumentMetadataSchema = z16.object({
|
|
1410
|
+
formInventory: z16.array(FormReferenceSchema).optional(),
|
|
1411
|
+
tableOfContents: z16.array(DocumentTableOfContentsEntrySchema).optional(),
|
|
1412
|
+
pageMap: z16.array(DocumentPageMapEntrySchema).optional(),
|
|
1413
|
+
agentGuidance: z16.array(DocumentAgentGuidanceSchema).optional()
|
|
1414
|
+
});
|
|
1415
|
+
var DocumentNodeSchema = z16.lazy(
|
|
1416
|
+
() => z16.object({
|
|
1417
|
+
id: z16.string(),
|
|
1418
|
+
title: z16.string(),
|
|
1419
|
+
originalTitle: z16.string().optional(),
|
|
1420
|
+
type: z16.string().optional(),
|
|
1421
|
+
label: z16.string().optional(),
|
|
1422
|
+
level: z16.number().int().positive().optional(),
|
|
1423
|
+
sectionNumber: z16.string().optional(),
|
|
1424
|
+
pageStart: z16.number().optional(),
|
|
1425
|
+
pageEnd: z16.number().optional(),
|
|
1426
|
+
formNumber: z16.string().optional(),
|
|
1427
|
+
formTitle: z16.string().optional(),
|
|
1428
|
+
excerpt: z16.string().optional(),
|
|
1429
|
+
content: z16.string().optional(),
|
|
1430
|
+
interpretationLabels: z16.array(z16.string()).optional(),
|
|
1431
|
+
sourceSpanIds: z16.array(z16.string()).optional(),
|
|
1432
|
+
sourceTextHash: z16.string().optional(),
|
|
1433
|
+
children: z16.array(DocumentNodeSchema).optional()
|
|
1434
|
+
})
|
|
1435
|
+
);
|
|
1363
1436
|
var BaseDocumentFields = {
|
|
1364
1437
|
id: z16.string(),
|
|
1365
1438
|
carrier: z16.string(),
|
|
@@ -1370,6 +1443,8 @@ var BaseDocumentFields = {
|
|
|
1370
1443
|
summary: z16.string().optional(),
|
|
1371
1444
|
policyTypes: z16.array(z16.string()).optional(),
|
|
1372
1445
|
coverages: z16.array(CoverageSchema),
|
|
1446
|
+
documentMetadata: DocumentMetadataSchema,
|
|
1447
|
+
documentOutline: z16.array(DocumentNodeSchema),
|
|
1373
1448
|
sections: z16.array(SectionSchema).optional(),
|
|
1374
1449
|
definitions: z16.array(DefinitionSchema).optional(),
|
|
1375
1450
|
coveredReasons: z16.array(CoveredReasonSchema).optional(),
|
|
@@ -1851,6 +1926,15 @@ var SourceSpanKindSchema = z20.enum([
|
|
|
1851
1926
|
"plain_text",
|
|
1852
1927
|
"structured_field"
|
|
1853
1928
|
]);
|
|
1929
|
+
var SourceSpanUnitSchema = z20.enum([
|
|
1930
|
+
"page",
|
|
1931
|
+
"section",
|
|
1932
|
+
"table",
|
|
1933
|
+
"table_row",
|
|
1934
|
+
"table_cell",
|
|
1935
|
+
"key_value",
|
|
1936
|
+
"text"
|
|
1937
|
+
]);
|
|
1854
1938
|
var SourceKindSchema = z20.enum([
|
|
1855
1939
|
"policy_pdf",
|
|
1856
1940
|
"application_pdf",
|
|
@@ -1875,6 +1959,15 @@ var SourceSpanLocationSchema = z20.object({
|
|
|
1875
1959
|
lineEnd: z20.number().int().positive().optional(),
|
|
1876
1960
|
fieldPath: z20.string().optional()
|
|
1877
1961
|
});
|
|
1962
|
+
var SourceSpanTableLocationSchema = z20.object({
|
|
1963
|
+
tableId: z20.string().optional(),
|
|
1964
|
+
rowIndex: z20.number().int().nonnegative().optional(),
|
|
1965
|
+
columnIndex: z20.number().int().nonnegative().optional(),
|
|
1966
|
+
columnName: z20.string().optional(),
|
|
1967
|
+
rowSpanId: z20.string().optional(),
|
|
1968
|
+
tableSpanId: z20.string().optional(),
|
|
1969
|
+
isHeader: z20.boolean().optional()
|
|
1970
|
+
});
|
|
1878
1971
|
var SourceSpanSchema = z20.object({
|
|
1879
1972
|
id: z20.string().min(1),
|
|
1880
1973
|
documentId: z20.string().min(1),
|
|
@@ -1888,6 +1981,9 @@ var SourceSpanSchema = z20.object({
|
|
|
1888
1981
|
pageEnd: z20.number().int().positive().optional(),
|
|
1889
1982
|
sectionId: z20.string().optional(),
|
|
1890
1983
|
formNumber: z20.string().optional(),
|
|
1984
|
+
sourceUnit: SourceSpanUnitSchema.optional(),
|
|
1985
|
+
parentSpanId: z20.string().optional(),
|
|
1986
|
+
table: SourceSpanTableLocationSchema.optional(),
|
|
1891
1987
|
bbox: z20.array(SourceSpanBBoxSchema).optional(),
|
|
1892
1988
|
location: SourceSpanLocationSchema.optional(),
|
|
1893
1989
|
metadata: z20.record(z20.string(), z20.string()).optional()
|
|
@@ -2006,6 +2102,9 @@ function buildSourceSpan(input, localIndex = 0) {
|
|
|
2006
2102
|
pageEnd: input.pageEnd,
|
|
2007
2103
|
sectionId: input.sectionId,
|
|
2008
2104
|
formNumber: input.formNumber,
|
|
2105
|
+
sourceUnit: input.sourceUnit,
|
|
2106
|
+
parentSpanId: input.parentSpanId,
|
|
2107
|
+
table: input.table,
|
|
2009
2108
|
location: {
|
|
2010
2109
|
page: input.pageStart === input.pageEnd ? input.pageStart : void 0,
|
|
2011
2110
|
startPage: input.pageStart,
|
|
@@ -2026,7 +2125,11 @@ function buildPageSourceSpans(pages) {
|
|
|
2026
2125
|
pageEnd: page.pageNumber,
|
|
2027
2126
|
sectionId: page.sectionId,
|
|
2028
2127
|
formNumber: page.formNumber,
|
|
2029
|
-
|
|
2128
|
+
sourceUnit: "page",
|
|
2129
|
+
metadata: {
|
|
2130
|
+
...page.metadata ?? {},
|
|
2131
|
+
sourceUnit: page.metadata?.sourceUnit ?? "page"
|
|
2132
|
+
}
|
|
2030
2133
|
},
|
|
2031
2134
|
index
|
|
2032
2135
|
)
|
|
@@ -2048,6 +2151,7 @@ function buildSectionSourceSpans(pages, options = {}) {
|
|
|
2048
2151
|
pageEnd: page.pageNumber,
|
|
2049
2152
|
sectionId: section.title,
|
|
2050
2153
|
formNumber: inferFormNumber(section.text),
|
|
2154
|
+
sourceUnit: "section",
|
|
2051
2155
|
metadata: {
|
|
2052
2156
|
...page.metadata ?? {},
|
|
2053
2157
|
sourceUnit: "section_candidate"
|
|
@@ -2080,6 +2184,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2080
2184
|
const chunks = [];
|
|
2081
2185
|
let current = [];
|
|
2082
2186
|
let currentLength = 0;
|
|
2187
|
+
const spansForChunking = filterChunkableSourceSpans(spans);
|
|
2083
2188
|
const flush = () => {
|
|
2084
2189
|
if (current.length === 0) return;
|
|
2085
2190
|
const text = current.map((span) => span.text).join("\n\n");
|
|
@@ -2103,7 +2208,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2103
2208
|
current = [];
|
|
2104
2209
|
currentLength = 0;
|
|
2105
2210
|
};
|
|
2106
|
-
for (const span of
|
|
2211
|
+
for (const span of spansForChunking) {
|
|
2107
2212
|
const nextLength = currentLength + span.text.length + (current.length > 0 ? 2 : 0);
|
|
2108
2213
|
if (current.length > 0 && nextLength > maxChars) {
|
|
2109
2214
|
flush();
|
|
@@ -2114,6 +2219,20 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2114
2219
|
flush();
|
|
2115
2220
|
return chunks;
|
|
2116
2221
|
}
|
|
2222
|
+
function sourceUnit(span) {
|
|
2223
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
2224
|
+
}
|
|
2225
|
+
function filterChunkableSourceSpans(spans) {
|
|
2226
|
+
const rowIds = new Set(
|
|
2227
|
+
spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
|
|
2228
|
+
);
|
|
2229
|
+
if (rowIds.size === 0) return spans;
|
|
2230
|
+
return spans.filter((span) => {
|
|
2231
|
+
if (sourceUnit(span) !== "table_cell") return true;
|
|
2232
|
+
const rowId = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
2233
|
+
return !rowId || !rowIds.has(rowId);
|
|
2234
|
+
});
|
|
2235
|
+
}
|
|
2117
2236
|
function splitPageIntoSections(text, headingPattern, minSectionChars) {
|
|
2118
2237
|
const lines = text.split(/\r?\n/);
|
|
2119
2238
|
const sections = [];
|
|
@@ -2506,36 +2625,18 @@ function normalizeDoclingDocument(document, options) {
|
|
|
2506
2625
|
pageTexts.set(page, appendText(pageTexts.get(page), unit.text));
|
|
2507
2626
|
}
|
|
2508
2627
|
const fullText = Array.from({ length: pageCount }, (_, index) => {
|
|
2509
|
-
const
|
|
2510
|
-
const text = pageTexts.get(
|
|
2511
|
-
return text ? `Page ${
|
|
2628
|
+
const pageNumber2 = index + 1;
|
|
2629
|
+
const text = pageTexts.get(pageNumber2)?.trim();
|
|
2630
|
+
return text ? `Page ${pageNumber2}
|
|
2512
2631
|
${text}` : "";
|
|
2513
2632
|
}).filter(Boolean).join("\n\n");
|
|
2514
2633
|
const sourceKind = options.sourceKind ?? "policy_pdf";
|
|
2515
|
-
const sourceSpans = units.
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
pageStart: unit.pageStart,
|
|
2522
|
-
pageEnd: unit.pageEnd,
|
|
2523
|
-
sectionId: unit.label,
|
|
2524
|
-
metadata: {
|
|
2525
|
-
sourceSystem: "docling",
|
|
2526
|
-
sourceUnit: "docling_item",
|
|
2527
|
-
doclingRef: unit.ref,
|
|
2528
|
-
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2529
|
-
}
|
|
2530
|
-
},
|
|
2531
|
-
index
|
|
2532
|
-
);
|
|
2533
|
-
return {
|
|
2534
|
-
...span,
|
|
2535
|
-
kind: "plain_text",
|
|
2536
|
-
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2537
|
-
};
|
|
2538
|
-
});
|
|
2634
|
+
const sourceSpans = units.flatMap(
|
|
2635
|
+
(unit, index) => buildSourceSpansForUnit(unit, index, {
|
|
2636
|
+
documentId: options.documentId,
|
|
2637
|
+
sourceKind
|
|
2638
|
+
})
|
|
2639
|
+
);
|
|
2539
2640
|
return {
|
|
2540
2641
|
pageCount,
|
|
2541
2642
|
fullText,
|
|
@@ -2573,6 +2674,11 @@ function mergeSourceSpans(spans) {
|
|
|
2573
2674
|
span.pageStart ?? span.location?.startPage ?? span.location?.page ?? "na",
|
|
2574
2675
|
span.pageEnd ?? span.location?.endPage ?? span.pageStart ?? "na",
|
|
2575
2676
|
span.sectionId ?? span.location?.fieldPath ?? "na",
|
|
2677
|
+
span.sourceUnit ?? span.metadata?.sourceUnit ?? "na",
|
|
2678
|
+
span.parentSpanId ?? "na",
|
|
2679
|
+
span.table?.tableId ?? span.metadata?.tableId ?? "na",
|
|
2680
|
+
span.table?.rowIndex ?? span.metadata?.rowIndex ?? "na",
|
|
2681
|
+
span.table?.columnIndex ?? span.metadata?.columnIndex ?? "na",
|
|
2576
2682
|
span.textHash ?? sourceSpanTextHash(span.text)
|
|
2577
2683
|
].join(":");
|
|
2578
2684
|
if (seen.has(key)) continue;
|
|
@@ -2635,6 +2741,7 @@ function getOrderedBodyRefs(document, itemMap) {
|
|
|
2635
2741
|
function normalizeItem(ref, item) {
|
|
2636
2742
|
const text = getItemText(item).trim();
|
|
2637
2743
|
if (!text) return void 0;
|
|
2744
|
+
const table = getItemTable(item);
|
|
2638
2745
|
const pages = (item.prov ?? []).map((prov) => getPageNumber(prov)).filter((page) => typeof page === "number" && page > 0);
|
|
2639
2746
|
const pageStart = pages.length ? Math.min(...pages) : void 0;
|
|
2640
2747
|
const pageEnd = pages.length ? Math.max(...pages) : pageStart;
|
|
@@ -2645,17 +2752,143 @@ function normalizeItem(ref, item) {
|
|
|
2645
2752
|
text,
|
|
2646
2753
|
pageStart,
|
|
2647
2754
|
pageEnd,
|
|
2648
|
-
bboxes: bboxes.length ? bboxes : void 0
|
|
2755
|
+
bboxes: bboxes.length ? bboxes : void 0,
|
|
2756
|
+
table
|
|
2649
2757
|
};
|
|
2650
2758
|
}
|
|
2759
|
+
function buildSourceSpansForUnit(unit, index, options) {
|
|
2760
|
+
const baseMetadata = {
|
|
2761
|
+
sourceSystem: "docling",
|
|
2762
|
+
sourceUnit: unit.table ? "table" : "docling_item",
|
|
2763
|
+
doclingRef: unit.ref,
|
|
2764
|
+
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2765
|
+
};
|
|
2766
|
+
const tableId = unit.table ? `${unit.ref}:table` : void 0;
|
|
2767
|
+
const tableSpan = withDoclingKind(buildSourceSpan(
|
|
2768
|
+
{
|
|
2769
|
+
documentId: options.documentId,
|
|
2770
|
+
sourceKind: options.sourceKind,
|
|
2771
|
+
text: unit.text,
|
|
2772
|
+
pageStart: unit.pageStart,
|
|
2773
|
+
pageEnd: unit.pageEnd,
|
|
2774
|
+
sectionId: unit.label,
|
|
2775
|
+
sourceUnit: unit.table ? "table" : labelToSourceUnit(unit.label),
|
|
2776
|
+
table: tableId ? { tableId } : void 0,
|
|
2777
|
+
metadata: baseMetadata
|
|
2778
|
+
},
|
|
2779
|
+
index * 1e3
|
|
2780
|
+
), unit);
|
|
2781
|
+
if (!unit.table) return [tableSpan];
|
|
2782
|
+
const spans = [tableSpan];
|
|
2783
|
+
const table = unit.table;
|
|
2784
|
+
for (let rowIndex = 0; rowIndex < table.rows.length; rowIndex += 1) {
|
|
2785
|
+
const row = table.rows[rowIndex];
|
|
2786
|
+
const isHeader = rowIndex === 0 && table.headers.length > 0;
|
|
2787
|
+
const rowText = tableRowText(table, rowIndex);
|
|
2788
|
+
if (!rowText) continue;
|
|
2789
|
+
const rowSpan = withDoclingKind(buildSourceSpan(
|
|
2790
|
+
{
|
|
2791
|
+
documentId: options.documentId,
|
|
2792
|
+
sourceKind: options.sourceKind,
|
|
2793
|
+
text: rowText,
|
|
2794
|
+
pageStart: unit.pageStart,
|
|
2795
|
+
pageEnd: unit.pageEnd,
|
|
2796
|
+
sectionId: unit.label,
|
|
2797
|
+
sourceUnit: "table_row",
|
|
2798
|
+
parentSpanId: tableSpan.id,
|
|
2799
|
+
table: {
|
|
2800
|
+
tableId,
|
|
2801
|
+
tableSpanId: tableSpan.id,
|
|
2802
|
+
rowIndex,
|
|
2803
|
+
isHeader
|
|
2804
|
+
},
|
|
2805
|
+
metadata: {
|
|
2806
|
+
...baseMetadata,
|
|
2807
|
+
sourceUnit: "table_row",
|
|
2808
|
+
tableId: tableId ?? "",
|
|
2809
|
+
tableSpanId: tableSpan.id,
|
|
2810
|
+
rowIndex: String(rowIndex),
|
|
2811
|
+
isHeader: String(isHeader)
|
|
2812
|
+
}
|
|
2813
|
+
},
|
|
2814
|
+
index * 1e3 + 1 + rowIndex
|
|
2815
|
+
), unit);
|
|
2816
|
+
spans.push(rowSpan);
|
|
2817
|
+
for (let columnIndex = 0; columnIndex < row.length; columnIndex += 1) {
|
|
2818
|
+
const text = row[columnIndex]?.trim();
|
|
2819
|
+
if (!text) continue;
|
|
2820
|
+
const columnName = table.headers[columnIndex]?.trim() || void 0;
|
|
2821
|
+
const cellSpan = withDoclingKind(buildSourceSpan(
|
|
2822
|
+
{
|
|
2823
|
+
documentId: options.documentId,
|
|
2824
|
+
sourceKind: options.sourceKind,
|
|
2825
|
+
text,
|
|
2826
|
+
pageStart: unit.pageStart,
|
|
2827
|
+
pageEnd: unit.pageEnd,
|
|
2828
|
+
sectionId: unit.label,
|
|
2829
|
+
sourceUnit: "table_cell",
|
|
2830
|
+
parentSpanId: rowSpan.id,
|
|
2831
|
+
table: {
|
|
2832
|
+
tableId,
|
|
2833
|
+
tableSpanId: tableSpan.id,
|
|
2834
|
+
rowSpanId: rowSpan.id,
|
|
2835
|
+
rowIndex,
|
|
2836
|
+
columnIndex,
|
|
2837
|
+
columnName,
|
|
2838
|
+
isHeader
|
|
2839
|
+
},
|
|
2840
|
+
metadata: {
|
|
2841
|
+
...baseMetadata,
|
|
2842
|
+
sourceUnit: "table_cell",
|
|
2843
|
+
tableId: tableId ?? "",
|
|
2844
|
+
tableSpanId: tableSpan.id,
|
|
2845
|
+
rowSpanId: rowSpan.id,
|
|
2846
|
+
rowIndex: String(rowIndex),
|
|
2847
|
+
columnIndex: String(columnIndex),
|
|
2848
|
+
...columnName ? { columnName } : {},
|
|
2849
|
+
isHeader: String(isHeader)
|
|
2850
|
+
}
|
|
2851
|
+
},
|
|
2852
|
+
index * 1e3 + 100 + rowIndex * 100 + columnIndex
|
|
2853
|
+
), unit);
|
|
2854
|
+
spans.push(cellSpan);
|
|
2855
|
+
}
|
|
2856
|
+
}
|
|
2857
|
+
return spans;
|
|
2858
|
+
}
|
|
2859
|
+
function withDoclingKind(span, unit) {
|
|
2860
|
+
return {
|
|
2861
|
+
...span,
|
|
2862
|
+
kind: "plain_text",
|
|
2863
|
+
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2864
|
+
};
|
|
2865
|
+
}
|
|
2866
|
+
function labelToSourceUnit(label) {
|
|
2867
|
+
const normalized = label?.toLowerCase() ?? "";
|
|
2868
|
+
if (normalized.includes("table")) return "table";
|
|
2869
|
+
if (normalized.includes("key")) return "key_value";
|
|
2870
|
+
if (normalized.includes("section") || normalized.includes("header")) return "section";
|
|
2871
|
+
return "text";
|
|
2872
|
+
}
|
|
2873
|
+
function tableRowText(table, rowIndex) {
|
|
2874
|
+
const row = table.rows[rowIndex] ?? [];
|
|
2875
|
+
const headers = table.headers;
|
|
2876
|
+
if (rowIndex === 0 && headers.length > 0) return row.filter(Boolean).join(" | ");
|
|
2877
|
+
return row.map((value, columnIndex) => {
|
|
2878
|
+
const trimmed = value.trim();
|
|
2879
|
+
if (!trimmed) return "";
|
|
2880
|
+
const header = headers[columnIndex]?.trim();
|
|
2881
|
+
return header ? `${header}: ${trimmed}` : trimmed;
|
|
2882
|
+
}).filter(Boolean).join(" | ");
|
|
2883
|
+
}
|
|
2651
2884
|
function getItemText(item) {
|
|
2652
2885
|
if (typeof item.text === "string" && item.text.trim()) return item.text;
|
|
2653
2886
|
if (typeof item.orig === "string" && item.orig.trim()) return item.orig;
|
|
2654
|
-
const table =
|
|
2655
|
-
if (table) return table;
|
|
2887
|
+
const table = parseTableData(item.data);
|
|
2888
|
+
if (table) return table.markdown;
|
|
2656
2889
|
return "";
|
|
2657
2890
|
}
|
|
2658
|
-
function
|
|
2891
|
+
function parseTableData(data) {
|
|
2659
2892
|
const record = asRecord(data);
|
|
2660
2893
|
const cells = Array.isArray(record?.table_cells) ? record.table_cells : Array.isArray(record?.tableCells) ? record.tableCells : void 0;
|
|
2661
2894
|
if (!cells) return void 0;
|
|
@@ -2671,10 +2904,20 @@ function tableToMarkdown(data) {
|
|
|
2671
2904
|
for (const cell of parsedCells) {
|
|
2672
2905
|
rows[cell.row][cell.col] = cell.text;
|
|
2673
2906
|
}
|
|
2674
|
-
if (rows.length === 1)
|
|
2907
|
+
if (rows.length === 1) {
|
|
2908
|
+
const markdown2 = rows[0].filter(Boolean).join(" | ");
|
|
2909
|
+
return { markdown: markdown2, headers: [], rows, cells: parsedCells };
|
|
2910
|
+
}
|
|
2675
2911
|
const header = rows[0];
|
|
2676
2912
|
const separator = header.map(() => "---");
|
|
2677
|
-
|
|
2913
|
+
const markdown = [header, separator, ...rows.slice(1)].map((row) => `| ${row.map((value) => value.trim()).join(" | ")} |`).join("\n");
|
|
2914
|
+
return { markdown, headers: header, rows, cells: parsedCells };
|
|
2915
|
+
}
|
|
2916
|
+
function getItemTable(item) {
|
|
2917
|
+
if (typeof item.text === "string" && item.text.trim() || typeof item.orig === "string" && item.orig.trim()) {
|
|
2918
|
+
return void 0;
|
|
2919
|
+
}
|
|
2920
|
+
return parseTableData(item.data);
|
|
2678
2921
|
}
|
|
2679
2922
|
function inferPageCount(document, units) {
|
|
2680
2923
|
const pages = document.pages;
|
|
@@ -2751,8 +2994,18 @@ function buildSourceContext(spans, maxChars = 12e3) {
|
|
|
2751
2994
|
if (spans.length === 0) return "";
|
|
2752
2995
|
const lines = [];
|
|
2753
2996
|
let length = 0;
|
|
2754
|
-
for (const span of spans) {
|
|
2755
|
-
const
|
|
2997
|
+
for (const span of orderSourceSpansForContext(spans)) {
|
|
2998
|
+
const unit = sourceUnit2(span);
|
|
2999
|
+
const table = span.table;
|
|
3000
|
+
const tableContext = [
|
|
3001
|
+
unit ? ` unit:${unit}` : "",
|
|
3002
|
+
table?.tableId ? ` table:${table.tableId}` : "",
|
|
3003
|
+
table?.rowIndex != null ? ` row:${table.rowIndex}` : "",
|
|
3004
|
+
table?.columnIndex != null ? ` col:${table.columnIndex}` : "",
|
|
3005
|
+
table?.columnName ? ` column:${table.columnName}` : "",
|
|
3006
|
+
span.parentSpanId ? ` parent:${span.parentSpanId}` : ""
|
|
3007
|
+
].join("");
|
|
3008
|
+
const header = `[sourceSpan:${span.id}${span.pageStart ? ` page:${span.pageStart}${span.pageEnd && span.pageEnd !== span.pageStart ? `-${span.pageEnd}` : ""}` : ""}${span.sectionId ? ` section:${span.sectionId}` : ""}${span.formNumber ? ` form:${span.formNumber}` : ""}${tableContext}]`;
|
|
2756
3009
|
const text = `${header}
|
|
2757
3010
|
${span.text}`;
|
|
2758
3011
|
if (length + text.length > maxChars && lines.length > 0) break;
|
|
@@ -2766,6 +3019,49 @@ ${lines.join("\n\n")}
|
|
|
2766
3019
|
|
|
2767
3020
|
Use sourceSpan IDs when grounding extracted contractual values.`;
|
|
2768
3021
|
}
|
|
3022
|
+
function sourceUnit2(span) {
|
|
3023
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
3024
|
+
}
|
|
3025
|
+
function sourceContextRank(span) {
|
|
3026
|
+
switch (sourceUnit2(span)) {
|
|
3027
|
+
case "table_row":
|
|
3028
|
+
return 0;
|
|
3029
|
+
case "table":
|
|
3030
|
+
return 1;
|
|
3031
|
+
case "section":
|
|
3032
|
+
return 2;
|
|
3033
|
+
case "page":
|
|
3034
|
+
return 3;
|
|
3035
|
+
case "key_value":
|
|
3036
|
+
return 4;
|
|
3037
|
+
case "table_cell":
|
|
3038
|
+
return 8;
|
|
3039
|
+
default:
|
|
3040
|
+
return 5;
|
|
3041
|
+
}
|
|
3042
|
+
}
|
|
3043
|
+
function orderSourceSpansForContext(spans) {
|
|
3044
|
+
const parentRows = new Set(
|
|
3045
|
+
spans.filter((span) => sourceUnit2(span) === "table_row").map((span) => span.id)
|
|
3046
|
+
);
|
|
3047
|
+
const filtered = spans.filter((span) => {
|
|
3048
|
+
if (sourceUnit2(span) !== "table_cell") return true;
|
|
3049
|
+
const parent = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
3050
|
+
return !parent || !parentRows.has(parent);
|
|
3051
|
+
});
|
|
3052
|
+
return [...filtered].sort((left, right) => {
|
|
3053
|
+
const leftPage = left.pageStart ?? left.location?.startPage ?? left.location?.page ?? 0;
|
|
3054
|
+
const rightPage = right.pageStart ?? right.location?.startPage ?? right.location?.page ?? 0;
|
|
3055
|
+
if (leftPage !== rightPage) return leftPage - rightPage;
|
|
3056
|
+
const leftRank = sourceContextRank(left);
|
|
3057
|
+
const rightRank = sourceContextRank(right);
|
|
3058
|
+
if (leftRank !== rightRank) return leftRank - rightRank;
|
|
3059
|
+
const leftRow = left.table?.rowIndex ?? Number(left.metadata?.rowIndex ?? Number.NaN);
|
|
3060
|
+
const rightRow = right.table?.rowIndex ?? Number(right.metadata?.rowIndex ?? Number.NaN);
|
|
3061
|
+
if (Number.isFinite(leftRow) && Number.isFinite(rightRow) && leftRow !== rightRow) return leftRow - rightRow;
|
|
3062
|
+
return left.id.localeCompare(right.id);
|
|
3063
|
+
});
|
|
3064
|
+
}
|
|
2769
3065
|
async function runExtractor(params) {
|
|
2770
3066
|
const {
|
|
2771
3067
|
name,
|
|
@@ -3056,6 +3352,8 @@ function assembleDocument(documentId, documentType, memory) {
|
|
|
3056
3352
|
carrier: readRecordValue(carrier, "carrierName") ?? "Unknown",
|
|
3057
3353
|
insuredName: readRecordValue(insured, "insuredName") ?? "Unknown",
|
|
3058
3354
|
coverages: coverageRecords,
|
|
3355
|
+
documentMetadata: {},
|
|
3356
|
+
documentOutline: [],
|
|
3059
3357
|
policyTypes: readRecordValue(classify, "policyTypes"),
|
|
3060
3358
|
...sanitizeNulls(carrier ?? {}),
|
|
3061
3359
|
...sanitizeNulls(insured ?? {}),
|
|
@@ -3107,6 +3405,247 @@ function assembleDocument(documentId, documentType, memory) {
|
|
|
3107
3405
|
return doc;
|
|
3108
3406
|
}
|
|
3109
3407
|
|
|
3408
|
+
// src/extraction/document-structure.ts
|
|
3409
|
+
var FACT_ARRAY_PATHS = [
|
|
3410
|
+
"coverages",
|
|
3411
|
+
"enrichedCoverages",
|
|
3412
|
+
"endorsements",
|
|
3413
|
+
"exclusions",
|
|
3414
|
+
"conditions",
|
|
3415
|
+
"definitions",
|
|
3416
|
+
"coveredReasons",
|
|
3417
|
+
"taxesAndFees",
|
|
3418
|
+
"premiumBreakdown",
|
|
3419
|
+
"supplementaryFacts"
|
|
3420
|
+
];
|
|
3421
|
+
function slugPart(value) {
|
|
3422
|
+
const text = String(value ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
3423
|
+
return text || "node";
|
|
3424
|
+
}
|
|
3425
|
+
function numberValue(record, ...keys) {
|
|
3426
|
+
for (const key of keys) {
|
|
3427
|
+
const value = record[key];
|
|
3428
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
3429
|
+
}
|
|
3430
|
+
return void 0;
|
|
3431
|
+
}
|
|
3432
|
+
function stringValue2(record, ...keys) {
|
|
3433
|
+
for (const key of keys) {
|
|
3434
|
+
const value = record[key];
|
|
3435
|
+
if (typeof value === "string" && value.trim()) return value.trim();
|
|
3436
|
+
}
|
|
3437
|
+
return void 0;
|
|
3438
|
+
}
|
|
3439
|
+
function stringArray(value) {
|
|
3440
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.length > 0) : [];
|
|
3441
|
+
}
|
|
3442
|
+
function sourceUnit3(span) {
|
|
3443
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
3444
|
+
}
|
|
3445
|
+
function spanPage(span) {
|
|
3446
|
+
return span.pageStart ?? span.location?.page ?? span.location?.startPage;
|
|
3447
|
+
}
|
|
3448
|
+
function spanPageEnd(span) {
|
|
3449
|
+
return span.pageEnd ?? span.location?.endPage ?? spanPage(span);
|
|
3450
|
+
}
|
|
3451
|
+
function sourceSpansForPage(sourceSpans, page) {
|
|
3452
|
+
return sourceSpans.filter((span) => spanPage(span) === page && sourceUnit3(span) === "page").map((span) => span.id);
|
|
3453
|
+
}
|
|
3454
|
+
function nodePageOverlaps(node, record) {
|
|
3455
|
+
const recordStart = numberValue(record, "pageNumber", "pageStart", "resolvedFromPage");
|
|
3456
|
+
const recordEnd = numberValue(record, "pageNumber", "pageEnd", "resolvedFromPage") ?? recordStart;
|
|
3457
|
+
if (!recordStart) return false;
|
|
3458
|
+
const nodeStart = node.pageStart ?? node.pageEnd;
|
|
3459
|
+
const nodeEnd = node.pageEnd ?? node.pageStart;
|
|
3460
|
+
if (!nodeStart) return false;
|
|
3461
|
+
return recordStart <= (nodeEnd ?? nodeStart) && (recordEnd ?? recordStart) >= nodeStart;
|
|
3462
|
+
}
|
|
3463
|
+
function nodeFormMatches(node, record) {
|
|
3464
|
+
const formNumber = stringValue2(record, "formNumber");
|
|
3465
|
+
return Boolean(formNumber && node.formNumber && formNumber.toLowerCase() === node.formNumber.toLowerCase());
|
|
3466
|
+
}
|
|
3467
|
+
function nodeSourceOverlaps(node, record) {
|
|
3468
|
+
const nodeSourceIds = new Set(node.sourceSpanIds ?? []);
|
|
3469
|
+
if (nodeSourceIds.size === 0) return false;
|
|
3470
|
+
return stringArray(record.sourceSpanIds).some((id) => nodeSourceIds.has(id));
|
|
3471
|
+
}
|
|
3472
|
+
function findBestNode(nodes, record) {
|
|
3473
|
+
let best;
|
|
3474
|
+
const visit = (node) => {
|
|
3475
|
+
let score = 0;
|
|
3476
|
+
if (nodeSourceOverlaps(node, record)) score += 8;
|
|
3477
|
+
if (nodeFormMatches(node, record)) score += 4;
|
|
3478
|
+
if (nodePageOverlaps(node, record)) score += 3;
|
|
3479
|
+
if (score > 0 && (!best || score > best.score)) {
|
|
3480
|
+
best = { node, score };
|
|
3481
|
+
}
|
|
3482
|
+
for (const child of node.children ?? []) visit(child);
|
|
3483
|
+
};
|
|
3484
|
+
for (const node of nodes) visit(node);
|
|
3485
|
+
return best?.node;
|
|
3486
|
+
}
|
|
3487
|
+
function attachDocumentNodeIds(document, nodes) {
|
|
3488
|
+
const doc = document;
|
|
3489
|
+
for (const path of FACT_ARRAY_PATHS) {
|
|
3490
|
+
const value = doc[path];
|
|
3491
|
+
if (!Array.isArray(value)) continue;
|
|
3492
|
+
doc[path] = value.map((item) => {
|
|
3493
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) return item;
|
|
3494
|
+
const record = item;
|
|
3495
|
+
if (typeof record.documentNodeId === "string" && record.documentNodeId.length > 0) return record;
|
|
3496
|
+
const node = findBestNode(nodes, record);
|
|
3497
|
+
return node ? { ...record, documentNodeId: node.id } : record;
|
|
3498
|
+
});
|
|
3499
|
+
}
|
|
3500
|
+
}
|
|
3501
|
+
function buildNodeFromSection(section, index) {
|
|
3502
|
+
const title = stringValue2(section, "title", "name", "sectionRef") ?? `Section ${index + 1}`;
|
|
3503
|
+
const id = stringValue2(section, "documentNodeId", "recordId") ?? `section:${index}:${slugPart(title)}`;
|
|
3504
|
+
const pageStart = numberValue(section, "pageStart", "pageNumber");
|
|
3505
|
+
const pageEnd = numberValue(section, "pageEnd", "pageNumber") ?? pageStart;
|
|
3506
|
+
const type = stringValue2(section, "type");
|
|
3507
|
+
const coverageType = stringValue2(section, "coverageType");
|
|
3508
|
+
const sourceSpanIds = stringArray(section.sourceSpanIds);
|
|
3509
|
+
const children = Array.isArray(section.subsections) ? section.subsections.filter((item) => Boolean(item) && typeof item === "object" && !Array.isArray(item)).map((subsection, childIndex) => {
|
|
3510
|
+
const childTitle = stringValue2(subsection, "title", "name") ?? `${title} ${childIndex + 1}`;
|
|
3511
|
+
const childId = stringValue2(subsection, "documentNodeId", "recordId") ?? `${id}:subsection:${childIndex}:${slugPart(childTitle)}`;
|
|
3512
|
+
return {
|
|
3513
|
+
id: childId,
|
|
3514
|
+
title: childTitle,
|
|
3515
|
+
originalTitle: childTitle,
|
|
3516
|
+
type,
|
|
3517
|
+
label: type,
|
|
3518
|
+
level: 2,
|
|
3519
|
+
sectionNumber: stringValue2(subsection, "sectionNumber"),
|
|
3520
|
+
pageStart: numberValue(subsection, "pageNumber") ?? pageStart,
|
|
3521
|
+
pageEnd: numberValue(subsection, "pageNumber") ?? pageStart,
|
|
3522
|
+
excerpt: stringValue2(subsection, "excerpt"),
|
|
3523
|
+
content: stringValue2(subsection, "content"),
|
|
3524
|
+
sourceSpanIds: stringArray(subsection.sourceSpanIds),
|
|
3525
|
+
sourceTextHash: stringValue2(subsection, "sourceTextHash")
|
|
3526
|
+
};
|
|
3527
|
+
}) : void 0;
|
|
3528
|
+
return {
|
|
3529
|
+
id,
|
|
3530
|
+
title,
|
|
3531
|
+
originalTitle: title,
|
|
3532
|
+
type,
|
|
3533
|
+
label: type,
|
|
3534
|
+
level: 1,
|
|
3535
|
+
sectionNumber: stringValue2(section, "sectionNumber"),
|
|
3536
|
+
pageStart,
|
|
3537
|
+
pageEnd,
|
|
3538
|
+
formNumber: stringValue2(section, "formNumber"),
|
|
3539
|
+
formTitle: stringValue2(section, "formTitle"),
|
|
3540
|
+
excerpt: stringValue2(section, "excerpt"),
|
|
3541
|
+
content: stringValue2(section, "content"),
|
|
3542
|
+
interpretationLabels: [type, coverageType].filter((value) => Boolean(value)),
|
|
3543
|
+
sourceSpanIds: sourceSpanIds.length > 0 ? sourceSpanIds : void 0,
|
|
3544
|
+
sourceTextHash: stringValue2(section, "sourceTextHash"),
|
|
3545
|
+
children: children && children.length > 0 ? children : void 0
|
|
3546
|
+
};
|
|
3547
|
+
}
|
|
3548
|
+
function buildNodesFromSourceSpans(sourceSpans) {
|
|
3549
|
+
const candidates = sourceSpans.filter((span) => {
|
|
3550
|
+
const unit = sourceUnit3(span);
|
|
3551
|
+
return unit === "section" || unit === "section_candidate" || unit === "page";
|
|
3552
|
+
});
|
|
3553
|
+
return candidates.sort((left, right) => (spanPage(left) ?? 0) - (spanPage(right) ?? 0) || left.id.localeCompare(right.id)).map((span, index) => {
|
|
3554
|
+
const title = span.sectionId ?? span.formNumber ?? (sourceUnit3(span) === "page" && spanPage(span) ? `Page ${spanPage(span)}` : `Source unit ${index + 1}`);
|
|
3555
|
+
return {
|
|
3556
|
+
id: `source:${index}:${slugPart(span.id)}`,
|
|
3557
|
+
title,
|
|
3558
|
+
originalTitle: title,
|
|
3559
|
+
type: sourceUnit3(span),
|
|
3560
|
+
label: sourceUnit3(span),
|
|
3561
|
+
level: 1,
|
|
3562
|
+
pageStart: spanPage(span),
|
|
3563
|
+
pageEnd: spanPageEnd(span),
|
|
3564
|
+
formNumber: span.formNumber,
|
|
3565
|
+
excerpt: span.text.slice(0, 500),
|
|
3566
|
+
sourceSpanIds: [span.id],
|
|
3567
|
+
sourceTextHash: span.textHash ?? span.hash
|
|
3568
|
+
};
|
|
3569
|
+
});
|
|
3570
|
+
}
|
|
3571
|
+
function flattenToc(nodes) {
|
|
3572
|
+
const entries = [];
|
|
3573
|
+
const visit = (node) => {
|
|
3574
|
+
entries.push({
|
|
3575
|
+
title: node.title,
|
|
3576
|
+
level: node.level,
|
|
3577
|
+
pageStart: node.pageStart,
|
|
3578
|
+
pageEnd: node.pageEnd,
|
|
3579
|
+
documentNodeId: node.id,
|
|
3580
|
+
sourceSpanIds: node.sourceSpanIds
|
|
3581
|
+
});
|
|
3582
|
+
for (const child of node.children ?? []) visit(child);
|
|
3583
|
+
};
|
|
3584
|
+
for (const node of nodes) visit(node);
|
|
3585
|
+
return entries;
|
|
3586
|
+
}
|
|
3587
|
+
function buildPageMap(pageAssignments, sourceSpans, document) {
|
|
3588
|
+
const forms = document.formInventory ?? [];
|
|
3589
|
+
return pageAssignments.sort((left, right) => left.localPageNumber - right.localPageNumber).map((assignment) => {
|
|
3590
|
+
const form = forms.find((item) => {
|
|
3591
|
+
const start = item.pageStart;
|
|
3592
|
+
const end = item.pageEnd ?? start;
|
|
3593
|
+
return start != null && end != null && assignment.localPageNumber >= start && assignment.localPageNumber <= end;
|
|
3594
|
+
});
|
|
3595
|
+
return {
|
|
3596
|
+
page: assignment.localPageNumber,
|
|
3597
|
+
label: assignment.notes,
|
|
3598
|
+
formNumber: form?.formNumber,
|
|
3599
|
+
formTitle: form?.title,
|
|
3600
|
+
extractorNames: assignment.extractorNames,
|
|
3601
|
+
sourceSpanIds: sourceSpansForPage(sourceSpans, assignment.localPageNumber)
|
|
3602
|
+
};
|
|
3603
|
+
});
|
|
3604
|
+
}
|
|
3605
|
+
function buildAgentGuidance(document) {
|
|
3606
|
+
const guidance = [
|
|
3607
|
+
{
|
|
3608
|
+
kind: "source_structure",
|
|
3609
|
+
title: "Use the source outline as navigation",
|
|
3610
|
+
detail: "The documentOutline preserves source order and page ranges. Treat interpretation labels as hints, not as a replacement for the source document structure."
|
|
3611
|
+
}
|
|
3612
|
+
];
|
|
3613
|
+
if (document.declarations) {
|
|
3614
|
+
guidance.push({
|
|
3615
|
+
kind: "declarations",
|
|
3616
|
+
title: "Declarations establish policy facts",
|
|
3617
|
+
detail: "Declarations and schedules generally establish named insured, policy period, limits, deductibles, premium, and rating facts unless modified elsewhere."
|
|
3618
|
+
});
|
|
3619
|
+
}
|
|
3620
|
+
if (document.endorsements?.length || document.formInventory?.some((form) => form.formType === "endorsement")) {
|
|
3621
|
+
guidance.push({
|
|
3622
|
+
kind: "endorsement_override",
|
|
3623
|
+
title: "Endorsements may override base terms",
|
|
3624
|
+
detail: "Endorsements and change forms can add, remove, or override base policy wording. When terms conflict, inspect the effective endorsement language and source page."
|
|
3625
|
+
});
|
|
3626
|
+
}
|
|
3627
|
+
return guidance;
|
|
3628
|
+
}
|
|
3629
|
+
function attachDocumentStructure(params) {
|
|
3630
|
+
const docRecord = params.document;
|
|
3631
|
+
const sections = Array.isArray(params.document.sections) ? params.document.sections : [];
|
|
3632
|
+
const existingOutline = Array.isArray(docRecord.documentOutline) ? docRecord.documentOutline : [];
|
|
3633
|
+
const documentOutline = existingOutline.length > 0 ? existingOutline : sections.length > 0 ? sections.map((section, index) => buildNodeFromSection(section, index)) : buildNodesFromSourceSpans(params.sourceSpans);
|
|
3634
|
+
if (documentOutline.length > 0) {
|
|
3635
|
+
attachDocumentNodeIds(params.document, documentOutline);
|
|
3636
|
+
}
|
|
3637
|
+
const metadata = {
|
|
3638
|
+
...docRecord.documentMetadata && typeof docRecord.documentMetadata === "object" ? docRecord.documentMetadata : {},
|
|
3639
|
+
...params.document.formInventory?.length ? { formInventory: params.document.formInventory } : {},
|
|
3640
|
+
...documentOutline.length > 0 ? { tableOfContents: flattenToc(documentOutline) } : {},
|
|
3641
|
+
...params.pageAssignments.length > 0 ? { pageMap: buildPageMap(params.pageAssignments, params.sourceSpans, params.document) } : {},
|
|
3642
|
+
agentGuidance: buildAgentGuidance(params.document)
|
|
3643
|
+
};
|
|
3644
|
+
docRecord.documentMetadata = metadata;
|
|
3645
|
+
docRecord.documentOutline = documentOutline;
|
|
3646
|
+
return params.document;
|
|
3647
|
+
}
|
|
3648
|
+
|
|
3110
3649
|
// src/prompts/coordinator/format.ts
|
|
3111
3650
|
function buildFormatPrompt(entries) {
|
|
3112
3651
|
const block = entries.map((e) => `===ENTRY ${e.id}===
|
|
@@ -3706,6 +4245,8 @@ function chunkDocument(doc) {
|
|
|
3706
4245
|
{
|
|
3707
4246
|
formNumber: form.formNumber,
|
|
3708
4247
|
formType: form.formType,
|
|
4248
|
+
documentNodeId: form.documentNodeId,
|
|
4249
|
+
sourceSpanIds: form.sourceSpanIds?.join(","),
|
|
3709
4250
|
documentType: doc.type
|
|
3710
4251
|
}
|
|
3711
4252
|
);
|
|
@@ -3732,6 +4273,7 @@ function chunkDocument(doc) {
|
|
|
3732
4273
|
formNumber: end.formNumber,
|
|
3733
4274
|
pageStart: end.pageStart,
|
|
3734
4275
|
pageEnd: end.pageEnd,
|
|
4276
|
+
documentNodeId: end.documentNodeId,
|
|
3735
4277
|
sourceSpanIds: end.sourceSpanIds?.join(","),
|
|
3736
4278
|
sourceTextHash: end.sourceTextHash,
|
|
3737
4279
|
documentType: doc.type
|
|
@@ -3743,6 +4285,8 @@ function chunkDocument(doc) {
|
|
|
3743
4285
|
${exc.content}`.trim(), {
|
|
3744
4286
|
formNumber: exc.formNumber,
|
|
3745
4287
|
pageNumber: exc.pageNumber,
|
|
4288
|
+
documentNodeId: exc.documentNodeId,
|
|
4289
|
+
sourceSpanIds: exc.sourceSpanIds?.join(","),
|
|
3746
4290
|
documentType: doc.type
|
|
3747
4291
|
});
|
|
3748
4292
|
});
|
|
@@ -3760,6 +4304,8 @@ ${exc.content}`.trim(), {
|
|
|
3760
4304
|
conditionName: cond.name,
|
|
3761
4305
|
conditionType: cond.conditionType,
|
|
3762
4306
|
pageNumber: cond.pageNumber,
|
|
4307
|
+
documentNodeId: cond.documentNodeId,
|
|
4308
|
+
sourceSpanIds: cond.sourceSpanIds?.join(","),
|
|
3763
4309
|
documentType: doc.type
|
|
3764
4310
|
}
|
|
3765
4311
|
);
|
|
@@ -3781,6 +4327,9 @@ ${exc.content}`.trim(), {
|
|
|
3781
4327
|
formTitle: firstString2(definition, ["formTitle"]),
|
|
3782
4328
|
pageNumber: typeof definition.pageNumber === "number" ? definition.pageNumber : void 0,
|
|
3783
4329
|
sectionRef: firstString2(definition, ["sectionRef", "sectionTitle"]),
|
|
4330
|
+
documentNodeId: firstString2(definition, ["documentNodeId"]),
|
|
4331
|
+
sourceSpanIds: Array.isArray(definition.sourceSpanIds) ? definition.sourceSpanIds.join(",") : void 0,
|
|
4332
|
+
sourceTextHash: firstString2(definition, ["sourceTextHash"]),
|
|
3784
4333
|
documentType: doc.type
|
|
3785
4334
|
}
|
|
3786
4335
|
);
|
|
@@ -3809,6 +4358,9 @@ ${exc.content}`.trim(), {
|
|
|
3809
4358
|
formTitle: firstString2(coveredReason, ["formTitle"]),
|
|
3810
4359
|
pageNumber: typeof coveredReason.pageNumber === "number" ? coveredReason.pageNumber : void 0,
|
|
3811
4360
|
sectionRef: firstString2(coveredReason, ["sectionRef", "sectionTitle"]),
|
|
4361
|
+
documentNodeId: firstString2(coveredReason, ["documentNodeId"]),
|
|
4362
|
+
sourceSpanIds: Array.isArray(coveredReason.sourceSpanIds) ? coveredReason.sourceSpanIds.join(",") : void 0,
|
|
4363
|
+
sourceTextHash: firstString2(coveredReason, ["sourceTextHash"]),
|
|
3812
4364
|
documentType: doc.type
|
|
3813
4365
|
}
|
|
3814
4366
|
);
|
|
@@ -3870,6 +4422,7 @@ ${declLines.join("\n")}`, declMeta);
|
|
|
3870
4422
|
evidenceKind: "navigation",
|
|
3871
4423
|
sectionType: sec.type,
|
|
3872
4424
|
sectionNumber: sec.sectionNumber,
|
|
4425
|
+
documentNodeId: sec.documentNodeId,
|
|
3873
4426
|
pageStart: sec.pageStart,
|
|
3874
4427
|
pageEnd: sec.pageEnd,
|
|
3875
4428
|
sourceSpanIds: sec.sourceSpanIds?.join(","),
|
|
@@ -3893,6 +4446,7 @@ ${declLines.join("\n")}`, declMeta);
|
|
|
3893
4446
|
sectionType: sec.type,
|
|
3894
4447
|
parentSection: sec.title,
|
|
3895
4448
|
sectionNumber: sub.sectionNumber,
|
|
4449
|
+
documentNodeId: sub.documentNodeId,
|
|
3896
4450
|
pageNumber: sub.pageNumber,
|
|
3897
4451
|
sourceSpanIds: sub.sourceSpanIds?.join(","),
|
|
3898
4452
|
sourceTextHash: sub.sourceTextHash,
|
|
@@ -3901,6 +4455,31 @@ ${declLines.join("\n")}`, declMeta);
|
|
|
3901
4455
|
);
|
|
3902
4456
|
});
|
|
3903
4457
|
});
|
|
4458
|
+
asRecordArray(extendedDoc.documentOutline).forEach((node, i) => {
|
|
4459
|
+
const title = firstString2(node, ["title", "originalTitle"]) ?? `Document Node ${i + 1}`;
|
|
4460
|
+
const children = asRecordArray(node.children);
|
|
4461
|
+
pushChunk(
|
|
4462
|
+
`section:outline:${i}`,
|
|
4463
|
+
"section",
|
|
4464
|
+
lines([
|
|
4465
|
+
`Document Outline: ${title}`,
|
|
4466
|
+
firstString2(node, ["label", "type"]) ? `Label: ${firstString2(node, ["label", "type"])}` : null,
|
|
4467
|
+
typeof node.pageStart === "number" ? `Pages: ${node.pageStart}${typeof node.pageEnd === "number" ? `-${node.pageEnd}` : ""}` : null,
|
|
4468
|
+
firstString2(node, ["excerpt", "content"]),
|
|
4469
|
+
children.length > 0 ? `Children: ${children.map((child, childIndex) => firstString2(child, ["title", "originalTitle"]) ?? `Child ${childIndex + 1}`).join(", ")}` : null
|
|
4470
|
+
]),
|
|
4471
|
+
{
|
|
4472
|
+
evidenceKind: "navigation",
|
|
4473
|
+
documentNodeId: firstString2(node, ["id"]),
|
|
4474
|
+
sectionType: firstString2(node, ["type", "label"]),
|
|
4475
|
+
pageStart: typeof node.pageStart === "number" ? node.pageStart : void 0,
|
|
4476
|
+
pageEnd: typeof node.pageEnd === "number" ? node.pageEnd : void 0,
|
|
4477
|
+
sourceSpanIds: Array.isArray(node.sourceSpanIds) ? node.sourceSpanIds.join(",") : void 0,
|
|
4478
|
+
sourceTextHash: firstString2(node, ["sourceTextHash"]),
|
|
4479
|
+
documentType: doc.type
|
|
4480
|
+
}
|
|
4481
|
+
);
|
|
4482
|
+
});
|
|
3904
4483
|
doc.locations?.forEach((loc, i) => {
|
|
3905
4484
|
chunks.push({
|
|
3906
4485
|
id: `${docId}:location:${i}`,
|
|
@@ -4364,7 +4943,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4364
4943
|
const merged = mergeShallowPreferPresent(existing, incoming);
|
|
4365
4944
|
const existingCoverages = Array.isArray(existing.coverages) ? existing.coverages : [];
|
|
4366
4945
|
const incomingCoverages = Array.isArray(incoming.coverages) ? incoming.coverages : [];
|
|
4367
|
-
const
|
|
4946
|
+
const coverageKey2 = (coverage) => keyFromParts(
|
|
4368
4947
|
coverage.name,
|
|
4369
4948
|
coverage.limitType,
|
|
4370
4949
|
coverage.limit,
|
|
@@ -4373,7 +4952,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4373
4952
|
);
|
|
4374
4953
|
const byKey = /* @__PURE__ */ new Map();
|
|
4375
4954
|
for (const coverage of [...existingCoverages, ...incomingCoverages]) {
|
|
4376
|
-
const key =
|
|
4955
|
+
const key = coverageKey2(coverage);
|
|
4377
4956
|
const current = byKey.get(key);
|
|
4378
4957
|
byKey.set(key, current ? mergeShallowPreferPresent(current, coverage) : coverage);
|
|
4379
4958
|
}
|
|
@@ -6720,6 +7299,190 @@ async function resolveReferentialCoverages(params) {
|
|
|
6720
7299
|
};
|
|
6721
7300
|
}
|
|
6722
7301
|
|
|
7302
|
+
// src/extraction/coverage-schedule-recovery.ts
|
|
7303
|
+
function textValue(value) {
|
|
7304
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
7305
|
+
}
|
|
7306
|
+
function numberValue2(value) {
|
|
7307
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
7308
|
+
}
|
|
7309
|
+
function normalize(value) {
|
|
7310
|
+
return value.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, " ").trim();
|
|
7311
|
+
}
|
|
7312
|
+
function sourceUnit4(span) {
|
|
7313
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
7314
|
+
}
|
|
7315
|
+
function pageNumber(span) {
|
|
7316
|
+
return span.pageStart ?? span.location?.page ?? span.location?.startPage;
|
|
7317
|
+
}
|
|
7318
|
+
function coveragePages(pageAssignments) {
|
|
7319
|
+
const pages = /* @__PURE__ */ new Set();
|
|
7320
|
+
for (const assignment of pageAssignments) {
|
|
7321
|
+
if (assignment.extractorNames.includes("coverage_limits") || assignment.hasScheduleValues) {
|
|
7322
|
+
pages.add(assignment.localPageNumber);
|
|
7323
|
+
}
|
|
7324
|
+
}
|
|
7325
|
+
return pages;
|
|
7326
|
+
}
|
|
7327
|
+
function parseCurrencyAmount(value) {
|
|
7328
|
+
const match = value.match(/(?:CAD|USD|US)?\s*\$?\s*([0-9][0-9,]*(?:\.\d+)?)/i);
|
|
7329
|
+
if (!match) return void 0;
|
|
7330
|
+
const amount = Number(match[1].replace(/,/g, ""));
|
|
7331
|
+
return Number.isFinite(amount) ? amount : void 0;
|
|
7332
|
+
}
|
|
7333
|
+
function limitTypeFrom(value) {
|
|
7334
|
+
const normalized = normalize(value);
|
|
7335
|
+
if (normalized.includes("/") || normalized.includes("each claim") && normalized.includes("aggregate")) return "scheduled";
|
|
7336
|
+
if (normalized.includes("each claim") || normalized.includes("per claim")) return "per_claim";
|
|
7337
|
+
if (normalized.includes("each occurrence") || normalized.includes("per occurrence")) return "per_occurrence";
|
|
7338
|
+
if (normalized.includes("aggregate")) return "aggregate";
|
|
7339
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage")) return "scheduled";
|
|
7340
|
+
if (normalized.includes("/")) return "scheduled";
|
|
7341
|
+
return void 0;
|
|
7342
|
+
}
|
|
7343
|
+
function limitValueTypeFrom(value) {
|
|
7344
|
+
const normalized = normalize(value);
|
|
7345
|
+
if (parseCurrencyAmount(value) !== void 0) return "numeric";
|
|
7346
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage") || normalized.includes("shown above")) {
|
|
7347
|
+
return "referential";
|
|
7348
|
+
}
|
|
7349
|
+
if (normalized.includes("included")) return "included";
|
|
7350
|
+
if (normalized.includes("not included") || normalized === "nil" || normalized === "none") return "not_included";
|
|
7351
|
+
if (normalized.includes("as stated")) return "as_stated";
|
|
7352
|
+
return "other";
|
|
7353
|
+
}
|
|
7354
|
+
function cleanName(value) {
|
|
7355
|
+
return value.replace(/\s*\([^)]*part of and not in addition to[^)]*\)\s*/gi, " ").replace(/\s+/g, " ").trim().replace(/[.:;-]+$/, "").trim();
|
|
7356
|
+
}
|
|
7357
|
+
function isDeductibleOnly(name, rowText) {
|
|
7358
|
+
const normalizedName = normalize(name);
|
|
7359
|
+
const normalizedRow = normalize(rowText);
|
|
7360
|
+
if (!/\b(deductible|retention|sir)\b/.test(`${normalizedName} ${normalizedRow}`)) return false;
|
|
7361
|
+
if (/\b(coverage|sub limit|sublimit|limit)\b/.test(normalizedName) && !/\b(enhanced|standard)\s+deductible\b/.test(normalizedName)) {
|
|
7362
|
+
return false;
|
|
7363
|
+
}
|
|
7364
|
+
return /\b(enhanced|standard)?\s*(deductible|retention|sir)\b/.test(normalizedName) || /^\s*(deductible|retention|sir)\b/.test(normalizedRow);
|
|
7365
|
+
}
|
|
7366
|
+
function splitRowFields(rowText) {
|
|
7367
|
+
return rowText.split(/\s+\|\s+/).map((part) => part.trim()).filter(Boolean).map((part) => {
|
|
7368
|
+
const match = part.match(/^([^:]{1,100}):\s*(.*)$/);
|
|
7369
|
+
if (!match) return { value: part };
|
|
7370
|
+
return { key: match[1].trim(), value: match[2].trim() };
|
|
7371
|
+
});
|
|
7372
|
+
}
|
|
7373
|
+
function firstField(fields, patterns) {
|
|
7374
|
+
for (const field of fields) {
|
|
7375
|
+
const target = `${field.key ?? ""} ${field.value}`;
|
|
7376
|
+
if (patterns.some((pattern) => pattern.test(target))) return field.value || field.key;
|
|
7377
|
+
}
|
|
7378
|
+
return void 0;
|
|
7379
|
+
}
|
|
7380
|
+
function coverageFromRow(span) {
|
|
7381
|
+
const rowText = span.text.trim();
|
|
7382
|
+
if (!rowText || sourceUnit4(span) !== "table_row") return void 0;
|
|
7383
|
+
if (span.table?.isHeader || span.metadata?.isHeader === "true") return void 0;
|
|
7384
|
+
const fields = splitRowFields(rowText);
|
|
7385
|
+
let name;
|
|
7386
|
+
let limit;
|
|
7387
|
+
for (const field of fields) {
|
|
7388
|
+
const key = field.key?.trim();
|
|
7389
|
+
const value = field.value.trim();
|
|
7390
|
+
if (!key || !value) continue;
|
|
7391
|
+
if (!name && /^coverage$/i.test(key)) name = cleanName(value);
|
|
7392
|
+
if (!limit && /\blimit\b/i.test(key)) limit = value;
|
|
7393
|
+
if (!name && /\b(sub[-\s]?limit|aggregate|each claim)\b/i.test(key) && parseCurrencyAmount(value) !== void 0) {
|
|
7394
|
+
name = cleanName(key);
|
|
7395
|
+
limit = limit ?? value;
|
|
7396
|
+
}
|
|
7397
|
+
}
|
|
7398
|
+
if (!name || !limit) {
|
|
7399
|
+
for (const field of fields) {
|
|
7400
|
+
const key = field.key?.trim();
|
|
7401
|
+
const value = field.value.trim();
|
|
7402
|
+
if (!key || !value) continue;
|
|
7403
|
+
if (!name && /\b(sub[-\s]?limit|coverage|aggregate|each claim)\b/i.test(key) && !parseCurrencyAmount(key)) {
|
|
7404
|
+
name = cleanName(key);
|
|
7405
|
+
limit = limit ?? value;
|
|
7406
|
+
}
|
|
7407
|
+
}
|
|
7408
|
+
}
|
|
7409
|
+
if (!name || !limit) return void 0;
|
|
7410
|
+
if (isDeductibleOnly(name, rowText)) return void 0;
|
|
7411
|
+
const normalizedLimit = normalize(limit);
|
|
7412
|
+
const hasUsableLimit = parseCurrencyAmount(limit) !== void 0 || /\b(shared within|within coverage|as stated|included|not included)\b/i.test(normalizedLimit);
|
|
7413
|
+
if (!hasUsableLimit) return void 0;
|
|
7414
|
+
const deductible = firstField(fields, [/\bdeductible\b/i, /\bretention\b/i, /\bsir\b/i]);
|
|
7415
|
+
const basis = firstField(fields, [/\bbasis\b/i]);
|
|
7416
|
+
const retroactiveDate = firstField(fields, [/\bretroactive date\b/i, /\bretro date\b/i]);
|
|
7417
|
+
const page = pageNumber(span);
|
|
7418
|
+
const limitAmount = parseCurrencyAmount(limit);
|
|
7419
|
+
const deductibleAmount = deductible ? parseCurrencyAmount(deductible) : void 0;
|
|
7420
|
+
return {
|
|
7421
|
+
name,
|
|
7422
|
+
limit,
|
|
7423
|
+
...limitAmount !== void 0 ? { limitAmount } : {},
|
|
7424
|
+
...limitTypeFrom(`${name} ${limit}`) ? { limitType: limitTypeFrom(`${name} ${limit}`) } : {},
|
|
7425
|
+
limitValueType: limitValueTypeFrom(limit),
|
|
7426
|
+
...deductible && !/^nil|none$/i.test(deductible.trim()) ? { deductible } : {},
|
|
7427
|
+
...deductibleAmount !== void 0 ? { deductibleAmount } : {},
|
|
7428
|
+
...deductible ? { deductibleValueType: limitValueTypeFrom(deductible) } : {},
|
|
7429
|
+
...basis && /claims[- ]made/i.test(basis) ? { trigger: "claims_made" } : {},
|
|
7430
|
+
...retroactiveDate ? { retroactiveDate } : {},
|
|
7431
|
+
...span.formNumber ? { formNumber: span.formNumber } : {},
|
|
7432
|
+
...page ? { pageNumber: page } : {},
|
|
7433
|
+
sectionRef: span.sectionId ?? "SCHEDULE",
|
|
7434
|
+
originalContent: rowText,
|
|
7435
|
+
sourceSpanIds: [span.id],
|
|
7436
|
+
sourceTextHash: span.textHash ?? span.hash
|
|
7437
|
+
};
|
|
7438
|
+
}
|
|
7439
|
+
function coverageKey(coverage) {
|
|
7440
|
+
return [
|
|
7441
|
+
textValue(coverage.name) ?? "",
|
|
7442
|
+
textValue(coverage.limit) ?? "",
|
|
7443
|
+
textValue(coverage.limitType) ?? "",
|
|
7444
|
+
numberValue2(coverage.pageNumber) ?? ""
|
|
7445
|
+
].map((part) => normalize(String(part))).join("|");
|
|
7446
|
+
}
|
|
7447
|
+
function rowMatchesExisting(row, existing) {
|
|
7448
|
+
const rowKey = coverageKey(row);
|
|
7449
|
+
const rowName = normalize(textValue(row.name) ?? "");
|
|
7450
|
+
const rowLimit = normalize(textValue(row.limit) ?? "");
|
|
7451
|
+
return existing.some((coverage) => {
|
|
7452
|
+
if (coverageKey(coverage) === rowKey) return true;
|
|
7453
|
+
const name = normalize(textValue(coverage.name) ?? "");
|
|
7454
|
+
const limit = normalize(textValue(coverage.limit) ?? "");
|
|
7455
|
+
return Boolean(rowName && rowLimit && name === rowName && limit === rowLimit);
|
|
7456
|
+
});
|
|
7457
|
+
}
|
|
7458
|
+
function recoverCoverageScheduleRows(params) {
|
|
7459
|
+
const payload = params.memory.get("coverage_limits");
|
|
7460
|
+
const existing = Array.isArray(payload?.coverages) ? payload.coverages : [];
|
|
7461
|
+
const pages = coveragePages(params.pageAssignments);
|
|
7462
|
+
const candidates = params.sourceSpans.filter((span) => sourceUnit4(span) === "table_row").filter((span) => {
|
|
7463
|
+
const page = pageNumber(span);
|
|
7464
|
+
return page !== void 0 && pages.has(page);
|
|
7465
|
+
}).map(coverageFromRow).filter((coverage) => Boolean(coverage));
|
|
7466
|
+
const recovered = [];
|
|
7467
|
+
for (const coverage of candidates) {
|
|
7468
|
+
if (rowMatchesExisting(coverage, [...existing, ...recovered])) continue;
|
|
7469
|
+
recovered.push(coverage);
|
|
7470
|
+
}
|
|
7471
|
+
if (recovered.length > 0) {
|
|
7472
|
+
params.memory.set("coverage_limits", {
|
|
7473
|
+
...payload ?? {},
|
|
7474
|
+
coverages: [...existing, ...recovered]
|
|
7475
|
+
});
|
|
7476
|
+
}
|
|
7477
|
+
return {
|
|
7478
|
+
recovered,
|
|
7479
|
+
missingSourceRows: recovered.map((coverage) => {
|
|
7480
|
+
const id = Array.isArray(coverage.sourceSpanIds) ? coverage.sourceSpanIds[0] : void 0;
|
|
7481
|
+
return params.sourceSpans.find((span) => span.id === id);
|
|
7482
|
+
}).filter((span) => Boolean(span))
|
|
7483
|
+
};
|
|
7484
|
+
}
|
|
7485
|
+
|
|
6723
7486
|
// src/extraction/focused-dispatch.ts
|
|
6724
7487
|
async function runFocusedExtractorWithFallback(params) {
|
|
6725
7488
|
const {
|
|
@@ -6922,6 +7685,29 @@ function buildExtractionReviewReport(params) {
|
|
|
6922
7685
|
addMissingSourceGroundingIssues(deterministicIssues, "definitions", "definitions", definitions, "term");
|
|
6923
7686
|
addMissingSourceGroundingIssues(deterministicIssues, "covered_reasons", "coveredReasons", coveredReasons, "name");
|
|
6924
7687
|
}
|
|
7688
|
+
if (params.sourceSpans?.length) {
|
|
7689
|
+
const tempCoveragePayload = {
|
|
7690
|
+
...memory.get("coverage_limits") ?? {},
|
|
7691
|
+
coverages: coverages.map((coverage) => ({ ...coverage }))
|
|
7692
|
+
};
|
|
7693
|
+
const tempMemory = new Map(memory);
|
|
7694
|
+
tempMemory.set("coverage_limits", tempCoveragePayload);
|
|
7695
|
+
const scheduleRecovery = recoverCoverageScheduleRows({
|
|
7696
|
+
memory: tempMemory,
|
|
7697
|
+
sourceSpans: params.sourceSpans,
|
|
7698
|
+
pageAssignments: params.pageAssignments
|
|
7699
|
+
});
|
|
7700
|
+
for (const recovered of scheduleRecovery.recovered) {
|
|
7701
|
+
deterministicIssues.push({
|
|
7702
|
+
code: "coverage_schedule_row_missing",
|
|
7703
|
+
severity: "blocking",
|
|
7704
|
+
message: `Coverage schedule row "${String(recovered.name ?? "unknown")}" is present in source table evidence but missing from extracted coverages.`,
|
|
7705
|
+
extractorName: "coverage_limits",
|
|
7706
|
+
pageNumber: typeof recovered.pageNumber === "number" ? recovered.pageNumber : void 0,
|
|
7707
|
+
itemName: typeof recovered.name === "string" ? recovered.name : void 0
|
|
7708
|
+
});
|
|
7709
|
+
}
|
|
7710
|
+
}
|
|
6925
7711
|
if (mappedDefinitions && definitions.length === 0) {
|
|
6926
7712
|
deterministicIssues.push({
|
|
6927
7713
|
code: "definitions_mapped_but_empty",
|
|
@@ -7348,19 +8134,22 @@ var ARRAY_PATHS = [
|
|
|
7348
8134
|
{ memoryKey: "sections", arrayKeys: ["sections"] },
|
|
7349
8135
|
{ memoryKey: "definitions", arrayKeys: ["definitions"] },
|
|
7350
8136
|
{ memoryKey: "covered_reasons", arrayKeys: ["coveredReasons", "covered_reasons"] },
|
|
7351
|
-
{ memoryKey: "declarations", arrayKeys: ["fields"] }
|
|
8137
|
+
{ memoryKey: "declarations", arrayKeys: ["fields"] },
|
|
8138
|
+
{ memoryKey: "premium_breakdown", arrayKeys: ["premiumBreakdown", "taxesAndFees"] },
|
|
8139
|
+
{ memoryKey: "supplementary", arrayKeys: ["auxiliaryFacts", "supplementaryFacts"] },
|
|
8140
|
+
{ memoryKey: "form_inventory", arrayKeys: ["forms"] }
|
|
7352
8141
|
];
|
|
7353
|
-
function
|
|
8142
|
+
function normalize2(value) {
|
|
7354
8143
|
return value.replace(/\s+/g, " ").trim().toLowerCase();
|
|
7355
8144
|
}
|
|
7356
|
-
function
|
|
8145
|
+
function textValue2(record, ...keys) {
|
|
7357
8146
|
for (const key of keys) {
|
|
7358
8147
|
const value = record[key];
|
|
7359
8148
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
7360
8149
|
}
|
|
7361
8150
|
return void 0;
|
|
7362
8151
|
}
|
|
7363
|
-
function
|
|
8152
|
+
function numberValue3(record, ...keys) {
|
|
7364
8153
|
for (const key of keys) {
|
|
7365
8154
|
const value = record[key];
|
|
7366
8155
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
@@ -7377,37 +8166,73 @@ function pageOverlaps(recordStart, recordEnd, span) {
|
|
|
7377
8166
|
return start <= (spanEnd ?? spanStart) && end >= spanStart;
|
|
7378
8167
|
}
|
|
7379
8168
|
function formMatches(record, span) {
|
|
7380
|
-
const formNumber =
|
|
8169
|
+
const formNumber = textValue2(record, "formNumber");
|
|
7381
8170
|
if (!formNumber || !span.formNumber) return false;
|
|
7382
|
-
return
|
|
8171
|
+
return normalize2(formNumber) === normalize2(span.formNumber);
|
|
7383
8172
|
}
|
|
7384
8173
|
function textMatches(record, span) {
|
|
7385
|
-
const spanText =
|
|
8174
|
+
const spanText = normalize2(span.text);
|
|
7386
8175
|
const candidates = [
|
|
7387
|
-
|
|
7388
|
-
|
|
7389
|
-
|
|
8176
|
+
textValue2(record, "originalContent", "content", "definition", "value"),
|
|
8177
|
+
textValue2(record, "name", "title", "term", "field", "coverageName"),
|
|
8178
|
+
textValue2(record, "limit", "deductible", "premium")
|
|
7390
8179
|
].filter((value) => !!value && value.length >= 3);
|
|
7391
|
-
return candidates.some((candidate) => spanText.includes(
|
|
8180
|
+
return candidates.some((candidate) => spanText.includes(normalize2(candidate)));
|
|
7392
8181
|
}
|
|
7393
8182
|
function sourceHashFor(spans) {
|
|
7394
8183
|
return spans.map((span) => span.textHash ?? span.hash).filter(Boolean).join(":") || void 0;
|
|
7395
8184
|
}
|
|
8185
|
+
function sourceUnit5(span) {
|
|
8186
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
8187
|
+
}
|
|
8188
|
+
function hierarchyScore(span) {
|
|
8189
|
+
switch (sourceUnit5(span)) {
|
|
8190
|
+
case "table_row":
|
|
8191
|
+
return 5;
|
|
8192
|
+
case "table":
|
|
8193
|
+
return 3;
|
|
8194
|
+
case "section":
|
|
8195
|
+
case "page":
|
|
8196
|
+
return 2;
|
|
8197
|
+
case "table_cell":
|
|
8198
|
+
return -2;
|
|
8199
|
+
default:
|
|
8200
|
+
return 0;
|
|
8201
|
+
}
|
|
8202
|
+
}
|
|
8203
|
+
function parentRowId(span) {
|
|
8204
|
+
return span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
8205
|
+
}
|
|
8206
|
+
function preferParentRows(matches, sourceSpans) {
|
|
8207
|
+
if (matches.length === 0) return matches;
|
|
8208
|
+
const byId = new Map(sourceSpans.map((span) => [span.id, span]));
|
|
8209
|
+
const expanded = [];
|
|
8210
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8211
|
+
for (const match of matches) {
|
|
8212
|
+
const parent = sourceUnit5(match) === "table_cell" ? byId.get(parentRowId(match) ?? "") : void 0;
|
|
8213
|
+
const preferred = parent && sourceUnit5(parent) === "table_row" ? parent : match;
|
|
8214
|
+
if (seen.has(preferred.id)) continue;
|
|
8215
|
+
seen.add(preferred.id);
|
|
8216
|
+
expanded.push(preferred);
|
|
8217
|
+
}
|
|
8218
|
+
return expanded;
|
|
8219
|
+
}
|
|
7396
8220
|
function findSourceSpansForRecord(record, sourceSpans) {
|
|
7397
8221
|
if (sourceSpans.length === 0) return [];
|
|
7398
|
-
const pageStart =
|
|
7399
|
-
const pageEnd =
|
|
8222
|
+
const pageStart = numberValue3(record, "pageNumber", "pageStart");
|
|
8223
|
+
const pageEnd = numberValue3(record, "pageNumber", "pageEnd");
|
|
7400
8224
|
const scored = sourceSpans.map((span) => {
|
|
7401
8225
|
let score = 0;
|
|
7402
8226
|
if (pageOverlaps(pageStart, pageEnd, span)) score += 4;
|
|
7403
8227
|
if (formMatches(record, span)) score += 3;
|
|
7404
8228
|
if (textMatches(record, span)) score += 2;
|
|
8229
|
+
if (score > 0) score += hierarchyScore(span);
|
|
7405
8230
|
return { span, score };
|
|
7406
8231
|
}).filter((item) => item.score >= 2).sort((left, right) => {
|
|
7407
8232
|
if (right.score !== left.score) return right.score - left.score;
|
|
7408
8233
|
return left.span.id.localeCompare(right.span.id);
|
|
7409
8234
|
});
|
|
7410
|
-
return scored.slice(0,
|
|
8235
|
+
return preferParentRows(scored.slice(0, 5).map((item) => item.span), sourceSpans).slice(0, 3);
|
|
7411
8236
|
}
|
|
7412
8237
|
function groundRecord(record, sourceSpans) {
|
|
7413
8238
|
if (Array.isArray(record.sourceSpanIds) && record.sourceSpanIds.length > 0 && record.sourceTextHash) {
|
|
@@ -8115,6 +8940,14 @@ ${pageText}`;
|
|
|
8115
8940
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8116
8941
|
}
|
|
8117
8942
|
}
|
|
8943
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
8944
|
+
memory,
|
|
8945
|
+
sourceSpans,
|
|
8946
|
+
pageAssignments
|
|
8947
|
+
});
|
|
8948
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
8949
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from table evidence`);
|
|
8950
|
+
}
|
|
8118
8951
|
const planIncludesSupplementary = tasks.some((task) => task.extractorName === "supplementary");
|
|
8119
8952
|
if (!planIncludesSupplementary && hasSupplementaryExtractionSignal(pageAssignments, formInventory, memory)) {
|
|
8120
8953
|
onProgress?.("Extracting supplementary retrieval facts...");
|
|
@@ -8211,7 +9044,8 @@ ${pageText}`;
|
|
|
8211
9044
|
memory,
|
|
8212
9045
|
pageAssignments,
|
|
8213
9046
|
reviewRounds,
|
|
8214
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9047
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9048
|
+
sourceSpans
|
|
8215
9049
|
});
|
|
8216
9050
|
if (shouldRunLlmReview(reviewMode, preReviewReport, sourceSpans.length > 0)) {
|
|
8217
9051
|
for (let round = 0; round < maxReviewRounds; round++) {
|
|
@@ -8280,6 +9114,14 @@ ${pageText}`;
|
|
|
8280
9114
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8281
9115
|
}
|
|
8282
9116
|
}
|
|
9117
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
9118
|
+
memory,
|
|
9119
|
+
sourceSpans,
|
|
9120
|
+
pageAssignments
|
|
9121
|
+
});
|
|
9122
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
9123
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from follow-up table evidence`);
|
|
9124
|
+
}
|
|
8283
9125
|
}
|
|
8284
9126
|
} else {
|
|
8285
9127
|
onProgress?.("Skipping LLM extraction review; deterministic checks passed.");
|
|
@@ -8289,7 +9131,8 @@ ${pageText}`;
|
|
|
8289
9131
|
memory,
|
|
8290
9132
|
pageAssignments,
|
|
8291
9133
|
reviewRounds,
|
|
8292
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9134
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9135
|
+
sourceSpans
|
|
8293
9136
|
});
|
|
8294
9137
|
if (reviewReport.issues.length > 0) {
|
|
8295
9138
|
await log?.(
|
|
@@ -8315,10 +9158,12 @@ ${pageText}`;
|
|
|
8315
9158
|
memory,
|
|
8316
9159
|
pageAssignments,
|
|
8317
9160
|
reviewRounds,
|
|
8318
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9161
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9162
|
+
sourceSpans
|
|
8319
9163
|
}));
|
|
8320
9164
|
onProgress?.("Assembling document...");
|
|
8321
9165
|
const document = assembleDocument(id, documentType, memory);
|
|
9166
|
+
attachDocumentStructure({ document, pageAssignments, sourceSpans });
|
|
8322
9167
|
await pipelineCtx.save("assemble", {
|
|
8323
9168
|
id,
|
|
8324
9169
|
pageCount,
|
|
@@ -12116,6 +12961,11 @@ export {
|
|
|
12116
12961
|
DeductibleTypeSchema,
|
|
12117
12962
|
DefenseCostTreatmentSchema,
|
|
12118
12963
|
DefinitionSchema,
|
|
12964
|
+
DocumentAgentGuidanceSchema,
|
|
12965
|
+
DocumentMetadataSchema,
|
|
12966
|
+
DocumentNodeSchema,
|
|
12967
|
+
DocumentPageMapEntrySchema,
|
|
12968
|
+
DocumentTableOfContentsEntrySchema,
|
|
12119
12969
|
DocumentTypeSchema,
|
|
12120
12970
|
DriverRecordSchema,
|
|
12121
12971
|
DwellingDetailsSchema,
|
|
@@ -12237,6 +13087,8 @@ export {
|
|
|
12237
13087
|
SourceSpanLocationSchema,
|
|
12238
13088
|
SourceSpanRefSchema,
|
|
12239
13089
|
SourceSpanSchema,
|
|
13090
|
+
SourceSpanTableLocationSchema,
|
|
13091
|
+
SourceSpanUnitSchema,
|
|
12240
13092
|
SubAnswerSchema,
|
|
12241
13093
|
SubQuestionSchema,
|
|
12242
13094
|
SubjectivityCategorySchema,
|