@claritylabs/cl-sdk 1.3.7 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/index.d.mts +2303 -467
- package/dist/index.d.ts +2303 -467
- package/dist/index.js +919 -60
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +912 -60
- package/dist/index.mjs.map +1 -1
- package/dist/storage-sqlite.d.mts +947 -178
- package/dist/storage-sqlite.d.ts +947 -178
- package/dist/storage-sqlite.js +33 -2
- package/dist/storage-sqlite.js.map +1 -1
- package/dist/storage-sqlite.mjs +33 -2
- package/dist/storage-sqlite.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -103,6 +103,11 @@ __export(index_exports, {
|
|
|
103
103
|
DeductibleTypeSchema: () => DeductibleTypeSchema,
|
|
104
104
|
DefenseCostTreatmentSchema: () => DefenseCostTreatmentSchema,
|
|
105
105
|
DefinitionSchema: () => DefinitionSchema,
|
|
106
|
+
DocumentAgentGuidanceSchema: () => DocumentAgentGuidanceSchema,
|
|
107
|
+
DocumentMetadataSchema: () => DocumentMetadataSchema,
|
|
108
|
+
DocumentNodeSchema: () => DocumentNodeSchema,
|
|
109
|
+
DocumentPageMapEntrySchema: () => DocumentPageMapEntrySchema,
|
|
110
|
+
DocumentTableOfContentsEntrySchema: () => DocumentTableOfContentsEntrySchema,
|
|
106
111
|
DocumentTypeSchema: () => DocumentTypeSchema,
|
|
107
112
|
DriverRecordSchema: () => DriverRecordSchema,
|
|
108
113
|
DwellingDetailsSchema: () => DwellingDetailsSchema,
|
|
@@ -224,6 +229,8 @@ __export(index_exports, {
|
|
|
224
229
|
SourceSpanLocationSchema: () => SourceSpanLocationSchema,
|
|
225
230
|
SourceSpanRefSchema: () => SourceSpanRefSchema,
|
|
226
231
|
SourceSpanSchema: () => SourceSpanSchema,
|
|
232
|
+
SourceSpanTableLocationSchema: () => SourceSpanTableLocationSchema,
|
|
233
|
+
SourceSpanUnitSchema: () => SourceSpanUnitSchema,
|
|
227
234
|
SubAnswerSchema: () => SubAnswerSchema,
|
|
228
235
|
SubQuestionSchema: () => SubQuestionSchema,
|
|
229
236
|
SubjectivityCategorySchema: () => SubjectivityCategorySchema,
|
|
@@ -903,14 +910,20 @@ var FormReferenceSchema = import_zod3.z.object({
|
|
|
903
910
|
title: import_zod3.z.string().optional(),
|
|
904
911
|
formType: import_zod3.z.enum(["coverage", "endorsement", "declarations", "application", "notice", "other"]),
|
|
905
912
|
pageStart: import_zod3.z.number().optional(),
|
|
906
|
-
pageEnd: import_zod3.z.number().optional()
|
|
913
|
+
pageEnd: import_zod3.z.number().optional(),
|
|
914
|
+
documentNodeId: import_zod3.z.string().optional(),
|
|
915
|
+
sourceSpanIds: import_zod3.z.array(import_zod3.z.string()).optional(),
|
|
916
|
+
sourceTextHash: import_zod3.z.string().optional()
|
|
907
917
|
});
|
|
908
918
|
var TaxFeeItemSchema = import_zod3.z.object({
|
|
909
919
|
name: import_zod3.z.string(),
|
|
910
920
|
amount: import_zod3.z.string(),
|
|
911
921
|
amountValue: import_zod3.z.number().optional(),
|
|
912
922
|
type: import_zod3.z.enum(["tax", "fee", "surcharge", "assessment"]).optional(),
|
|
913
|
-
description: import_zod3.z.string().optional()
|
|
923
|
+
description: import_zod3.z.string().optional(),
|
|
924
|
+
documentNodeId: import_zod3.z.string().optional(),
|
|
925
|
+
sourceSpanIds: import_zod3.z.array(import_zod3.z.string()).optional(),
|
|
926
|
+
sourceTextHash: import_zod3.z.string().optional()
|
|
914
927
|
});
|
|
915
928
|
var RatingBasisSchema = import_zod3.z.object({
|
|
916
929
|
type: RatingBasisTypeSchema,
|
|
@@ -959,11 +972,14 @@ var CoverageSchema = import_zod4.z.object({
|
|
|
959
972
|
deductible: import_zod4.z.string().optional(),
|
|
960
973
|
deductibleAmount: import_zod4.z.number().optional(),
|
|
961
974
|
deductibleValueType: CoverageValueTypeSchema.optional(),
|
|
975
|
+
trigger: CoverageTriggerSchema.optional(),
|
|
976
|
+
retroactiveDate: import_zod4.z.string().optional(),
|
|
962
977
|
formNumber: import_zod4.z.string().optional(),
|
|
963
978
|
pageNumber: import_zod4.z.number().optional(),
|
|
964
979
|
sectionRef: import_zod4.z.string().optional(),
|
|
965
980
|
originalContent: import_zod4.z.string().optional(),
|
|
966
981
|
recordId: import_zod4.z.string().optional(),
|
|
982
|
+
documentNodeId: import_zod4.z.string().optional(),
|
|
967
983
|
sourceSpanIds: import_zod4.z.array(import_zod4.z.string()).optional(),
|
|
968
984
|
sourceTextHash: import_zod4.z.string().optional()
|
|
969
985
|
});
|
|
@@ -993,6 +1009,7 @@ var EnrichedCoverageSchema = import_zod4.z.object({
|
|
|
993
1009
|
sectionRef: import_zod4.z.string().optional(),
|
|
994
1010
|
originalContent: import_zod4.z.string().optional(),
|
|
995
1011
|
recordId: import_zod4.z.string().optional(),
|
|
1012
|
+
documentNodeId: import_zod4.z.string().optional(),
|
|
996
1013
|
sourceSpanIds: import_zod4.z.array(import_zod4.z.string()).optional(),
|
|
997
1014
|
sourceTextHash: import_zod4.z.string().optional()
|
|
998
1015
|
});
|
|
@@ -1021,6 +1038,7 @@ var EndorsementSchema = import_zod5.z.object({
|
|
|
1021
1038
|
pageStart: import_zod5.z.number(),
|
|
1022
1039
|
pageEnd: import_zod5.z.number().optional(),
|
|
1023
1040
|
recordId: import_zod5.z.string().optional(),
|
|
1041
|
+
documentNodeId: import_zod5.z.string().optional(),
|
|
1024
1042
|
sourceSpanIds: import_zod5.z.array(import_zod5.z.string()).optional(),
|
|
1025
1043
|
sourceTextHash: import_zod5.z.string().optional()
|
|
1026
1044
|
});
|
|
@@ -1039,6 +1057,7 @@ var ExclusionSchema = import_zod6.z.object({
|
|
|
1039
1057
|
content: import_zod6.z.string(),
|
|
1040
1058
|
pageNumber: import_zod6.z.number().optional(),
|
|
1041
1059
|
recordId: import_zod6.z.string().optional(),
|
|
1060
|
+
documentNodeId: import_zod6.z.string().optional(),
|
|
1042
1061
|
sourceSpanIds: import_zod6.z.array(import_zod6.z.string()).optional(),
|
|
1043
1062
|
sourceTextHash: import_zod6.z.string().optional()
|
|
1044
1063
|
});
|
|
@@ -1056,6 +1075,7 @@ var PolicyConditionSchema = import_zod7.z.object({
|
|
|
1056
1075
|
keyValues: import_zod7.z.array(ConditionKeyValueSchema).optional(),
|
|
1057
1076
|
pageNumber: import_zod7.z.number().optional(),
|
|
1058
1077
|
recordId: import_zod7.z.string().optional(),
|
|
1078
|
+
documentNodeId: import_zod7.z.string().optional(),
|
|
1059
1079
|
sourceSpanIds: import_zod7.z.array(import_zod7.z.string()).optional(),
|
|
1060
1080
|
sourceTextHash: import_zod7.z.string().optional()
|
|
1061
1081
|
});
|
|
@@ -1632,6 +1652,7 @@ var SubsectionSchema = import_zod16.z.object({
|
|
|
1632
1652
|
pageNumber: import_zod16.z.number().optional(),
|
|
1633
1653
|
excerpt: import_zod16.z.string().optional(),
|
|
1634
1654
|
content: import_zod16.z.string().optional(),
|
|
1655
|
+
documentNodeId: import_zod16.z.string().optional(),
|
|
1635
1656
|
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1636
1657
|
sourceTextHash: import_zod16.z.string().optional()
|
|
1637
1658
|
});
|
|
@@ -1646,6 +1667,7 @@ var SectionSchema = import_zod16.z.object({
|
|
|
1646
1667
|
content: import_zod16.z.string().optional(),
|
|
1647
1668
|
subsections: import_zod16.z.array(SubsectionSchema).optional(),
|
|
1648
1669
|
recordId: import_zod16.z.string().optional(),
|
|
1670
|
+
documentNodeId: import_zod16.z.string().optional(),
|
|
1649
1671
|
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1650
1672
|
sourceTextHash: import_zod16.z.string().optional()
|
|
1651
1673
|
});
|
|
@@ -1659,13 +1681,19 @@ var UnderwritingConditionSchema = import_zod16.z.object({
|
|
|
1659
1681
|
var PremiumLineSchema = import_zod16.z.object({
|
|
1660
1682
|
line: import_zod16.z.string(),
|
|
1661
1683
|
amount: import_zod16.z.string(),
|
|
1662
|
-
amountValue: import_zod16.z.number().optional()
|
|
1684
|
+
amountValue: import_zod16.z.number().optional(),
|
|
1685
|
+
documentNodeId: import_zod16.z.string().optional(),
|
|
1686
|
+
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1687
|
+
sourceTextHash: import_zod16.z.string().optional()
|
|
1663
1688
|
});
|
|
1664
1689
|
var AuxiliaryFactSchema = import_zod16.z.object({
|
|
1665
1690
|
key: import_zod16.z.string(),
|
|
1666
1691
|
value: import_zod16.z.string(),
|
|
1667
1692
|
subject: import_zod16.z.string().optional(),
|
|
1668
|
-
context: import_zod16.z.string().optional()
|
|
1693
|
+
context: import_zod16.z.string().optional(),
|
|
1694
|
+
documentNodeId: import_zod16.z.string().optional(),
|
|
1695
|
+
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1696
|
+
sourceTextHash: import_zod16.z.string().optional()
|
|
1669
1697
|
});
|
|
1670
1698
|
var DefinitionSchema = import_zod16.z.object({
|
|
1671
1699
|
term: import_zod16.z.string(),
|
|
@@ -1676,6 +1704,7 @@ var DefinitionSchema = import_zod16.z.object({
|
|
|
1676
1704
|
sectionRef: import_zod16.z.string().optional(),
|
|
1677
1705
|
originalContent: import_zod16.z.string().optional(),
|
|
1678
1706
|
recordId: import_zod16.z.string().optional(),
|
|
1707
|
+
documentNodeId: import_zod16.z.string().optional(),
|
|
1679
1708
|
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1680
1709
|
sourceTextHash: import_zod16.z.string().optional()
|
|
1681
1710
|
});
|
|
@@ -1693,9 +1722,60 @@ var CoveredReasonSchema = import_zod16.z.object({
|
|
|
1693
1722
|
sectionRef: import_zod16.z.string().optional(),
|
|
1694
1723
|
originalContent: import_zod16.z.string().optional(),
|
|
1695
1724
|
recordId: import_zod16.z.string().optional(),
|
|
1725
|
+
documentNodeId: import_zod16.z.string().optional(),
|
|
1696
1726
|
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1697
1727
|
sourceTextHash: import_zod16.z.string().optional()
|
|
1698
1728
|
});
|
|
1729
|
+
var DocumentTableOfContentsEntrySchema = import_zod16.z.object({
|
|
1730
|
+
title: import_zod16.z.string(),
|
|
1731
|
+
level: import_zod16.z.number().int().positive().optional(),
|
|
1732
|
+
pageStart: import_zod16.z.number().optional(),
|
|
1733
|
+
pageEnd: import_zod16.z.number().optional(),
|
|
1734
|
+
documentNodeId: import_zod16.z.string().optional(),
|
|
1735
|
+
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional()
|
|
1736
|
+
});
|
|
1737
|
+
var DocumentPageMapEntrySchema = import_zod16.z.object({
|
|
1738
|
+
page: import_zod16.z.number().int().positive(),
|
|
1739
|
+
label: import_zod16.z.string().optional(),
|
|
1740
|
+
formNumber: import_zod16.z.string().optional(),
|
|
1741
|
+
formTitle: import_zod16.z.string().optional(),
|
|
1742
|
+
sectionTitle: import_zod16.z.string().optional(),
|
|
1743
|
+
extractorNames: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1744
|
+
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional()
|
|
1745
|
+
});
|
|
1746
|
+
var DocumentAgentGuidanceSchema = import_zod16.z.object({
|
|
1747
|
+
kind: import_zod16.z.string(),
|
|
1748
|
+
title: import_zod16.z.string(),
|
|
1749
|
+
detail: import_zod16.z.string(),
|
|
1750
|
+
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional()
|
|
1751
|
+
});
|
|
1752
|
+
var DocumentMetadataSchema = import_zod16.z.object({
|
|
1753
|
+
formInventory: import_zod16.z.array(FormReferenceSchema).optional(),
|
|
1754
|
+
tableOfContents: import_zod16.z.array(DocumentTableOfContentsEntrySchema).optional(),
|
|
1755
|
+
pageMap: import_zod16.z.array(DocumentPageMapEntrySchema).optional(),
|
|
1756
|
+
agentGuidance: import_zod16.z.array(DocumentAgentGuidanceSchema).optional()
|
|
1757
|
+
});
|
|
1758
|
+
var DocumentNodeSchema = import_zod16.z.lazy(
|
|
1759
|
+
() => import_zod16.z.object({
|
|
1760
|
+
id: import_zod16.z.string(),
|
|
1761
|
+
title: import_zod16.z.string(),
|
|
1762
|
+
originalTitle: import_zod16.z.string().optional(),
|
|
1763
|
+
type: import_zod16.z.string().optional(),
|
|
1764
|
+
label: import_zod16.z.string().optional(),
|
|
1765
|
+
level: import_zod16.z.number().int().positive().optional(),
|
|
1766
|
+
sectionNumber: import_zod16.z.string().optional(),
|
|
1767
|
+
pageStart: import_zod16.z.number().optional(),
|
|
1768
|
+
pageEnd: import_zod16.z.number().optional(),
|
|
1769
|
+
formNumber: import_zod16.z.string().optional(),
|
|
1770
|
+
formTitle: import_zod16.z.string().optional(),
|
|
1771
|
+
excerpt: import_zod16.z.string().optional(),
|
|
1772
|
+
content: import_zod16.z.string().optional(),
|
|
1773
|
+
interpretationLabels: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1774
|
+
sourceSpanIds: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1775
|
+
sourceTextHash: import_zod16.z.string().optional(),
|
|
1776
|
+
children: import_zod16.z.array(DocumentNodeSchema).optional()
|
|
1777
|
+
})
|
|
1778
|
+
);
|
|
1699
1779
|
var BaseDocumentFields = {
|
|
1700
1780
|
id: import_zod16.z.string(),
|
|
1701
1781
|
carrier: import_zod16.z.string(),
|
|
@@ -1706,6 +1786,8 @@ var BaseDocumentFields = {
|
|
|
1706
1786
|
summary: import_zod16.z.string().optional(),
|
|
1707
1787
|
policyTypes: import_zod16.z.array(import_zod16.z.string()).optional(),
|
|
1708
1788
|
coverages: import_zod16.z.array(CoverageSchema),
|
|
1789
|
+
documentMetadata: DocumentMetadataSchema,
|
|
1790
|
+
documentOutline: import_zod16.z.array(DocumentNodeSchema),
|
|
1709
1791
|
sections: import_zod16.z.array(SectionSchema).optional(),
|
|
1710
1792
|
definitions: import_zod16.z.array(DefinitionSchema).optional(),
|
|
1711
1793
|
coveredReasons: import_zod16.z.array(CoveredReasonSchema).optional(),
|
|
@@ -2187,6 +2269,15 @@ var SourceSpanKindSchema = import_zod20.z.enum([
|
|
|
2187
2269
|
"plain_text",
|
|
2188
2270
|
"structured_field"
|
|
2189
2271
|
]);
|
|
2272
|
+
var SourceSpanUnitSchema = import_zod20.z.enum([
|
|
2273
|
+
"page",
|
|
2274
|
+
"section",
|
|
2275
|
+
"table",
|
|
2276
|
+
"table_row",
|
|
2277
|
+
"table_cell",
|
|
2278
|
+
"key_value",
|
|
2279
|
+
"text"
|
|
2280
|
+
]);
|
|
2190
2281
|
var SourceKindSchema = import_zod20.z.enum([
|
|
2191
2282
|
"policy_pdf",
|
|
2192
2283
|
"application_pdf",
|
|
@@ -2211,6 +2302,15 @@ var SourceSpanLocationSchema = import_zod20.z.object({
|
|
|
2211
2302
|
lineEnd: import_zod20.z.number().int().positive().optional(),
|
|
2212
2303
|
fieldPath: import_zod20.z.string().optional()
|
|
2213
2304
|
});
|
|
2305
|
+
var SourceSpanTableLocationSchema = import_zod20.z.object({
|
|
2306
|
+
tableId: import_zod20.z.string().optional(),
|
|
2307
|
+
rowIndex: import_zod20.z.number().int().nonnegative().optional(),
|
|
2308
|
+
columnIndex: import_zod20.z.number().int().nonnegative().optional(),
|
|
2309
|
+
columnName: import_zod20.z.string().optional(),
|
|
2310
|
+
rowSpanId: import_zod20.z.string().optional(),
|
|
2311
|
+
tableSpanId: import_zod20.z.string().optional(),
|
|
2312
|
+
isHeader: import_zod20.z.boolean().optional()
|
|
2313
|
+
});
|
|
2214
2314
|
var SourceSpanSchema = import_zod20.z.object({
|
|
2215
2315
|
id: import_zod20.z.string().min(1),
|
|
2216
2316
|
documentId: import_zod20.z.string().min(1),
|
|
@@ -2224,6 +2324,9 @@ var SourceSpanSchema = import_zod20.z.object({
|
|
|
2224
2324
|
pageEnd: import_zod20.z.number().int().positive().optional(),
|
|
2225
2325
|
sectionId: import_zod20.z.string().optional(),
|
|
2226
2326
|
formNumber: import_zod20.z.string().optional(),
|
|
2327
|
+
sourceUnit: SourceSpanUnitSchema.optional(),
|
|
2328
|
+
parentSpanId: import_zod20.z.string().optional(),
|
|
2329
|
+
table: SourceSpanTableLocationSchema.optional(),
|
|
2227
2330
|
bbox: import_zod20.z.array(SourceSpanBBoxSchema).optional(),
|
|
2228
2331
|
location: SourceSpanLocationSchema.optional(),
|
|
2229
2332
|
metadata: import_zod20.z.record(import_zod20.z.string(), import_zod20.z.string()).optional()
|
|
@@ -2342,6 +2445,9 @@ function buildSourceSpan(input, localIndex = 0) {
|
|
|
2342
2445
|
pageEnd: input.pageEnd,
|
|
2343
2446
|
sectionId: input.sectionId,
|
|
2344
2447
|
formNumber: input.formNumber,
|
|
2448
|
+
sourceUnit: input.sourceUnit,
|
|
2449
|
+
parentSpanId: input.parentSpanId,
|
|
2450
|
+
table: input.table,
|
|
2345
2451
|
location: {
|
|
2346
2452
|
page: input.pageStart === input.pageEnd ? input.pageStart : void 0,
|
|
2347
2453
|
startPage: input.pageStart,
|
|
@@ -2362,7 +2468,11 @@ function buildPageSourceSpans(pages) {
|
|
|
2362
2468
|
pageEnd: page.pageNumber,
|
|
2363
2469
|
sectionId: page.sectionId,
|
|
2364
2470
|
formNumber: page.formNumber,
|
|
2365
|
-
|
|
2471
|
+
sourceUnit: "page",
|
|
2472
|
+
metadata: {
|
|
2473
|
+
...page.metadata ?? {},
|
|
2474
|
+
sourceUnit: page.metadata?.sourceUnit ?? "page"
|
|
2475
|
+
}
|
|
2366
2476
|
},
|
|
2367
2477
|
index
|
|
2368
2478
|
)
|
|
@@ -2384,6 +2494,7 @@ function buildSectionSourceSpans(pages, options = {}) {
|
|
|
2384
2494
|
pageEnd: page.pageNumber,
|
|
2385
2495
|
sectionId: section.title,
|
|
2386
2496
|
formNumber: inferFormNumber(section.text),
|
|
2497
|
+
sourceUnit: "section",
|
|
2387
2498
|
metadata: {
|
|
2388
2499
|
...page.metadata ?? {},
|
|
2389
2500
|
sourceUnit: "section_candidate"
|
|
@@ -2416,6 +2527,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2416
2527
|
const chunks = [];
|
|
2417
2528
|
let current = [];
|
|
2418
2529
|
let currentLength = 0;
|
|
2530
|
+
const spansForChunking = filterChunkableSourceSpans(spans);
|
|
2419
2531
|
const flush = () => {
|
|
2420
2532
|
if (current.length === 0) return;
|
|
2421
2533
|
const text = current.map((span) => span.text).join("\n\n");
|
|
@@ -2439,7 +2551,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2439
2551
|
current = [];
|
|
2440
2552
|
currentLength = 0;
|
|
2441
2553
|
};
|
|
2442
|
-
for (const span of
|
|
2554
|
+
for (const span of spansForChunking) {
|
|
2443
2555
|
const nextLength = currentLength + span.text.length + (current.length > 0 ? 2 : 0);
|
|
2444
2556
|
if (current.length > 0 && nextLength > maxChars) {
|
|
2445
2557
|
flush();
|
|
@@ -2450,6 +2562,20 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2450
2562
|
flush();
|
|
2451
2563
|
return chunks;
|
|
2452
2564
|
}
|
|
2565
|
+
function sourceUnit(span) {
|
|
2566
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
2567
|
+
}
|
|
2568
|
+
function filterChunkableSourceSpans(spans) {
|
|
2569
|
+
const rowIds = new Set(
|
|
2570
|
+
spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
|
|
2571
|
+
);
|
|
2572
|
+
if (rowIds.size === 0) return spans;
|
|
2573
|
+
return spans.filter((span) => {
|
|
2574
|
+
if (sourceUnit(span) !== "table_cell") return true;
|
|
2575
|
+
const rowId = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
2576
|
+
return !rowId || !rowIds.has(rowId);
|
|
2577
|
+
});
|
|
2578
|
+
}
|
|
2453
2579
|
function splitPageIntoSections(text, headingPattern, minSectionChars) {
|
|
2454
2580
|
const lines = text.split(/\r?\n/);
|
|
2455
2581
|
const sections = [];
|
|
@@ -2834,36 +2960,18 @@ function normalizeDoclingDocument(document, options) {
|
|
|
2834
2960
|
pageTexts.set(page, appendText(pageTexts.get(page), unit.text));
|
|
2835
2961
|
}
|
|
2836
2962
|
const fullText = Array.from({ length: pageCount }, (_, index) => {
|
|
2837
|
-
const
|
|
2838
|
-
const text = pageTexts.get(
|
|
2839
|
-
return text ? `Page ${
|
|
2963
|
+
const pageNumber2 = index + 1;
|
|
2964
|
+
const text = pageTexts.get(pageNumber2)?.trim();
|
|
2965
|
+
return text ? `Page ${pageNumber2}
|
|
2840
2966
|
${text}` : "";
|
|
2841
2967
|
}).filter(Boolean).join("\n\n");
|
|
2842
2968
|
const sourceKind = options.sourceKind ?? "policy_pdf";
|
|
2843
|
-
const sourceSpans = units.
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
pageStart: unit.pageStart,
|
|
2850
|
-
pageEnd: unit.pageEnd,
|
|
2851
|
-
sectionId: unit.label,
|
|
2852
|
-
metadata: {
|
|
2853
|
-
sourceSystem: "docling",
|
|
2854
|
-
sourceUnit: "docling_item",
|
|
2855
|
-
doclingRef: unit.ref,
|
|
2856
|
-
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2857
|
-
}
|
|
2858
|
-
},
|
|
2859
|
-
index
|
|
2860
|
-
);
|
|
2861
|
-
return {
|
|
2862
|
-
...span,
|
|
2863
|
-
kind: "plain_text",
|
|
2864
|
-
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2865
|
-
};
|
|
2866
|
-
});
|
|
2969
|
+
const sourceSpans = units.flatMap(
|
|
2970
|
+
(unit, index) => buildSourceSpansForUnit(unit, index, {
|
|
2971
|
+
documentId: options.documentId,
|
|
2972
|
+
sourceKind
|
|
2973
|
+
})
|
|
2974
|
+
);
|
|
2867
2975
|
return {
|
|
2868
2976
|
pageCount,
|
|
2869
2977
|
fullText,
|
|
@@ -2901,6 +3009,11 @@ function mergeSourceSpans(spans) {
|
|
|
2901
3009
|
span.pageStart ?? span.location?.startPage ?? span.location?.page ?? "na",
|
|
2902
3010
|
span.pageEnd ?? span.location?.endPage ?? span.pageStart ?? "na",
|
|
2903
3011
|
span.sectionId ?? span.location?.fieldPath ?? "na",
|
|
3012
|
+
span.sourceUnit ?? span.metadata?.sourceUnit ?? "na",
|
|
3013
|
+
span.parentSpanId ?? "na",
|
|
3014
|
+
span.table?.tableId ?? span.metadata?.tableId ?? "na",
|
|
3015
|
+
span.table?.rowIndex ?? span.metadata?.rowIndex ?? "na",
|
|
3016
|
+
span.table?.columnIndex ?? span.metadata?.columnIndex ?? "na",
|
|
2904
3017
|
span.textHash ?? sourceSpanTextHash(span.text)
|
|
2905
3018
|
].join(":");
|
|
2906
3019
|
if (seen.has(key)) continue;
|
|
@@ -2963,6 +3076,7 @@ function getOrderedBodyRefs(document, itemMap) {
|
|
|
2963
3076
|
function normalizeItem(ref, item) {
|
|
2964
3077
|
const text = getItemText(item).trim();
|
|
2965
3078
|
if (!text) return void 0;
|
|
3079
|
+
const table = getItemTable(item);
|
|
2966
3080
|
const pages = (item.prov ?? []).map((prov) => getPageNumber(prov)).filter((page) => typeof page === "number" && page > 0);
|
|
2967
3081
|
const pageStart = pages.length ? Math.min(...pages) : void 0;
|
|
2968
3082
|
const pageEnd = pages.length ? Math.max(...pages) : pageStart;
|
|
@@ -2973,17 +3087,143 @@ function normalizeItem(ref, item) {
|
|
|
2973
3087
|
text,
|
|
2974
3088
|
pageStart,
|
|
2975
3089
|
pageEnd,
|
|
2976
|
-
bboxes: bboxes.length ? bboxes : void 0
|
|
3090
|
+
bboxes: bboxes.length ? bboxes : void 0,
|
|
3091
|
+
table
|
|
2977
3092
|
};
|
|
2978
3093
|
}
|
|
3094
|
+
function buildSourceSpansForUnit(unit, index, options) {
|
|
3095
|
+
const baseMetadata = {
|
|
3096
|
+
sourceSystem: "docling",
|
|
3097
|
+
sourceUnit: unit.table ? "table" : "docling_item",
|
|
3098
|
+
doclingRef: unit.ref,
|
|
3099
|
+
...unit.label ? { doclingLabel: unit.label } : {}
|
|
3100
|
+
};
|
|
3101
|
+
const tableId = unit.table ? `${unit.ref}:table` : void 0;
|
|
3102
|
+
const tableSpan = withDoclingKind(buildSourceSpan(
|
|
3103
|
+
{
|
|
3104
|
+
documentId: options.documentId,
|
|
3105
|
+
sourceKind: options.sourceKind,
|
|
3106
|
+
text: unit.text,
|
|
3107
|
+
pageStart: unit.pageStart,
|
|
3108
|
+
pageEnd: unit.pageEnd,
|
|
3109
|
+
sectionId: unit.label,
|
|
3110
|
+
sourceUnit: unit.table ? "table" : labelToSourceUnit(unit.label),
|
|
3111
|
+
table: tableId ? { tableId } : void 0,
|
|
3112
|
+
metadata: baseMetadata
|
|
3113
|
+
},
|
|
3114
|
+
index * 1e3
|
|
3115
|
+
), unit);
|
|
3116
|
+
if (!unit.table) return [tableSpan];
|
|
3117
|
+
const spans = [tableSpan];
|
|
3118
|
+
const table = unit.table;
|
|
3119
|
+
for (let rowIndex = 0; rowIndex < table.rows.length; rowIndex += 1) {
|
|
3120
|
+
const row = table.rows[rowIndex];
|
|
3121
|
+
const isHeader = rowIndex === 0 && table.headers.length > 0;
|
|
3122
|
+
const rowText = tableRowText(table, rowIndex);
|
|
3123
|
+
if (!rowText) continue;
|
|
3124
|
+
const rowSpan = withDoclingKind(buildSourceSpan(
|
|
3125
|
+
{
|
|
3126
|
+
documentId: options.documentId,
|
|
3127
|
+
sourceKind: options.sourceKind,
|
|
3128
|
+
text: rowText,
|
|
3129
|
+
pageStart: unit.pageStart,
|
|
3130
|
+
pageEnd: unit.pageEnd,
|
|
3131
|
+
sectionId: unit.label,
|
|
3132
|
+
sourceUnit: "table_row",
|
|
3133
|
+
parentSpanId: tableSpan.id,
|
|
3134
|
+
table: {
|
|
3135
|
+
tableId,
|
|
3136
|
+
tableSpanId: tableSpan.id,
|
|
3137
|
+
rowIndex,
|
|
3138
|
+
isHeader
|
|
3139
|
+
},
|
|
3140
|
+
metadata: {
|
|
3141
|
+
...baseMetadata,
|
|
3142
|
+
sourceUnit: "table_row",
|
|
3143
|
+
tableId: tableId ?? "",
|
|
3144
|
+
tableSpanId: tableSpan.id,
|
|
3145
|
+
rowIndex: String(rowIndex),
|
|
3146
|
+
isHeader: String(isHeader)
|
|
3147
|
+
}
|
|
3148
|
+
},
|
|
3149
|
+
index * 1e3 + 1 + rowIndex
|
|
3150
|
+
), unit);
|
|
3151
|
+
spans.push(rowSpan);
|
|
3152
|
+
for (let columnIndex = 0; columnIndex < row.length; columnIndex += 1) {
|
|
3153
|
+
const text = row[columnIndex]?.trim();
|
|
3154
|
+
if (!text) continue;
|
|
3155
|
+
const columnName = table.headers[columnIndex]?.trim() || void 0;
|
|
3156
|
+
const cellSpan = withDoclingKind(buildSourceSpan(
|
|
3157
|
+
{
|
|
3158
|
+
documentId: options.documentId,
|
|
3159
|
+
sourceKind: options.sourceKind,
|
|
3160
|
+
text,
|
|
3161
|
+
pageStart: unit.pageStart,
|
|
3162
|
+
pageEnd: unit.pageEnd,
|
|
3163
|
+
sectionId: unit.label,
|
|
3164
|
+
sourceUnit: "table_cell",
|
|
3165
|
+
parentSpanId: rowSpan.id,
|
|
3166
|
+
table: {
|
|
3167
|
+
tableId,
|
|
3168
|
+
tableSpanId: tableSpan.id,
|
|
3169
|
+
rowSpanId: rowSpan.id,
|
|
3170
|
+
rowIndex,
|
|
3171
|
+
columnIndex,
|
|
3172
|
+
columnName,
|
|
3173
|
+
isHeader
|
|
3174
|
+
},
|
|
3175
|
+
metadata: {
|
|
3176
|
+
...baseMetadata,
|
|
3177
|
+
sourceUnit: "table_cell",
|
|
3178
|
+
tableId: tableId ?? "",
|
|
3179
|
+
tableSpanId: tableSpan.id,
|
|
3180
|
+
rowSpanId: rowSpan.id,
|
|
3181
|
+
rowIndex: String(rowIndex),
|
|
3182
|
+
columnIndex: String(columnIndex),
|
|
3183
|
+
...columnName ? { columnName } : {},
|
|
3184
|
+
isHeader: String(isHeader)
|
|
3185
|
+
}
|
|
3186
|
+
},
|
|
3187
|
+
index * 1e3 + 100 + rowIndex * 100 + columnIndex
|
|
3188
|
+
), unit);
|
|
3189
|
+
spans.push(cellSpan);
|
|
3190
|
+
}
|
|
3191
|
+
}
|
|
3192
|
+
return spans;
|
|
3193
|
+
}
|
|
3194
|
+
function withDoclingKind(span, unit) {
|
|
3195
|
+
return {
|
|
3196
|
+
...span,
|
|
3197
|
+
kind: "plain_text",
|
|
3198
|
+
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
3199
|
+
};
|
|
3200
|
+
}
|
|
3201
|
+
function labelToSourceUnit(label) {
|
|
3202
|
+
const normalized = label?.toLowerCase() ?? "";
|
|
3203
|
+
if (normalized.includes("table")) return "table";
|
|
3204
|
+
if (normalized.includes("key")) return "key_value";
|
|
3205
|
+
if (normalized.includes("section") || normalized.includes("header")) return "section";
|
|
3206
|
+
return "text";
|
|
3207
|
+
}
|
|
3208
|
+
function tableRowText(table, rowIndex) {
|
|
3209
|
+
const row = table.rows[rowIndex] ?? [];
|
|
3210
|
+
const headers = table.headers;
|
|
3211
|
+
if (rowIndex === 0 && headers.length > 0) return row.filter(Boolean).join(" | ");
|
|
3212
|
+
return row.map((value, columnIndex) => {
|
|
3213
|
+
const trimmed = value.trim();
|
|
3214
|
+
if (!trimmed) return "";
|
|
3215
|
+
const header = headers[columnIndex]?.trim();
|
|
3216
|
+
return header ? `${header}: ${trimmed}` : trimmed;
|
|
3217
|
+
}).filter(Boolean).join(" | ");
|
|
3218
|
+
}
|
|
2979
3219
|
function getItemText(item) {
|
|
2980
3220
|
if (typeof item.text === "string" && item.text.trim()) return item.text;
|
|
2981
3221
|
if (typeof item.orig === "string" && item.orig.trim()) return item.orig;
|
|
2982
|
-
const table =
|
|
2983
|
-
if (table) return table;
|
|
3222
|
+
const table = parseTableData(item.data);
|
|
3223
|
+
if (table) return table.markdown;
|
|
2984
3224
|
return "";
|
|
2985
3225
|
}
|
|
2986
|
-
function
|
|
3226
|
+
function parseTableData(data) {
|
|
2987
3227
|
const record = asRecord(data);
|
|
2988
3228
|
const cells = Array.isArray(record?.table_cells) ? record.table_cells : Array.isArray(record?.tableCells) ? record.tableCells : void 0;
|
|
2989
3229
|
if (!cells) return void 0;
|
|
@@ -2999,10 +3239,20 @@ function tableToMarkdown(data) {
|
|
|
2999
3239
|
for (const cell of parsedCells) {
|
|
3000
3240
|
rows[cell.row][cell.col] = cell.text;
|
|
3001
3241
|
}
|
|
3002
|
-
if (rows.length === 1)
|
|
3242
|
+
if (rows.length === 1) {
|
|
3243
|
+
const markdown2 = rows[0].filter(Boolean).join(" | ");
|
|
3244
|
+
return { markdown: markdown2, headers: [], rows, cells: parsedCells };
|
|
3245
|
+
}
|
|
3003
3246
|
const header = rows[0];
|
|
3004
3247
|
const separator = header.map(() => "---");
|
|
3005
|
-
|
|
3248
|
+
const markdown = [header, separator, ...rows.slice(1)].map((row) => `| ${row.map((value) => value.trim()).join(" | ")} |`).join("\n");
|
|
3249
|
+
return { markdown, headers: header, rows, cells: parsedCells };
|
|
3250
|
+
}
|
|
3251
|
+
function getItemTable(item) {
|
|
3252
|
+
if (typeof item.text === "string" && item.text.trim() || typeof item.orig === "string" && item.orig.trim()) {
|
|
3253
|
+
return void 0;
|
|
3254
|
+
}
|
|
3255
|
+
return parseTableData(item.data);
|
|
3006
3256
|
}
|
|
3007
3257
|
function inferPageCount(document, units) {
|
|
3008
3258
|
const pages = document.pages;
|
|
@@ -3079,8 +3329,18 @@ function buildSourceContext(spans, maxChars = 12e3) {
|
|
|
3079
3329
|
if (spans.length === 0) return "";
|
|
3080
3330
|
const lines = [];
|
|
3081
3331
|
let length = 0;
|
|
3082
|
-
for (const span of spans) {
|
|
3083
|
-
const
|
|
3332
|
+
for (const span of orderSourceSpansForContext(spans)) {
|
|
3333
|
+
const unit = sourceUnit2(span);
|
|
3334
|
+
const table = span.table;
|
|
3335
|
+
const tableContext = [
|
|
3336
|
+
unit ? ` unit:${unit}` : "",
|
|
3337
|
+
table?.tableId ? ` table:${table.tableId}` : "",
|
|
3338
|
+
table?.rowIndex != null ? ` row:${table.rowIndex}` : "",
|
|
3339
|
+
table?.columnIndex != null ? ` col:${table.columnIndex}` : "",
|
|
3340
|
+
table?.columnName ? ` column:${table.columnName}` : "",
|
|
3341
|
+
span.parentSpanId ? ` parent:${span.parentSpanId}` : ""
|
|
3342
|
+
].join("");
|
|
3343
|
+
const header = `[sourceSpan:${span.id}${span.pageStart ? ` page:${span.pageStart}${span.pageEnd && span.pageEnd !== span.pageStart ? `-${span.pageEnd}` : ""}` : ""}${span.sectionId ? ` section:${span.sectionId}` : ""}${span.formNumber ? ` form:${span.formNumber}` : ""}${tableContext}]`;
|
|
3084
3344
|
const text = `${header}
|
|
3085
3345
|
${span.text}`;
|
|
3086
3346
|
if (length + text.length > maxChars && lines.length > 0) break;
|
|
@@ -3094,6 +3354,49 @@ ${lines.join("\n\n")}
|
|
|
3094
3354
|
|
|
3095
3355
|
Use sourceSpan IDs when grounding extracted contractual values.`;
|
|
3096
3356
|
}
|
|
3357
|
+
function sourceUnit2(span) {
|
|
3358
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
3359
|
+
}
|
|
3360
|
+
function sourceContextRank(span) {
|
|
3361
|
+
switch (sourceUnit2(span)) {
|
|
3362
|
+
case "table_row":
|
|
3363
|
+
return 0;
|
|
3364
|
+
case "table":
|
|
3365
|
+
return 1;
|
|
3366
|
+
case "section":
|
|
3367
|
+
return 2;
|
|
3368
|
+
case "page":
|
|
3369
|
+
return 3;
|
|
3370
|
+
case "key_value":
|
|
3371
|
+
return 4;
|
|
3372
|
+
case "table_cell":
|
|
3373
|
+
return 8;
|
|
3374
|
+
default:
|
|
3375
|
+
return 5;
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
function orderSourceSpansForContext(spans) {
|
|
3379
|
+
const parentRows = new Set(
|
|
3380
|
+
spans.filter((span) => sourceUnit2(span) === "table_row").map((span) => span.id)
|
|
3381
|
+
);
|
|
3382
|
+
const filtered = spans.filter((span) => {
|
|
3383
|
+
if (sourceUnit2(span) !== "table_cell") return true;
|
|
3384
|
+
const parent = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
3385
|
+
return !parent || !parentRows.has(parent);
|
|
3386
|
+
});
|
|
3387
|
+
return [...filtered].sort((left, right) => {
|
|
3388
|
+
const leftPage = left.pageStart ?? left.location?.startPage ?? left.location?.page ?? 0;
|
|
3389
|
+
const rightPage = right.pageStart ?? right.location?.startPage ?? right.location?.page ?? 0;
|
|
3390
|
+
if (leftPage !== rightPage) return leftPage - rightPage;
|
|
3391
|
+
const leftRank = sourceContextRank(left);
|
|
3392
|
+
const rightRank = sourceContextRank(right);
|
|
3393
|
+
if (leftRank !== rightRank) return leftRank - rightRank;
|
|
3394
|
+
const leftRow = left.table?.rowIndex ?? Number(left.metadata?.rowIndex ?? Number.NaN);
|
|
3395
|
+
const rightRow = right.table?.rowIndex ?? Number(right.metadata?.rowIndex ?? Number.NaN);
|
|
3396
|
+
if (Number.isFinite(leftRow) && Number.isFinite(rightRow) && leftRow !== rightRow) return leftRow - rightRow;
|
|
3397
|
+
return left.id.localeCompare(right.id);
|
|
3398
|
+
});
|
|
3399
|
+
}
|
|
3097
3400
|
async function runExtractor(params) {
|
|
3098
3401
|
const {
|
|
3099
3402
|
name,
|
|
@@ -3384,6 +3687,8 @@ function assembleDocument(documentId, documentType, memory) {
|
|
|
3384
3687
|
carrier: readRecordValue(carrier, "carrierName") ?? "Unknown",
|
|
3385
3688
|
insuredName: readRecordValue(insured, "insuredName") ?? "Unknown",
|
|
3386
3689
|
coverages: coverageRecords,
|
|
3690
|
+
documentMetadata: {},
|
|
3691
|
+
documentOutline: [],
|
|
3387
3692
|
policyTypes: readRecordValue(classify, "policyTypes"),
|
|
3388
3693
|
...sanitizeNulls(carrier ?? {}),
|
|
3389
3694
|
...sanitizeNulls(insured ?? {}),
|
|
@@ -3435,6 +3740,247 @@ function assembleDocument(documentId, documentType, memory) {
|
|
|
3435
3740
|
return doc;
|
|
3436
3741
|
}
|
|
3437
3742
|
|
|
3743
|
+
// src/extraction/document-structure.ts
|
|
3744
|
+
var FACT_ARRAY_PATHS = [
|
|
3745
|
+
"coverages",
|
|
3746
|
+
"enrichedCoverages",
|
|
3747
|
+
"endorsements",
|
|
3748
|
+
"exclusions",
|
|
3749
|
+
"conditions",
|
|
3750
|
+
"definitions",
|
|
3751
|
+
"coveredReasons",
|
|
3752
|
+
"taxesAndFees",
|
|
3753
|
+
"premiumBreakdown",
|
|
3754
|
+
"supplementaryFacts"
|
|
3755
|
+
];
|
|
3756
|
+
function slugPart(value) {
|
|
3757
|
+
const text = String(value ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
3758
|
+
return text || "node";
|
|
3759
|
+
}
|
|
3760
|
+
function numberValue(record, ...keys) {
|
|
3761
|
+
for (const key of keys) {
|
|
3762
|
+
const value = record[key];
|
|
3763
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
3764
|
+
}
|
|
3765
|
+
return void 0;
|
|
3766
|
+
}
|
|
3767
|
+
function stringValue2(record, ...keys) {
|
|
3768
|
+
for (const key of keys) {
|
|
3769
|
+
const value = record[key];
|
|
3770
|
+
if (typeof value === "string" && value.trim()) return value.trim();
|
|
3771
|
+
}
|
|
3772
|
+
return void 0;
|
|
3773
|
+
}
|
|
3774
|
+
function stringArray(value) {
|
|
3775
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.length > 0) : [];
|
|
3776
|
+
}
|
|
3777
|
+
function sourceUnit3(span) {
|
|
3778
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
3779
|
+
}
|
|
3780
|
+
function spanPage(span) {
|
|
3781
|
+
return span.pageStart ?? span.location?.page ?? span.location?.startPage;
|
|
3782
|
+
}
|
|
3783
|
+
function spanPageEnd(span) {
|
|
3784
|
+
return span.pageEnd ?? span.location?.endPage ?? spanPage(span);
|
|
3785
|
+
}
|
|
3786
|
+
function sourceSpansForPage(sourceSpans, page) {
|
|
3787
|
+
return sourceSpans.filter((span) => spanPage(span) === page && sourceUnit3(span) === "page").map((span) => span.id);
|
|
3788
|
+
}
|
|
3789
|
+
function nodePageOverlaps(node, record) {
|
|
3790
|
+
const recordStart = numberValue(record, "pageNumber", "pageStart", "resolvedFromPage");
|
|
3791
|
+
const recordEnd = numberValue(record, "pageNumber", "pageEnd", "resolvedFromPage") ?? recordStart;
|
|
3792
|
+
if (!recordStart) return false;
|
|
3793
|
+
const nodeStart = node.pageStart ?? node.pageEnd;
|
|
3794
|
+
const nodeEnd = node.pageEnd ?? node.pageStart;
|
|
3795
|
+
if (!nodeStart) return false;
|
|
3796
|
+
return recordStart <= (nodeEnd ?? nodeStart) && (recordEnd ?? recordStart) >= nodeStart;
|
|
3797
|
+
}
|
|
3798
|
+
function nodeFormMatches(node, record) {
|
|
3799
|
+
const formNumber = stringValue2(record, "formNumber");
|
|
3800
|
+
return Boolean(formNumber && node.formNumber && formNumber.toLowerCase() === node.formNumber.toLowerCase());
|
|
3801
|
+
}
|
|
3802
|
+
function nodeSourceOverlaps(node, record) {
|
|
3803
|
+
const nodeSourceIds = new Set(node.sourceSpanIds ?? []);
|
|
3804
|
+
if (nodeSourceIds.size === 0) return false;
|
|
3805
|
+
return stringArray(record.sourceSpanIds).some((id) => nodeSourceIds.has(id));
|
|
3806
|
+
}
|
|
3807
|
+
function findBestNode(nodes, record) {
|
|
3808
|
+
let best;
|
|
3809
|
+
const visit = (node) => {
|
|
3810
|
+
let score = 0;
|
|
3811
|
+
if (nodeSourceOverlaps(node, record)) score += 8;
|
|
3812
|
+
if (nodeFormMatches(node, record)) score += 4;
|
|
3813
|
+
if (nodePageOverlaps(node, record)) score += 3;
|
|
3814
|
+
if (score > 0 && (!best || score > best.score)) {
|
|
3815
|
+
best = { node, score };
|
|
3816
|
+
}
|
|
3817
|
+
for (const child of node.children ?? []) visit(child);
|
|
3818
|
+
};
|
|
3819
|
+
for (const node of nodes) visit(node);
|
|
3820
|
+
return best?.node;
|
|
3821
|
+
}
|
|
3822
|
+
function attachDocumentNodeIds(document, nodes) {
|
|
3823
|
+
const doc = document;
|
|
3824
|
+
for (const path of FACT_ARRAY_PATHS) {
|
|
3825
|
+
const value = doc[path];
|
|
3826
|
+
if (!Array.isArray(value)) continue;
|
|
3827
|
+
doc[path] = value.map((item) => {
|
|
3828
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) return item;
|
|
3829
|
+
const record = item;
|
|
3830
|
+
if (typeof record.documentNodeId === "string" && record.documentNodeId.length > 0) return record;
|
|
3831
|
+
const node = findBestNode(nodes, record);
|
|
3832
|
+
return node ? { ...record, documentNodeId: node.id } : record;
|
|
3833
|
+
});
|
|
3834
|
+
}
|
|
3835
|
+
}
|
|
3836
|
+
function buildNodeFromSection(section, index) {
|
|
3837
|
+
const title = stringValue2(section, "title", "name", "sectionRef") ?? `Section ${index + 1}`;
|
|
3838
|
+
const id = stringValue2(section, "documentNodeId", "recordId") ?? `section:${index}:${slugPart(title)}`;
|
|
3839
|
+
const pageStart = numberValue(section, "pageStart", "pageNumber");
|
|
3840
|
+
const pageEnd = numberValue(section, "pageEnd", "pageNumber") ?? pageStart;
|
|
3841
|
+
const type = stringValue2(section, "type");
|
|
3842
|
+
const coverageType = stringValue2(section, "coverageType");
|
|
3843
|
+
const sourceSpanIds = stringArray(section.sourceSpanIds);
|
|
3844
|
+
const children = Array.isArray(section.subsections) ? section.subsections.filter((item) => Boolean(item) && typeof item === "object" && !Array.isArray(item)).map((subsection, childIndex) => {
|
|
3845
|
+
const childTitle = stringValue2(subsection, "title", "name") ?? `${title} ${childIndex + 1}`;
|
|
3846
|
+
const childId = stringValue2(subsection, "documentNodeId", "recordId") ?? `${id}:subsection:${childIndex}:${slugPart(childTitle)}`;
|
|
3847
|
+
return {
|
|
3848
|
+
id: childId,
|
|
3849
|
+
title: childTitle,
|
|
3850
|
+
originalTitle: childTitle,
|
|
3851
|
+
type,
|
|
3852
|
+
label: type,
|
|
3853
|
+
level: 2,
|
|
3854
|
+
sectionNumber: stringValue2(subsection, "sectionNumber"),
|
|
3855
|
+
pageStart: numberValue(subsection, "pageNumber") ?? pageStart,
|
|
3856
|
+
pageEnd: numberValue(subsection, "pageNumber") ?? pageStart,
|
|
3857
|
+
excerpt: stringValue2(subsection, "excerpt"),
|
|
3858
|
+
content: stringValue2(subsection, "content"),
|
|
3859
|
+
sourceSpanIds: stringArray(subsection.sourceSpanIds),
|
|
3860
|
+
sourceTextHash: stringValue2(subsection, "sourceTextHash")
|
|
3861
|
+
};
|
|
3862
|
+
}) : void 0;
|
|
3863
|
+
return {
|
|
3864
|
+
id,
|
|
3865
|
+
title,
|
|
3866
|
+
originalTitle: title,
|
|
3867
|
+
type,
|
|
3868
|
+
label: type,
|
|
3869
|
+
level: 1,
|
|
3870
|
+
sectionNumber: stringValue2(section, "sectionNumber"),
|
|
3871
|
+
pageStart,
|
|
3872
|
+
pageEnd,
|
|
3873
|
+
formNumber: stringValue2(section, "formNumber"),
|
|
3874
|
+
formTitle: stringValue2(section, "formTitle"),
|
|
3875
|
+
excerpt: stringValue2(section, "excerpt"),
|
|
3876
|
+
content: stringValue2(section, "content"),
|
|
3877
|
+
interpretationLabels: [type, coverageType].filter((value) => Boolean(value)),
|
|
3878
|
+
sourceSpanIds: sourceSpanIds.length > 0 ? sourceSpanIds : void 0,
|
|
3879
|
+
sourceTextHash: stringValue2(section, "sourceTextHash"),
|
|
3880
|
+
children: children && children.length > 0 ? children : void 0
|
|
3881
|
+
};
|
|
3882
|
+
}
|
|
3883
|
+
function buildNodesFromSourceSpans(sourceSpans) {
|
|
3884
|
+
const candidates = sourceSpans.filter((span) => {
|
|
3885
|
+
const unit = sourceUnit3(span);
|
|
3886
|
+
return unit === "section" || unit === "section_candidate" || unit === "page";
|
|
3887
|
+
});
|
|
3888
|
+
return candidates.sort((left, right) => (spanPage(left) ?? 0) - (spanPage(right) ?? 0) || left.id.localeCompare(right.id)).map((span, index) => {
|
|
3889
|
+
const title = span.sectionId ?? span.formNumber ?? (sourceUnit3(span) === "page" && spanPage(span) ? `Page ${spanPage(span)}` : `Source unit ${index + 1}`);
|
|
3890
|
+
return {
|
|
3891
|
+
id: `source:${index}:${slugPart(span.id)}`,
|
|
3892
|
+
title,
|
|
3893
|
+
originalTitle: title,
|
|
3894
|
+
type: sourceUnit3(span),
|
|
3895
|
+
label: sourceUnit3(span),
|
|
3896
|
+
level: 1,
|
|
3897
|
+
pageStart: spanPage(span),
|
|
3898
|
+
pageEnd: spanPageEnd(span),
|
|
3899
|
+
formNumber: span.formNumber,
|
|
3900
|
+
excerpt: span.text.slice(0, 500),
|
|
3901
|
+
sourceSpanIds: [span.id],
|
|
3902
|
+
sourceTextHash: span.textHash ?? span.hash
|
|
3903
|
+
};
|
|
3904
|
+
});
|
|
3905
|
+
}
|
|
3906
|
+
function flattenToc(nodes) {
|
|
3907
|
+
const entries = [];
|
|
3908
|
+
const visit = (node) => {
|
|
3909
|
+
entries.push({
|
|
3910
|
+
title: node.title,
|
|
3911
|
+
level: node.level,
|
|
3912
|
+
pageStart: node.pageStart,
|
|
3913
|
+
pageEnd: node.pageEnd,
|
|
3914
|
+
documentNodeId: node.id,
|
|
3915
|
+
sourceSpanIds: node.sourceSpanIds
|
|
3916
|
+
});
|
|
3917
|
+
for (const child of node.children ?? []) visit(child);
|
|
3918
|
+
};
|
|
3919
|
+
for (const node of nodes) visit(node);
|
|
3920
|
+
return entries;
|
|
3921
|
+
}
|
|
3922
|
+
function buildPageMap(pageAssignments, sourceSpans, document) {
|
|
3923
|
+
const forms = document.formInventory ?? [];
|
|
3924
|
+
return pageAssignments.sort((left, right) => left.localPageNumber - right.localPageNumber).map((assignment) => {
|
|
3925
|
+
const form = forms.find((item) => {
|
|
3926
|
+
const start = item.pageStart;
|
|
3927
|
+
const end = item.pageEnd ?? start;
|
|
3928
|
+
return start != null && end != null && assignment.localPageNumber >= start && assignment.localPageNumber <= end;
|
|
3929
|
+
});
|
|
3930
|
+
return {
|
|
3931
|
+
page: assignment.localPageNumber,
|
|
3932
|
+
label: assignment.notes,
|
|
3933
|
+
formNumber: form?.formNumber,
|
|
3934
|
+
formTitle: form?.title,
|
|
3935
|
+
extractorNames: assignment.extractorNames,
|
|
3936
|
+
sourceSpanIds: sourceSpansForPage(sourceSpans, assignment.localPageNumber)
|
|
3937
|
+
};
|
|
3938
|
+
});
|
|
3939
|
+
}
|
|
3940
|
+
function buildAgentGuidance(document) {
|
|
3941
|
+
const guidance = [
|
|
3942
|
+
{
|
|
3943
|
+
kind: "source_structure",
|
|
3944
|
+
title: "Use the source outline as navigation",
|
|
3945
|
+
detail: "The documentOutline preserves source order and page ranges. Treat interpretation labels as hints, not as a replacement for the source document structure."
|
|
3946
|
+
}
|
|
3947
|
+
];
|
|
3948
|
+
if (document.declarations) {
|
|
3949
|
+
guidance.push({
|
|
3950
|
+
kind: "declarations",
|
|
3951
|
+
title: "Declarations establish policy facts",
|
|
3952
|
+
detail: "Declarations and schedules generally establish named insured, policy period, limits, deductibles, premium, and rating facts unless modified elsewhere."
|
|
3953
|
+
});
|
|
3954
|
+
}
|
|
3955
|
+
if (document.endorsements?.length || document.formInventory?.some((form) => form.formType === "endorsement")) {
|
|
3956
|
+
guidance.push({
|
|
3957
|
+
kind: "endorsement_override",
|
|
3958
|
+
title: "Endorsements may override base terms",
|
|
3959
|
+
detail: "Endorsements and change forms can add, remove, or override base policy wording. When terms conflict, inspect the effective endorsement language and source page."
|
|
3960
|
+
});
|
|
3961
|
+
}
|
|
3962
|
+
return guidance;
|
|
3963
|
+
}
|
|
3964
|
+
function attachDocumentStructure(params) {
|
|
3965
|
+
const docRecord = params.document;
|
|
3966
|
+
const sections = Array.isArray(params.document.sections) ? params.document.sections : [];
|
|
3967
|
+
const existingOutline = Array.isArray(docRecord.documentOutline) ? docRecord.documentOutline : [];
|
|
3968
|
+
const documentOutline = existingOutline.length > 0 ? existingOutline : sections.length > 0 ? sections.map((section, index) => buildNodeFromSection(section, index)) : buildNodesFromSourceSpans(params.sourceSpans);
|
|
3969
|
+
if (documentOutline.length > 0) {
|
|
3970
|
+
attachDocumentNodeIds(params.document, documentOutline);
|
|
3971
|
+
}
|
|
3972
|
+
const metadata = {
|
|
3973
|
+
...docRecord.documentMetadata && typeof docRecord.documentMetadata === "object" ? docRecord.documentMetadata : {},
|
|
3974
|
+
...params.document.formInventory?.length ? { formInventory: params.document.formInventory } : {},
|
|
3975
|
+
...documentOutline.length > 0 ? { tableOfContents: flattenToc(documentOutline) } : {},
|
|
3976
|
+
...params.pageAssignments.length > 0 ? { pageMap: buildPageMap(params.pageAssignments, params.sourceSpans, params.document) } : {},
|
|
3977
|
+
agentGuidance: buildAgentGuidance(params.document)
|
|
3978
|
+
};
|
|
3979
|
+
docRecord.documentMetadata = metadata;
|
|
3980
|
+
docRecord.documentOutline = documentOutline;
|
|
3981
|
+
return params.document;
|
|
3982
|
+
}
|
|
3983
|
+
|
|
3438
3984
|
// src/prompts/coordinator/format.ts
|
|
3439
3985
|
function buildFormatPrompt(entries) {
|
|
3440
3986
|
const block = entries.map((e) => `===ENTRY ${e.id}===
|
|
@@ -4034,6 +4580,8 @@ function chunkDocument(doc) {
|
|
|
4034
4580
|
{
|
|
4035
4581
|
formNumber: form.formNumber,
|
|
4036
4582
|
formType: form.formType,
|
|
4583
|
+
documentNodeId: form.documentNodeId,
|
|
4584
|
+
sourceSpanIds: form.sourceSpanIds?.join(","),
|
|
4037
4585
|
documentType: doc.type
|
|
4038
4586
|
}
|
|
4039
4587
|
);
|
|
@@ -4060,6 +4608,7 @@ function chunkDocument(doc) {
|
|
|
4060
4608
|
formNumber: end.formNumber,
|
|
4061
4609
|
pageStart: end.pageStart,
|
|
4062
4610
|
pageEnd: end.pageEnd,
|
|
4611
|
+
documentNodeId: end.documentNodeId,
|
|
4063
4612
|
sourceSpanIds: end.sourceSpanIds?.join(","),
|
|
4064
4613
|
sourceTextHash: end.sourceTextHash,
|
|
4065
4614
|
documentType: doc.type
|
|
@@ -4071,6 +4620,8 @@ function chunkDocument(doc) {
|
|
|
4071
4620
|
${exc.content}`.trim(), {
|
|
4072
4621
|
formNumber: exc.formNumber,
|
|
4073
4622
|
pageNumber: exc.pageNumber,
|
|
4623
|
+
documentNodeId: exc.documentNodeId,
|
|
4624
|
+
sourceSpanIds: exc.sourceSpanIds?.join(","),
|
|
4074
4625
|
documentType: doc.type
|
|
4075
4626
|
});
|
|
4076
4627
|
});
|
|
@@ -4088,6 +4639,8 @@ ${exc.content}`.trim(), {
|
|
|
4088
4639
|
conditionName: cond.name,
|
|
4089
4640
|
conditionType: cond.conditionType,
|
|
4090
4641
|
pageNumber: cond.pageNumber,
|
|
4642
|
+
documentNodeId: cond.documentNodeId,
|
|
4643
|
+
sourceSpanIds: cond.sourceSpanIds?.join(","),
|
|
4091
4644
|
documentType: doc.type
|
|
4092
4645
|
}
|
|
4093
4646
|
);
|
|
@@ -4109,6 +4662,9 @@ ${exc.content}`.trim(), {
|
|
|
4109
4662
|
formTitle: firstString2(definition, ["formTitle"]),
|
|
4110
4663
|
pageNumber: typeof definition.pageNumber === "number" ? definition.pageNumber : void 0,
|
|
4111
4664
|
sectionRef: firstString2(definition, ["sectionRef", "sectionTitle"]),
|
|
4665
|
+
documentNodeId: firstString2(definition, ["documentNodeId"]),
|
|
4666
|
+
sourceSpanIds: Array.isArray(definition.sourceSpanIds) ? definition.sourceSpanIds.join(",") : void 0,
|
|
4667
|
+
sourceTextHash: firstString2(definition, ["sourceTextHash"]),
|
|
4112
4668
|
documentType: doc.type
|
|
4113
4669
|
}
|
|
4114
4670
|
);
|
|
@@ -4137,6 +4693,9 @@ ${exc.content}`.trim(), {
|
|
|
4137
4693
|
formTitle: firstString2(coveredReason, ["formTitle"]),
|
|
4138
4694
|
pageNumber: typeof coveredReason.pageNumber === "number" ? coveredReason.pageNumber : void 0,
|
|
4139
4695
|
sectionRef: firstString2(coveredReason, ["sectionRef", "sectionTitle"]),
|
|
4696
|
+
documentNodeId: firstString2(coveredReason, ["documentNodeId"]),
|
|
4697
|
+
sourceSpanIds: Array.isArray(coveredReason.sourceSpanIds) ? coveredReason.sourceSpanIds.join(",") : void 0,
|
|
4698
|
+
sourceTextHash: firstString2(coveredReason, ["sourceTextHash"]),
|
|
4140
4699
|
documentType: doc.type
|
|
4141
4700
|
}
|
|
4142
4701
|
);
|
|
@@ -4198,6 +4757,7 @@ ${declLines.join("\n")}`, declMeta);
|
|
|
4198
4757
|
evidenceKind: "navigation",
|
|
4199
4758
|
sectionType: sec.type,
|
|
4200
4759
|
sectionNumber: sec.sectionNumber,
|
|
4760
|
+
documentNodeId: sec.documentNodeId,
|
|
4201
4761
|
pageStart: sec.pageStart,
|
|
4202
4762
|
pageEnd: sec.pageEnd,
|
|
4203
4763
|
sourceSpanIds: sec.sourceSpanIds?.join(","),
|
|
@@ -4221,6 +4781,7 @@ ${declLines.join("\n")}`, declMeta);
|
|
|
4221
4781
|
sectionType: sec.type,
|
|
4222
4782
|
parentSection: sec.title,
|
|
4223
4783
|
sectionNumber: sub.sectionNumber,
|
|
4784
|
+
documentNodeId: sub.documentNodeId,
|
|
4224
4785
|
pageNumber: sub.pageNumber,
|
|
4225
4786
|
sourceSpanIds: sub.sourceSpanIds?.join(","),
|
|
4226
4787
|
sourceTextHash: sub.sourceTextHash,
|
|
@@ -4229,6 +4790,31 @@ ${declLines.join("\n")}`, declMeta);
|
|
|
4229
4790
|
);
|
|
4230
4791
|
});
|
|
4231
4792
|
});
|
|
4793
|
+
asRecordArray(extendedDoc.documentOutline).forEach((node, i) => {
|
|
4794
|
+
const title = firstString2(node, ["title", "originalTitle"]) ?? `Document Node ${i + 1}`;
|
|
4795
|
+
const children = asRecordArray(node.children);
|
|
4796
|
+
pushChunk(
|
|
4797
|
+
`section:outline:${i}`,
|
|
4798
|
+
"section",
|
|
4799
|
+
lines([
|
|
4800
|
+
`Document Outline: ${title}`,
|
|
4801
|
+
firstString2(node, ["label", "type"]) ? `Label: ${firstString2(node, ["label", "type"])}` : null,
|
|
4802
|
+
typeof node.pageStart === "number" ? `Pages: ${node.pageStart}${typeof node.pageEnd === "number" ? `-${node.pageEnd}` : ""}` : null,
|
|
4803
|
+
firstString2(node, ["excerpt", "content"]),
|
|
4804
|
+
children.length > 0 ? `Children: ${children.map((child, childIndex) => firstString2(child, ["title", "originalTitle"]) ?? `Child ${childIndex + 1}`).join(", ")}` : null
|
|
4805
|
+
]),
|
|
4806
|
+
{
|
|
4807
|
+
evidenceKind: "navigation",
|
|
4808
|
+
documentNodeId: firstString2(node, ["id"]),
|
|
4809
|
+
sectionType: firstString2(node, ["type", "label"]),
|
|
4810
|
+
pageStart: typeof node.pageStart === "number" ? node.pageStart : void 0,
|
|
4811
|
+
pageEnd: typeof node.pageEnd === "number" ? node.pageEnd : void 0,
|
|
4812
|
+
sourceSpanIds: Array.isArray(node.sourceSpanIds) ? node.sourceSpanIds.join(",") : void 0,
|
|
4813
|
+
sourceTextHash: firstString2(node, ["sourceTextHash"]),
|
|
4814
|
+
documentType: doc.type
|
|
4815
|
+
}
|
|
4816
|
+
);
|
|
4817
|
+
});
|
|
4232
4818
|
doc.locations?.forEach((loc, i) => {
|
|
4233
4819
|
chunks.push({
|
|
4234
4820
|
id: `${docId}:location:${i}`,
|
|
@@ -4692,7 +5278,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4692
5278
|
const merged = mergeShallowPreferPresent(existing, incoming);
|
|
4693
5279
|
const existingCoverages = Array.isArray(existing.coverages) ? existing.coverages : [];
|
|
4694
5280
|
const incomingCoverages = Array.isArray(incoming.coverages) ? incoming.coverages : [];
|
|
4695
|
-
const
|
|
5281
|
+
const coverageKey2 = (coverage) => keyFromParts(
|
|
4696
5282
|
coverage.name,
|
|
4697
5283
|
coverage.limitType,
|
|
4698
5284
|
coverage.limit,
|
|
@@ -4701,7 +5287,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4701
5287
|
);
|
|
4702
5288
|
const byKey = /* @__PURE__ */ new Map();
|
|
4703
5289
|
for (const coverage of [...existingCoverages, ...incomingCoverages]) {
|
|
4704
|
-
const key =
|
|
5290
|
+
const key = coverageKey2(coverage);
|
|
4705
5291
|
const current = byKey.get(key);
|
|
4706
5292
|
byKey.set(key, current ? mergeShallowPreferPresent(current, coverage) : coverage);
|
|
4707
5293
|
}
|
|
@@ -7048,6 +7634,190 @@ async function resolveReferentialCoverages(params) {
|
|
|
7048
7634
|
};
|
|
7049
7635
|
}
|
|
7050
7636
|
|
|
7637
|
+
// src/extraction/coverage-schedule-recovery.ts
|
|
7638
|
+
function textValue(value) {
|
|
7639
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
7640
|
+
}
|
|
7641
|
+
function numberValue2(value) {
|
|
7642
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
7643
|
+
}
|
|
7644
|
+
function normalize(value) {
|
|
7645
|
+
return value.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, " ").trim();
|
|
7646
|
+
}
|
|
7647
|
+
function sourceUnit4(span) {
|
|
7648
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
7649
|
+
}
|
|
7650
|
+
function pageNumber(span) {
|
|
7651
|
+
return span.pageStart ?? span.location?.page ?? span.location?.startPage;
|
|
7652
|
+
}
|
|
7653
|
+
function coveragePages(pageAssignments) {
|
|
7654
|
+
const pages = /* @__PURE__ */ new Set();
|
|
7655
|
+
for (const assignment of pageAssignments) {
|
|
7656
|
+
if (assignment.extractorNames.includes("coverage_limits") || assignment.hasScheduleValues) {
|
|
7657
|
+
pages.add(assignment.localPageNumber);
|
|
7658
|
+
}
|
|
7659
|
+
}
|
|
7660
|
+
return pages;
|
|
7661
|
+
}
|
|
7662
|
+
function parseCurrencyAmount(value) {
|
|
7663
|
+
const match = value.match(/(?:CAD|USD|US)?\s*\$?\s*([0-9][0-9,]*(?:\.\d+)?)/i);
|
|
7664
|
+
if (!match) return void 0;
|
|
7665
|
+
const amount = Number(match[1].replace(/,/g, ""));
|
|
7666
|
+
return Number.isFinite(amount) ? amount : void 0;
|
|
7667
|
+
}
|
|
7668
|
+
function limitTypeFrom(value) {
|
|
7669
|
+
const normalized = normalize(value);
|
|
7670
|
+
if (normalized.includes("/") || normalized.includes("each claim") && normalized.includes("aggregate")) return "scheduled";
|
|
7671
|
+
if (normalized.includes("each claim") || normalized.includes("per claim")) return "per_claim";
|
|
7672
|
+
if (normalized.includes("each occurrence") || normalized.includes("per occurrence")) return "per_occurrence";
|
|
7673
|
+
if (normalized.includes("aggregate")) return "aggregate";
|
|
7674
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage")) return "scheduled";
|
|
7675
|
+
if (normalized.includes("/")) return "scheduled";
|
|
7676
|
+
return void 0;
|
|
7677
|
+
}
|
|
7678
|
+
function limitValueTypeFrom(value) {
|
|
7679
|
+
const normalized = normalize(value);
|
|
7680
|
+
if (parseCurrencyAmount(value) !== void 0) return "numeric";
|
|
7681
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage") || normalized.includes("shown above")) {
|
|
7682
|
+
return "referential";
|
|
7683
|
+
}
|
|
7684
|
+
if (normalized.includes("included")) return "included";
|
|
7685
|
+
if (normalized.includes("not included") || normalized === "nil" || normalized === "none") return "not_included";
|
|
7686
|
+
if (normalized.includes("as stated")) return "as_stated";
|
|
7687
|
+
return "other";
|
|
7688
|
+
}
|
|
7689
|
+
function cleanName(value) {
|
|
7690
|
+
return value.replace(/\s*\([^)]*part of and not in addition to[^)]*\)\s*/gi, " ").replace(/\s+/g, " ").trim().replace(/[.:;-]+$/, "").trim();
|
|
7691
|
+
}
|
|
7692
|
+
function isDeductibleOnly(name, rowText) {
|
|
7693
|
+
const normalizedName = normalize(name);
|
|
7694
|
+
const normalizedRow = normalize(rowText);
|
|
7695
|
+
if (!/\b(deductible|retention|sir)\b/.test(`${normalizedName} ${normalizedRow}`)) return false;
|
|
7696
|
+
if (/\b(coverage|sub limit|sublimit|limit)\b/.test(normalizedName) && !/\b(enhanced|standard)\s+deductible\b/.test(normalizedName)) {
|
|
7697
|
+
return false;
|
|
7698
|
+
}
|
|
7699
|
+
return /\b(enhanced|standard)?\s*(deductible|retention|sir)\b/.test(normalizedName) || /^\s*(deductible|retention|sir)\b/.test(normalizedRow);
|
|
7700
|
+
}
|
|
7701
|
+
function splitRowFields(rowText) {
|
|
7702
|
+
return rowText.split(/\s+\|\s+/).map((part) => part.trim()).filter(Boolean).map((part) => {
|
|
7703
|
+
const match = part.match(/^([^:]{1,100}):\s*(.*)$/);
|
|
7704
|
+
if (!match) return { value: part };
|
|
7705
|
+
return { key: match[1].trim(), value: match[2].trim() };
|
|
7706
|
+
});
|
|
7707
|
+
}
|
|
7708
|
+
function firstField(fields, patterns) {
|
|
7709
|
+
for (const field of fields) {
|
|
7710
|
+
const target = `${field.key ?? ""} ${field.value}`;
|
|
7711
|
+
if (patterns.some((pattern) => pattern.test(target))) return field.value || field.key;
|
|
7712
|
+
}
|
|
7713
|
+
return void 0;
|
|
7714
|
+
}
|
|
7715
|
+
function coverageFromRow(span) {
|
|
7716
|
+
const rowText = span.text.trim();
|
|
7717
|
+
if (!rowText || sourceUnit4(span) !== "table_row") return void 0;
|
|
7718
|
+
if (span.table?.isHeader || span.metadata?.isHeader === "true") return void 0;
|
|
7719
|
+
const fields = splitRowFields(rowText);
|
|
7720
|
+
let name;
|
|
7721
|
+
let limit;
|
|
7722
|
+
for (const field of fields) {
|
|
7723
|
+
const key = field.key?.trim();
|
|
7724
|
+
const value = field.value.trim();
|
|
7725
|
+
if (!key || !value) continue;
|
|
7726
|
+
if (!name && /^coverage$/i.test(key)) name = cleanName(value);
|
|
7727
|
+
if (!limit && /\blimit\b/i.test(key)) limit = value;
|
|
7728
|
+
if (!name && /\b(sub[-\s]?limit|aggregate|each claim)\b/i.test(key) && parseCurrencyAmount(value) !== void 0) {
|
|
7729
|
+
name = cleanName(key);
|
|
7730
|
+
limit = limit ?? value;
|
|
7731
|
+
}
|
|
7732
|
+
}
|
|
7733
|
+
if (!name || !limit) {
|
|
7734
|
+
for (const field of fields) {
|
|
7735
|
+
const key = field.key?.trim();
|
|
7736
|
+
const value = field.value.trim();
|
|
7737
|
+
if (!key || !value) continue;
|
|
7738
|
+
if (!name && /\b(sub[-\s]?limit|coverage|aggregate|each claim)\b/i.test(key) && !parseCurrencyAmount(key)) {
|
|
7739
|
+
name = cleanName(key);
|
|
7740
|
+
limit = limit ?? value;
|
|
7741
|
+
}
|
|
7742
|
+
}
|
|
7743
|
+
}
|
|
7744
|
+
if (!name || !limit) return void 0;
|
|
7745
|
+
if (isDeductibleOnly(name, rowText)) return void 0;
|
|
7746
|
+
const normalizedLimit = normalize(limit);
|
|
7747
|
+
const hasUsableLimit = parseCurrencyAmount(limit) !== void 0 || /\b(shared within|within coverage|as stated|included|not included)\b/i.test(normalizedLimit);
|
|
7748
|
+
if (!hasUsableLimit) return void 0;
|
|
7749
|
+
const deductible = firstField(fields, [/\bdeductible\b/i, /\bretention\b/i, /\bsir\b/i]);
|
|
7750
|
+
const basis = firstField(fields, [/\bbasis\b/i]);
|
|
7751
|
+
const retroactiveDate = firstField(fields, [/\bretroactive date\b/i, /\bretro date\b/i]);
|
|
7752
|
+
const page = pageNumber(span);
|
|
7753
|
+
const limitAmount = parseCurrencyAmount(limit);
|
|
7754
|
+
const deductibleAmount = deductible ? parseCurrencyAmount(deductible) : void 0;
|
|
7755
|
+
return {
|
|
7756
|
+
name,
|
|
7757
|
+
limit,
|
|
7758
|
+
...limitAmount !== void 0 ? { limitAmount } : {},
|
|
7759
|
+
...limitTypeFrom(`${name} ${limit}`) ? { limitType: limitTypeFrom(`${name} ${limit}`) } : {},
|
|
7760
|
+
limitValueType: limitValueTypeFrom(limit),
|
|
7761
|
+
...deductible && !/^nil|none$/i.test(deductible.trim()) ? { deductible } : {},
|
|
7762
|
+
...deductibleAmount !== void 0 ? { deductibleAmount } : {},
|
|
7763
|
+
...deductible ? { deductibleValueType: limitValueTypeFrom(deductible) } : {},
|
|
7764
|
+
...basis && /claims[- ]made/i.test(basis) ? { trigger: "claims_made" } : {},
|
|
7765
|
+
...retroactiveDate ? { retroactiveDate } : {},
|
|
7766
|
+
...span.formNumber ? { formNumber: span.formNumber } : {},
|
|
7767
|
+
...page ? { pageNumber: page } : {},
|
|
7768
|
+
sectionRef: span.sectionId ?? "SCHEDULE",
|
|
7769
|
+
originalContent: rowText,
|
|
7770
|
+
sourceSpanIds: [span.id],
|
|
7771
|
+
sourceTextHash: span.textHash ?? span.hash
|
|
7772
|
+
};
|
|
7773
|
+
}
|
|
7774
|
+
function coverageKey(coverage) {
|
|
7775
|
+
return [
|
|
7776
|
+
textValue(coverage.name) ?? "",
|
|
7777
|
+
textValue(coverage.limit) ?? "",
|
|
7778
|
+
textValue(coverage.limitType) ?? "",
|
|
7779
|
+
numberValue2(coverage.pageNumber) ?? ""
|
|
7780
|
+
].map((part) => normalize(String(part))).join("|");
|
|
7781
|
+
}
|
|
7782
|
+
function rowMatchesExisting(row, existing) {
|
|
7783
|
+
const rowKey = coverageKey(row);
|
|
7784
|
+
const rowName = normalize(textValue(row.name) ?? "");
|
|
7785
|
+
const rowLimit = normalize(textValue(row.limit) ?? "");
|
|
7786
|
+
return existing.some((coverage) => {
|
|
7787
|
+
if (coverageKey(coverage) === rowKey) return true;
|
|
7788
|
+
const name = normalize(textValue(coverage.name) ?? "");
|
|
7789
|
+
const limit = normalize(textValue(coverage.limit) ?? "");
|
|
7790
|
+
return Boolean(rowName && rowLimit && name === rowName && limit === rowLimit);
|
|
7791
|
+
});
|
|
7792
|
+
}
|
|
7793
|
+
function recoverCoverageScheduleRows(params) {
|
|
7794
|
+
const payload = params.memory.get("coverage_limits");
|
|
7795
|
+
const existing = Array.isArray(payload?.coverages) ? payload.coverages : [];
|
|
7796
|
+
const pages = coveragePages(params.pageAssignments);
|
|
7797
|
+
const candidates = params.sourceSpans.filter((span) => sourceUnit4(span) === "table_row").filter((span) => {
|
|
7798
|
+
const page = pageNumber(span);
|
|
7799
|
+
return page !== void 0 && pages.has(page);
|
|
7800
|
+
}).map(coverageFromRow).filter((coverage) => Boolean(coverage));
|
|
7801
|
+
const recovered = [];
|
|
7802
|
+
for (const coverage of candidates) {
|
|
7803
|
+
if (rowMatchesExisting(coverage, [...existing, ...recovered])) continue;
|
|
7804
|
+
recovered.push(coverage);
|
|
7805
|
+
}
|
|
7806
|
+
if (recovered.length > 0) {
|
|
7807
|
+
params.memory.set("coverage_limits", {
|
|
7808
|
+
...payload ?? {},
|
|
7809
|
+
coverages: [...existing, ...recovered]
|
|
7810
|
+
});
|
|
7811
|
+
}
|
|
7812
|
+
return {
|
|
7813
|
+
recovered,
|
|
7814
|
+
missingSourceRows: recovered.map((coverage) => {
|
|
7815
|
+
const id = Array.isArray(coverage.sourceSpanIds) ? coverage.sourceSpanIds[0] : void 0;
|
|
7816
|
+
return params.sourceSpans.find((span) => span.id === id);
|
|
7817
|
+
}).filter((span) => Boolean(span))
|
|
7818
|
+
};
|
|
7819
|
+
}
|
|
7820
|
+
|
|
7051
7821
|
// src/extraction/focused-dispatch.ts
|
|
7052
7822
|
async function runFocusedExtractorWithFallback(params) {
|
|
7053
7823
|
const {
|
|
@@ -7250,6 +8020,29 @@ function buildExtractionReviewReport(params) {
|
|
|
7250
8020
|
addMissingSourceGroundingIssues(deterministicIssues, "definitions", "definitions", definitions, "term");
|
|
7251
8021
|
addMissingSourceGroundingIssues(deterministicIssues, "covered_reasons", "coveredReasons", coveredReasons, "name");
|
|
7252
8022
|
}
|
|
8023
|
+
if (params.sourceSpans?.length) {
|
|
8024
|
+
const tempCoveragePayload = {
|
|
8025
|
+
...memory.get("coverage_limits") ?? {},
|
|
8026
|
+
coverages: coverages.map((coverage) => ({ ...coverage }))
|
|
8027
|
+
};
|
|
8028
|
+
const tempMemory = new Map(memory);
|
|
8029
|
+
tempMemory.set("coverage_limits", tempCoveragePayload);
|
|
8030
|
+
const scheduleRecovery = recoverCoverageScheduleRows({
|
|
8031
|
+
memory: tempMemory,
|
|
8032
|
+
sourceSpans: params.sourceSpans,
|
|
8033
|
+
pageAssignments: params.pageAssignments
|
|
8034
|
+
});
|
|
8035
|
+
for (const recovered of scheduleRecovery.recovered) {
|
|
8036
|
+
deterministicIssues.push({
|
|
8037
|
+
code: "coverage_schedule_row_missing",
|
|
8038
|
+
severity: "blocking",
|
|
8039
|
+
message: `Coverage schedule row "${String(recovered.name ?? "unknown")}" is present in source table evidence but missing from extracted coverages.`,
|
|
8040
|
+
extractorName: "coverage_limits",
|
|
8041
|
+
pageNumber: typeof recovered.pageNumber === "number" ? recovered.pageNumber : void 0,
|
|
8042
|
+
itemName: typeof recovered.name === "string" ? recovered.name : void 0
|
|
8043
|
+
});
|
|
8044
|
+
}
|
|
8045
|
+
}
|
|
7253
8046
|
if (mappedDefinitions && definitions.length === 0) {
|
|
7254
8047
|
deterministicIssues.push({
|
|
7255
8048
|
code: "definitions_mapped_but_empty",
|
|
@@ -7676,19 +8469,22 @@ var ARRAY_PATHS = [
|
|
|
7676
8469
|
{ memoryKey: "sections", arrayKeys: ["sections"] },
|
|
7677
8470
|
{ memoryKey: "definitions", arrayKeys: ["definitions"] },
|
|
7678
8471
|
{ memoryKey: "covered_reasons", arrayKeys: ["coveredReasons", "covered_reasons"] },
|
|
7679
|
-
{ memoryKey: "declarations", arrayKeys: ["fields"] }
|
|
8472
|
+
{ memoryKey: "declarations", arrayKeys: ["fields"] },
|
|
8473
|
+
{ memoryKey: "premium_breakdown", arrayKeys: ["premiumBreakdown", "taxesAndFees"] },
|
|
8474
|
+
{ memoryKey: "supplementary", arrayKeys: ["auxiliaryFacts", "supplementaryFacts"] },
|
|
8475
|
+
{ memoryKey: "form_inventory", arrayKeys: ["forms"] }
|
|
7680
8476
|
];
|
|
7681
|
-
function
|
|
8477
|
+
function normalize2(value) {
|
|
7682
8478
|
return value.replace(/\s+/g, " ").trim().toLowerCase();
|
|
7683
8479
|
}
|
|
7684
|
-
function
|
|
8480
|
+
function textValue2(record, ...keys) {
|
|
7685
8481
|
for (const key of keys) {
|
|
7686
8482
|
const value = record[key];
|
|
7687
8483
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
7688
8484
|
}
|
|
7689
8485
|
return void 0;
|
|
7690
8486
|
}
|
|
7691
|
-
function
|
|
8487
|
+
function numberValue3(record, ...keys) {
|
|
7692
8488
|
for (const key of keys) {
|
|
7693
8489
|
const value = record[key];
|
|
7694
8490
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
@@ -7705,37 +8501,73 @@ function pageOverlaps(recordStart, recordEnd, span) {
|
|
|
7705
8501
|
return start <= (spanEnd ?? spanStart) && end >= spanStart;
|
|
7706
8502
|
}
|
|
7707
8503
|
function formMatches(record, span) {
|
|
7708
|
-
const formNumber =
|
|
8504
|
+
const formNumber = textValue2(record, "formNumber");
|
|
7709
8505
|
if (!formNumber || !span.formNumber) return false;
|
|
7710
|
-
return
|
|
8506
|
+
return normalize2(formNumber) === normalize2(span.formNumber);
|
|
7711
8507
|
}
|
|
7712
8508
|
function textMatches(record, span) {
|
|
7713
|
-
const spanText =
|
|
8509
|
+
const spanText = normalize2(span.text);
|
|
7714
8510
|
const candidates = [
|
|
7715
|
-
|
|
7716
|
-
|
|
7717
|
-
|
|
8511
|
+
textValue2(record, "originalContent", "content", "definition", "value"),
|
|
8512
|
+
textValue2(record, "name", "title", "term", "field", "coverageName"),
|
|
8513
|
+
textValue2(record, "limit", "deductible", "premium")
|
|
7718
8514
|
].filter((value) => !!value && value.length >= 3);
|
|
7719
|
-
return candidates.some((candidate) => spanText.includes(
|
|
8515
|
+
return candidates.some((candidate) => spanText.includes(normalize2(candidate)));
|
|
7720
8516
|
}
|
|
7721
8517
|
function sourceHashFor(spans) {
|
|
7722
8518
|
return spans.map((span) => span.textHash ?? span.hash).filter(Boolean).join(":") || void 0;
|
|
7723
8519
|
}
|
|
8520
|
+
function sourceUnit5(span) {
|
|
8521
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
8522
|
+
}
|
|
8523
|
+
function hierarchyScore(span) {
|
|
8524
|
+
switch (sourceUnit5(span)) {
|
|
8525
|
+
case "table_row":
|
|
8526
|
+
return 5;
|
|
8527
|
+
case "table":
|
|
8528
|
+
return 3;
|
|
8529
|
+
case "section":
|
|
8530
|
+
case "page":
|
|
8531
|
+
return 2;
|
|
8532
|
+
case "table_cell":
|
|
8533
|
+
return -2;
|
|
8534
|
+
default:
|
|
8535
|
+
return 0;
|
|
8536
|
+
}
|
|
8537
|
+
}
|
|
8538
|
+
function parentRowId(span) {
|
|
8539
|
+
return span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
8540
|
+
}
|
|
8541
|
+
function preferParentRows(matches, sourceSpans) {
|
|
8542
|
+
if (matches.length === 0) return matches;
|
|
8543
|
+
const byId = new Map(sourceSpans.map((span) => [span.id, span]));
|
|
8544
|
+
const expanded = [];
|
|
8545
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8546
|
+
for (const match of matches) {
|
|
8547
|
+
const parent = sourceUnit5(match) === "table_cell" ? byId.get(parentRowId(match) ?? "") : void 0;
|
|
8548
|
+
const preferred = parent && sourceUnit5(parent) === "table_row" ? parent : match;
|
|
8549
|
+
if (seen.has(preferred.id)) continue;
|
|
8550
|
+
seen.add(preferred.id);
|
|
8551
|
+
expanded.push(preferred);
|
|
8552
|
+
}
|
|
8553
|
+
return expanded;
|
|
8554
|
+
}
|
|
7724
8555
|
function findSourceSpansForRecord(record, sourceSpans) {
|
|
7725
8556
|
if (sourceSpans.length === 0) return [];
|
|
7726
|
-
const pageStart =
|
|
7727
|
-
const pageEnd =
|
|
8557
|
+
const pageStart = numberValue3(record, "pageNumber", "pageStart");
|
|
8558
|
+
const pageEnd = numberValue3(record, "pageNumber", "pageEnd");
|
|
7728
8559
|
const scored = sourceSpans.map((span) => {
|
|
7729
8560
|
let score = 0;
|
|
7730
8561
|
if (pageOverlaps(pageStart, pageEnd, span)) score += 4;
|
|
7731
8562
|
if (formMatches(record, span)) score += 3;
|
|
7732
8563
|
if (textMatches(record, span)) score += 2;
|
|
8564
|
+
if (score > 0) score += hierarchyScore(span);
|
|
7733
8565
|
return { span, score };
|
|
7734
8566
|
}).filter((item) => item.score >= 2).sort((left, right) => {
|
|
7735
8567
|
if (right.score !== left.score) return right.score - left.score;
|
|
7736
8568
|
return left.span.id.localeCompare(right.span.id);
|
|
7737
8569
|
});
|
|
7738
|
-
return scored.slice(0,
|
|
8570
|
+
return preferParentRows(scored.slice(0, 5).map((item) => item.span), sourceSpans).slice(0, 3);
|
|
7739
8571
|
}
|
|
7740
8572
|
function groundRecord(record, sourceSpans) {
|
|
7741
8573
|
if (Array.isArray(record.sourceSpanIds) && record.sourceSpanIds.length > 0 && record.sourceTextHash) {
|
|
@@ -8443,6 +9275,14 @@ ${pageText}`;
|
|
|
8443
9275
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8444
9276
|
}
|
|
8445
9277
|
}
|
|
9278
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
9279
|
+
memory,
|
|
9280
|
+
sourceSpans,
|
|
9281
|
+
pageAssignments
|
|
9282
|
+
});
|
|
9283
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
9284
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from table evidence`);
|
|
9285
|
+
}
|
|
8446
9286
|
const planIncludesSupplementary = tasks.some((task) => task.extractorName === "supplementary");
|
|
8447
9287
|
if (!planIncludesSupplementary && hasSupplementaryExtractionSignal(pageAssignments, formInventory, memory)) {
|
|
8448
9288
|
onProgress?.("Extracting supplementary retrieval facts...");
|
|
@@ -8539,7 +9379,8 @@ ${pageText}`;
|
|
|
8539
9379
|
memory,
|
|
8540
9380
|
pageAssignments,
|
|
8541
9381
|
reviewRounds,
|
|
8542
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9382
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9383
|
+
sourceSpans
|
|
8543
9384
|
});
|
|
8544
9385
|
if (shouldRunLlmReview(reviewMode, preReviewReport, sourceSpans.length > 0)) {
|
|
8545
9386
|
for (let round = 0; round < maxReviewRounds; round++) {
|
|
@@ -8608,6 +9449,14 @@ ${pageText}`;
|
|
|
8608
9449
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8609
9450
|
}
|
|
8610
9451
|
}
|
|
9452
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
9453
|
+
memory,
|
|
9454
|
+
sourceSpans,
|
|
9455
|
+
pageAssignments
|
|
9456
|
+
});
|
|
9457
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
9458
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from follow-up table evidence`);
|
|
9459
|
+
}
|
|
8611
9460
|
}
|
|
8612
9461
|
} else {
|
|
8613
9462
|
onProgress?.("Skipping LLM extraction review; deterministic checks passed.");
|
|
@@ -8617,7 +9466,8 @@ ${pageText}`;
|
|
|
8617
9466
|
memory,
|
|
8618
9467
|
pageAssignments,
|
|
8619
9468
|
reviewRounds,
|
|
8620
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9469
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9470
|
+
sourceSpans
|
|
8621
9471
|
});
|
|
8622
9472
|
if (reviewReport.issues.length > 0) {
|
|
8623
9473
|
await log?.(
|
|
@@ -8643,10 +9493,12 @@ ${pageText}`;
|
|
|
8643
9493
|
memory,
|
|
8644
9494
|
pageAssignments,
|
|
8645
9495
|
reviewRounds,
|
|
8646
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9496
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9497
|
+
sourceSpans
|
|
8647
9498
|
}));
|
|
8648
9499
|
onProgress?.("Assembling document...");
|
|
8649
9500
|
const document = assembleDocument(id, documentType, memory);
|
|
9501
|
+
attachDocumentStructure({ document, pageAssignments, sourceSpans });
|
|
8650
9502
|
await pipelineCtx.save("assemble", {
|
|
8651
9503
|
id,
|
|
8652
9504
|
pageCount,
|
|
@@ -12445,6 +13297,11 @@ var AGENT_TOOLS = [
|
|
|
12445
13297
|
DeductibleTypeSchema,
|
|
12446
13298
|
DefenseCostTreatmentSchema,
|
|
12447
13299
|
DefinitionSchema,
|
|
13300
|
+
DocumentAgentGuidanceSchema,
|
|
13301
|
+
DocumentMetadataSchema,
|
|
13302
|
+
DocumentNodeSchema,
|
|
13303
|
+
DocumentPageMapEntrySchema,
|
|
13304
|
+
DocumentTableOfContentsEntrySchema,
|
|
12448
13305
|
DocumentTypeSchema,
|
|
12449
13306
|
DriverRecordSchema,
|
|
12450
13307
|
DwellingDetailsSchema,
|
|
@@ -12566,6 +13423,8 @@ var AGENT_TOOLS = [
|
|
|
12566
13423
|
SourceSpanLocationSchema,
|
|
12567
13424
|
SourceSpanRefSchema,
|
|
12568
13425
|
SourceSpanSchema,
|
|
13426
|
+
SourceSpanTableLocationSchema,
|
|
13427
|
+
SourceSpanUnitSchema,
|
|
12569
13428
|
SubAnswerSchema,
|
|
12570
13429
|
SubQuestionSchema,
|
|
12571
13430
|
SubjectivityCategorySchema,
|