@claritylabs/cl-sdk 1.3.6 → 1.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.d.mts +185 -47
- package/dist/index.d.ts +185 -47
- package/dist/index.js +569 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +567 -55
- package/dist/index.mjs.map +1 -1
- package/dist/storage-sqlite.d.mts +89 -20
- package/dist/storage-sqlite.d.ts +89 -20
- package/dist/storage-sqlite.js +33 -2
- package/dist/storage-sqlite.js.map +1 -1
- package/dist/storage-sqlite.mjs +33 -2
- package/dist/storage-sqlite.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -224,6 +224,8 @@ __export(index_exports, {
|
|
|
224
224
|
SourceSpanLocationSchema: () => SourceSpanLocationSchema,
|
|
225
225
|
SourceSpanRefSchema: () => SourceSpanRefSchema,
|
|
226
226
|
SourceSpanSchema: () => SourceSpanSchema,
|
|
227
|
+
SourceSpanTableLocationSchema: () => SourceSpanTableLocationSchema,
|
|
228
|
+
SourceSpanUnitSchema: () => SourceSpanUnitSchema,
|
|
227
229
|
SubAnswerSchema: () => SubAnswerSchema,
|
|
228
230
|
SubQuestionSchema: () => SubQuestionSchema,
|
|
229
231
|
SubjectivityCategorySchema: () => SubjectivityCategorySchema,
|
|
@@ -959,6 +961,8 @@ var CoverageSchema = import_zod4.z.object({
|
|
|
959
961
|
deductible: import_zod4.z.string().optional(),
|
|
960
962
|
deductibleAmount: import_zod4.z.number().optional(),
|
|
961
963
|
deductibleValueType: CoverageValueTypeSchema.optional(),
|
|
964
|
+
trigger: CoverageTriggerSchema.optional(),
|
|
965
|
+
retroactiveDate: import_zod4.z.string().optional(),
|
|
962
966
|
formNumber: import_zod4.z.string().optional(),
|
|
963
967
|
pageNumber: import_zod4.z.number().optional(),
|
|
964
968
|
sectionRef: import_zod4.z.string().optional(),
|
|
@@ -2187,6 +2191,15 @@ var SourceSpanKindSchema = import_zod20.z.enum([
|
|
|
2187
2191
|
"plain_text",
|
|
2188
2192
|
"structured_field"
|
|
2189
2193
|
]);
|
|
2194
|
+
var SourceSpanUnitSchema = import_zod20.z.enum([
|
|
2195
|
+
"page",
|
|
2196
|
+
"section",
|
|
2197
|
+
"table",
|
|
2198
|
+
"table_row",
|
|
2199
|
+
"table_cell",
|
|
2200
|
+
"key_value",
|
|
2201
|
+
"text"
|
|
2202
|
+
]);
|
|
2190
2203
|
var SourceKindSchema = import_zod20.z.enum([
|
|
2191
2204
|
"policy_pdf",
|
|
2192
2205
|
"application_pdf",
|
|
@@ -2211,6 +2224,15 @@ var SourceSpanLocationSchema = import_zod20.z.object({
|
|
|
2211
2224
|
lineEnd: import_zod20.z.number().int().positive().optional(),
|
|
2212
2225
|
fieldPath: import_zod20.z.string().optional()
|
|
2213
2226
|
});
|
|
2227
|
+
var SourceSpanTableLocationSchema = import_zod20.z.object({
|
|
2228
|
+
tableId: import_zod20.z.string().optional(),
|
|
2229
|
+
rowIndex: import_zod20.z.number().int().nonnegative().optional(),
|
|
2230
|
+
columnIndex: import_zod20.z.number().int().nonnegative().optional(),
|
|
2231
|
+
columnName: import_zod20.z.string().optional(),
|
|
2232
|
+
rowSpanId: import_zod20.z.string().optional(),
|
|
2233
|
+
tableSpanId: import_zod20.z.string().optional(),
|
|
2234
|
+
isHeader: import_zod20.z.boolean().optional()
|
|
2235
|
+
});
|
|
2214
2236
|
var SourceSpanSchema = import_zod20.z.object({
|
|
2215
2237
|
id: import_zod20.z.string().min(1),
|
|
2216
2238
|
documentId: import_zod20.z.string().min(1),
|
|
@@ -2224,6 +2246,9 @@ var SourceSpanSchema = import_zod20.z.object({
|
|
|
2224
2246
|
pageEnd: import_zod20.z.number().int().positive().optional(),
|
|
2225
2247
|
sectionId: import_zod20.z.string().optional(),
|
|
2226
2248
|
formNumber: import_zod20.z.string().optional(),
|
|
2249
|
+
sourceUnit: SourceSpanUnitSchema.optional(),
|
|
2250
|
+
parentSpanId: import_zod20.z.string().optional(),
|
|
2251
|
+
table: SourceSpanTableLocationSchema.optional(),
|
|
2227
2252
|
bbox: import_zod20.z.array(SourceSpanBBoxSchema).optional(),
|
|
2228
2253
|
location: SourceSpanLocationSchema.optional(),
|
|
2229
2254
|
metadata: import_zod20.z.record(import_zod20.z.string(), import_zod20.z.string()).optional()
|
|
@@ -2342,6 +2367,9 @@ function buildSourceSpan(input, localIndex = 0) {
|
|
|
2342
2367
|
pageEnd: input.pageEnd,
|
|
2343
2368
|
sectionId: input.sectionId,
|
|
2344
2369
|
formNumber: input.formNumber,
|
|
2370
|
+
sourceUnit: input.sourceUnit,
|
|
2371
|
+
parentSpanId: input.parentSpanId,
|
|
2372
|
+
table: input.table,
|
|
2345
2373
|
location: {
|
|
2346
2374
|
page: input.pageStart === input.pageEnd ? input.pageStart : void 0,
|
|
2347
2375
|
startPage: input.pageStart,
|
|
@@ -2362,7 +2390,11 @@ function buildPageSourceSpans(pages) {
|
|
|
2362
2390
|
pageEnd: page.pageNumber,
|
|
2363
2391
|
sectionId: page.sectionId,
|
|
2364
2392
|
formNumber: page.formNumber,
|
|
2365
|
-
|
|
2393
|
+
sourceUnit: "page",
|
|
2394
|
+
metadata: {
|
|
2395
|
+
...page.metadata ?? {},
|
|
2396
|
+
sourceUnit: page.metadata?.sourceUnit ?? "page"
|
|
2397
|
+
}
|
|
2366
2398
|
},
|
|
2367
2399
|
index
|
|
2368
2400
|
)
|
|
@@ -2384,6 +2416,7 @@ function buildSectionSourceSpans(pages, options = {}) {
|
|
|
2384
2416
|
pageEnd: page.pageNumber,
|
|
2385
2417
|
sectionId: section.title,
|
|
2386
2418
|
formNumber: inferFormNumber(section.text),
|
|
2419
|
+
sourceUnit: "section",
|
|
2387
2420
|
metadata: {
|
|
2388
2421
|
...page.metadata ?? {},
|
|
2389
2422
|
sourceUnit: "section_candidate"
|
|
@@ -2416,6 +2449,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2416
2449
|
const chunks = [];
|
|
2417
2450
|
let current = [];
|
|
2418
2451
|
let currentLength = 0;
|
|
2452
|
+
const spansForChunking = filterChunkableSourceSpans(spans);
|
|
2419
2453
|
const flush = () => {
|
|
2420
2454
|
if (current.length === 0) return;
|
|
2421
2455
|
const text = current.map((span) => span.text).join("\n\n");
|
|
@@ -2439,7 +2473,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2439
2473
|
current = [];
|
|
2440
2474
|
currentLength = 0;
|
|
2441
2475
|
};
|
|
2442
|
-
for (const span of
|
|
2476
|
+
for (const span of spansForChunking) {
|
|
2443
2477
|
const nextLength = currentLength + span.text.length + (current.length > 0 ? 2 : 0);
|
|
2444
2478
|
if (current.length > 0 && nextLength > maxChars) {
|
|
2445
2479
|
flush();
|
|
@@ -2450,6 +2484,20 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2450
2484
|
flush();
|
|
2451
2485
|
return chunks;
|
|
2452
2486
|
}
|
|
2487
|
+
function sourceUnit(span) {
|
|
2488
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
2489
|
+
}
|
|
2490
|
+
function filterChunkableSourceSpans(spans) {
|
|
2491
|
+
const rowIds = new Set(
|
|
2492
|
+
spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
|
|
2493
|
+
);
|
|
2494
|
+
if (rowIds.size === 0) return spans;
|
|
2495
|
+
return spans.filter((span) => {
|
|
2496
|
+
if (sourceUnit(span) !== "table_cell") return true;
|
|
2497
|
+
const rowId = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
2498
|
+
return !rowId || !rowIds.has(rowId);
|
|
2499
|
+
});
|
|
2500
|
+
}
|
|
2453
2501
|
function splitPageIntoSections(text, headingPattern, minSectionChars) {
|
|
2454
2502
|
const lines = text.split(/\r?\n/);
|
|
2455
2503
|
const sections = [];
|
|
@@ -2834,36 +2882,18 @@ function normalizeDoclingDocument(document, options) {
|
|
|
2834
2882
|
pageTexts.set(page, appendText(pageTexts.get(page), unit.text));
|
|
2835
2883
|
}
|
|
2836
2884
|
const fullText = Array.from({ length: pageCount }, (_, index) => {
|
|
2837
|
-
const
|
|
2838
|
-
const text = pageTexts.get(
|
|
2839
|
-
return text ? `Page ${
|
|
2885
|
+
const pageNumber2 = index + 1;
|
|
2886
|
+
const text = pageTexts.get(pageNumber2)?.trim();
|
|
2887
|
+
return text ? `Page ${pageNumber2}
|
|
2840
2888
|
${text}` : "";
|
|
2841
2889
|
}).filter(Boolean).join("\n\n");
|
|
2842
2890
|
const sourceKind = options.sourceKind ?? "policy_pdf";
|
|
2843
|
-
const sourceSpans = units.
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
pageStart: unit.pageStart,
|
|
2850
|
-
pageEnd: unit.pageEnd,
|
|
2851
|
-
sectionId: unit.label,
|
|
2852
|
-
metadata: {
|
|
2853
|
-
sourceSystem: "docling",
|
|
2854
|
-
sourceUnit: "docling_item",
|
|
2855
|
-
doclingRef: unit.ref,
|
|
2856
|
-
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2857
|
-
}
|
|
2858
|
-
},
|
|
2859
|
-
index
|
|
2860
|
-
);
|
|
2861
|
-
return {
|
|
2862
|
-
...span,
|
|
2863
|
-
kind: "plain_text",
|
|
2864
|
-
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2865
|
-
};
|
|
2866
|
-
});
|
|
2891
|
+
const sourceSpans = units.flatMap(
|
|
2892
|
+
(unit, index) => buildSourceSpansForUnit(unit, index, {
|
|
2893
|
+
documentId: options.documentId,
|
|
2894
|
+
sourceKind
|
|
2895
|
+
})
|
|
2896
|
+
);
|
|
2867
2897
|
return {
|
|
2868
2898
|
pageCount,
|
|
2869
2899
|
fullText,
|
|
@@ -2901,6 +2931,11 @@ function mergeSourceSpans(spans) {
|
|
|
2901
2931
|
span.pageStart ?? span.location?.startPage ?? span.location?.page ?? "na",
|
|
2902
2932
|
span.pageEnd ?? span.location?.endPage ?? span.pageStart ?? "na",
|
|
2903
2933
|
span.sectionId ?? span.location?.fieldPath ?? "na",
|
|
2934
|
+
span.sourceUnit ?? span.metadata?.sourceUnit ?? "na",
|
|
2935
|
+
span.parentSpanId ?? "na",
|
|
2936
|
+
span.table?.tableId ?? span.metadata?.tableId ?? "na",
|
|
2937
|
+
span.table?.rowIndex ?? span.metadata?.rowIndex ?? "na",
|
|
2938
|
+
span.table?.columnIndex ?? span.metadata?.columnIndex ?? "na",
|
|
2904
2939
|
span.textHash ?? sourceSpanTextHash(span.text)
|
|
2905
2940
|
].join(":");
|
|
2906
2941
|
if (seen.has(key)) continue;
|
|
@@ -2963,6 +2998,7 @@ function getOrderedBodyRefs(document, itemMap) {
|
|
|
2963
2998
|
function normalizeItem(ref, item) {
|
|
2964
2999
|
const text = getItemText(item).trim();
|
|
2965
3000
|
if (!text) return void 0;
|
|
3001
|
+
const table = getItemTable(item);
|
|
2966
3002
|
const pages = (item.prov ?? []).map((prov) => getPageNumber(prov)).filter((page) => typeof page === "number" && page > 0);
|
|
2967
3003
|
const pageStart = pages.length ? Math.min(...pages) : void 0;
|
|
2968
3004
|
const pageEnd = pages.length ? Math.max(...pages) : pageStart;
|
|
@@ -2973,17 +3009,143 @@ function normalizeItem(ref, item) {
|
|
|
2973
3009
|
text,
|
|
2974
3010
|
pageStart,
|
|
2975
3011
|
pageEnd,
|
|
2976
|
-
bboxes: bboxes.length ? bboxes : void 0
|
|
3012
|
+
bboxes: bboxes.length ? bboxes : void 0,
|
|
3013
|
+
table
|
|
3014
|
+
};
|
|
3015
|
+
}
|
|
3016
|
+
function buildSourceSpansForUnit(unit, index, options) {
|
|
3017
|
+
const baseMetadata = {
|
|
3018
|
+
sourceSystem: "docling",
|
|
3019
|
+
sourceUnit: unit.table ? "table" : "docling_item",
|
|
3020
|
+
doclingRef: unit.ref,
|
|
3021
|
+
...unit.label ? { doclingLabel: unit.label } : {}
|
|
3022
|
+
};
|
|
3023
|
+
const tableId = unit.table ? `${unit.ref}:table` : void 0;
|
|
3024
|
+
const tableSpan = withDoclingKind(buildSourceSpan(
|
|
3025
|
+
{
|
|
3026
|
+
documentId: options.documentId,
|
|
3027
|
+
sourceKind: options.sourceKind,
|
|
3028
|
+
text: unit.text,
|
|
3029
|
+
pageStart: unit.pageStart,
|
|
3030
|
+
pageEnd: unit.pageEnd,
|
|
3031
|
+
sectionId: unit.label,
|
|
3032
|
+
sourceUnit: unit.table ? "table" : labelToSourceUnit(unit.label),
|
|
3033
|
+
table: tableId ? { tableId } : void 0,
|
|
3034
|
+
metadata: baseMetadata
|
|
3035
|
+
},
|
|
3036
|
+
index * 1e3
|
|
3037
|
+
), unit);
|
|
3038
|
+
if (!unit.table) return [tableSpan];
|
|
3039
|
+
const spans = [tableSpan];
|
|
3040
|
+
const table = unit.table;
|
|
3041
|
+
for (let rowIndex = 0; rowIndex < table.rows.length; rowIndex += 1) {
|
|
3042
|
+
const row = table.rows[rowIndex];
|
|
3043
|
+
const isHeader = rowIndex === 0 && table.headers.length > 0;
|
|
3044
|
+
const rowText = tableRowText(table, rowIndex);
|
|
3045
|
+
if (!rowText) continue;
|
|
3046
|
+
const rowSpan = withDoclingKind(buildSourceSpan(
|
|
3047
|
+
{
|
|
3048
|
+
documentId: options.documentId,
|
|
3049
|
+
sourceKind: options.sourceKind,
|
|
3050
|
+
text: rowText,
|
|
3051
|
+
pageStart: unit.pageStart,
|
|
3052
|
+
pageEnd: unit.pageEnd,
|
|
3053
|
+
sectionId: unit.label,
|
|
3054
|
+
sourceUnit: "table_row",
|
|
3055
|
+
parentSpanId: tableSpan.id,
|
|
3056
|
+
table: {
|
|
3057
|
+
tableId,
|
|
3058
|
+
tableSpanId: tableSpan.id,
|
|
3059
|
+
rowIndex,
|
|
3060
|
+
isHeader
|
|
3061
|
+
},
|
|
3062
|
+
metadata: {
|
|
3063
|
+
...baseMetadata,
|
|
3064
|
+
sourceUnit: "table_row",
|
|
3065
|
+
tableId: tableId ?? "",
|
|
3066
|
+
tableSpanId: tableSpan.id,
|
|
3067
|
+
rowIndex: String(rowIndex),
|
|
3068
|
+
isHeader: String(isHeader)
|
|
3069
|
+
}
|
|
3070
|
+
},
|
|
3071
|
+
index * 1e3 + 1 + rowIndex
|
|
3072
|
+
), unit);
|
|
3073
|
+
spans.push(rowSpan);
|
|
3074
|
+
for (let columnIndex = 0; columnIndex < row.length; columnIndex += 1) {
|
|
3075
|
+
const text = row[columnIndex]?.trim();
|
|
3076
|
+
if (!text) continue;
|
|
3077
|
+
const columnName = table.headers[columnIndex]?.trim() || void 0;
|
|
3078
|
+
const cellSpan = withDoclingKind(buildSourceSpan(
|
|
3079
|
+
{
|
|
3080
|
+
documentId: options.documentId,
|
|
3081
|
+
sourceKind: options.sourceKind,
|
|
3082
|
+
text,
|
|
3083
|
+
pageStart: unit.pageStart,
|
|
3084
|
+
pageEnd: unit.pageEnd,
|
|
3085
|
+
sectionId: unit.label,
|
|
3086
|
+
sourceUnit: "table_cell",
|
|
3087
|
+
parentSpanId: rowSpan.id,
|
|
3088
|
+
table: {
|
|
3089
|
+
tableId,
|
|
3090
|
+
tableSpanId: tableSpan.id,
|
|
3091
|
+
rowSpanId: rowSpan.id,
|
|
3092
|
+
rowIndex,
|
|
3093
|
+
columnIndex,
|
|
3094
|
+
columnName,
|
|
3095
|
+
isHeader
|
|
3096
|
+
},
|
|
3097
|
+
metadata: {
|
|
3098
|
+
...baseMetadata,
|
|
3099
|
+
sourceUnit: "table_cell",
|
|
3100
|
+
tableId: tableId ?? "",
|
|
3101
|
+
tableSpanId: tableSpan.id,
|
|
3102
|
+
rowSpanId: rowSpan.id,
|
|
3103
|
+
rowIndex: String(rowIndex),
|
|
3104
|
+
columnIndex: String(columnIndex),
|
|
3105
|
+
...columnName ? { columnName } : {},
|
|
3106
|
+
isHeader: String(isHeader)
|
|
3107
|
+
}
|
|
3108
|
+
},
|
|
3109
|
+
index * 1e3 + 100 + rowIndex * 100 + columnIndex
|
|
3110
|
+
), unit);
|
|
3111
|
+
spans.push(cellSpan);
|
|
3112
|
+
}
|
|
3113
|
+
}
|
|
3114
|
+
return spans;
|
|
3115
|
+
}
|
|
3116
|
+
function withDoclingKind(span, unit) {
|
|
3117
|
+
return {
|
|
3118
|
+
...span,
|
|
3119
|
+
kind: "plain_text",
|
|
3120
|
+
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2977
3121
|
};
|
|
2978
3122
|
}
|
|
3123
|
+
function labelToSourceUnit(label) {
|
|
3124
|
+
const normalized = label?.toLowerCase() ?? "";
|
|
3125
|
+
if (normalized.includes("table")) return "table";
|
|
3126
|
+
if (normalized.includes("key")) return "key_value";
|
|
3127
|
+
if (normalized.includes("section") || normalized.includes("header")) return "section";
|
|
3128
|
+
return "text";
|
|
3129
|
+
}
|
|
3130
|
+
function tableRowText(table, rowIndex) {
|
|
3131
|
+
const row = table.rows[rowIndex] ?? [];
|
|
3132
|
+
const headers = table.headers;
|
|
3133
|
+
if (rowIndex === 0 && headers.length > 0) return row.filter(Boolean).join(" | ");
|
|
3134
|
+
return row.map((value, columnIndex) => {
|
|
3135
|
+
const trimmed = value.trim();
|
|
3136
|
+
if (!trimmed) return "";
|
|
3137
|
+
const header = headers[columnIndex]?.trim();
|
|
3138
|
+
return header ? `${header}: ${trimmed}` : trimmed;
|
|
3139
|
+
}).filter(Boolean).join(" | ");
|
|
3140
|
+
}
|
|
2979
3141
|
function getItemText(item) {
|
|
2980
3142
|
if (typeof item.text === "string" && item.text.trim()) return item.text;
|
|
2981
3143
|
if (typeof item.orig === "string" && item.orig.trim()) return item.orig;
|
|
2982
|
-
const table =
|
|
2983
|
-
if (table) return table;
|
|
3144
|
+
const table = parseTableData(item.data);
|
|
3145
|
+
if (table) return table.markdown;
|
|
2984
3146
|
return "";
|
|
2985
3147
|
}
|
|
2986
|
-
function
|
|
3148
|
+
function parseTableData(data) {
|
|
2987
3149
|
const record = asRecord(data);
|
|
2988
3150
|
const cells = Array.isArray(record?.table_cells) ? record.table_cells : Array.isArray(record?.tableCells) ? record.tableCells : void 0;
|
|
2989
3151
|
if (!cells) return void 0;
|
|
@@ -2999,10 +3161,20 @@ function tableToMarkdown(data) {
|
|
|
2999
3161
|
for (const cell of parsedCells) {
|
|
3000
3162
|
rows[cell.row][cell.col] = cell.text;
|
|
3001
3163
|
}
|
|
3002
|
-
if (rows.length === 1)
|
|
3164
|
+
if (rows.length === 1) {
|
|
3165
|
+
const markdown2 = rows[0].filter(Boolean).join(" | ");
|
|
3166
|
+
return { markdown: markdown2, headers: [], rows, cells: parsedCells };
|
|
3167
|
+
}
|
|
3003
3168
|
const header = rows[0];
|
|
3004
3169
|
const separator = header.map(() => "---");
|
|
3005
|
-
|
|
3170
|
+
const markdown = [header, separator, ...rows.slice(1)].map((row) => `| ${row.map((value) => value.trim()).join(" | ")} |`).join("\n");
|
|
3171
|
+
return { markdown, headers: header, rows, cells: parsedCells };
|
|
3172
|
+
}
|
|
3173
|
+
function getItemTable(item) {
|
|
3174
|
+
if (typeof item.text === "string" && item.text.trim() || typeof item.orig === "string" && item.orig.trim()) {
|
|
3175
|
+
return void 0;
|
|
3176
|
+
}
|
|
3177
|
+
return parseTableData(item.data);
|
|
3006
3178
|
}
|
|
3007
3179
|
function inferPageCount(document, units) {
|
|
3008
3180
|
const pages = document.pages;
|
|
@@ -3079,8 +3251,18 @@ function buildSourceContext(spans, maxChars = 12e3) {
|
|
|
3079
3251
|
if (spans.length === 0) return "";
|
|
3080
3252
|
const lines = [];
|
|
3081
3253
|
let length = 0;
|
|
3082
|
-
for (const span of spans) {
|
|
3083
|
-
const
|
|
3254
|
+
for (const span of orderSourceSpansForContext(spans)) {
|
|
3255
|
+
const unit = sourceUnit2(span);
|
|
3256
|
+
const table = span.table;
|
|
3257
|
+
const tableContext = [
|
|
3258
|
+
unit ? ` unit:${unit}` : "",
|
|
3259
|
+
table?.tableId ? ` table:${table.tableId}` : "",
|
|
3260
|
+
table?.rowIndex != null ? ` row:${table.rowIndex}` : "",
|
|
3261
|
+
table?.columnIndex != null ? ` col:${table.columnIndex}` : "",
|
|
3262
|
+
table?.columnName ? ` column:${table.columnName}` : "",
|
|
3263
|
+
span.parentSpanId ? ` parent:${span.parentSpanId}` : ""
|
|
3264
|
+
].join("");
|
|
3265
|
+
const header = `[sourceSpan:${span.id}${span.pageStart ? ` page:${span.pageStart}${span.pageEnd && span.pageEnd !== span.pageStart ? `-${span.pageEnd}` : ""}` : ""}${span.sectionId ? ` section:${span.sectionId}` : ""}${span.formNumber ? ` form:${span.formNumber}` : ""}${tableContext}]`;
|
|
3084
3266
|
const text = `${header}
|
|
3085
3267
|
${span.text}`;
|
|
3086
3268
|
if (length + text.length > maxChars && lines.length > 0) break;
|
|
@@ -3094,6 +3276,49 @@ ${lines.join("\n\n")}
|
|
|
3094
3276
|
|
|
3095
3277
|
Use sourceSpan IDs when grounding extracted contractual values.`;
|
|
3096
3278
|
}
|
|
3279
|
+
function sourceUnit2(span) {
|
|
3280
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
3281
|
+
}
|
|
3282
|
+
function sourceContextRank(span) {
|
|
3283
|
+
switch (sourceUnit2(span)) {
|
|
3284
|
+
case "table_row":
|
|
3285
|
+
return 0;
|
|
3286
|
+
case "table":
|
|
3287
|
+
return 1;
|
|
3288
|
+
case "section":
|
|
3289
|
+
return 2;
|
|
3290
|
+
case "page":
|
|
3291
|
+
return 3;
|
|
3292
|
+
case "key_value":
|
|
3293
|
+
return 4;
|
|
3294
|
+
case "table_cell":
|
|
3295
|
+
return 8;
|
|
3296
|
+
default:
|
|
3297
|
+
return 5;
|
|
3298
|
+
}
|
|
3299
|
+
}
|
|
3300
|
+
function orderSourceSpansForContext(spans) {
|
|
3301
|
+
const parentRows = new Set(
|
|
3302
|
+
spans.filter((span) => sourceUnit2(span) === "table_row").map((span) => span.id)
|
|
3303
|
+
);
|
|
3304
|
+
const filtered = spans.filter((span) => {
|
|
3305
|
+
if (sourceUnit2(span) !== "table_cell") return true;
|
|
3306
|
+
const parent = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
3307
|
+
return !parent || !parentRows.has(parent);
|
|
3308
|
+
});
|
|
3309
|
+
return [...filtered].sort((left, right) => {
|
|
3310
|
+
const leftPage = left.pageStart ?? left.location?.startPage ?? left.location?.page ?? 0;
|
|
3311
|
+
const rightPage = right.pageStart ?? right.location?.startPage ?? right.location?.page ?? 0;
|
|
3312
|
+
if (leftPage !== rightPage) return leftPage - rightPage;
|
|
3313
|
+
const leftRank = sourceContextRank(left);
|
|
3314
|
+
const rightRank = sourceContextRank(right);
|
|
3315
|
+
if (leftRank !== rightRank) return leftRank - rightRank;
|
|
3316
|
+
const leftRow = left.table?.rowIndex ?? Number(left.metadata?.rowIndex ?? Number.NaN);
|
|
3317
|
+
const rightRow = right.table?.rowIndex ?? Number(right.metadata?.rowIndex ?? Number.NaN);
|
|
3318
|
+
if (Number.isFinite(leftRow) && Number.isFinite(rightRow) && leftRow !== rightRow) return leftRow - rightRow;
|
|
3319
|
+
return left.id.localeCompare(right.id);
|
|
3320
|
+
});
|
|
3321
|
+
}
|
|
3097
3322
|
async function runExtractor(params) {
|
|
3098
3323
|
const {
|
|
3099
3324
|
name,
|
|
@@ -4692,7 +4917,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4692
4917
|
const merged = mergeShallowPreferPresent(existing, incoming);
|
|
4693
4918
|
const existingCoverages = Array.isArray(existing.coverages) ? existing.coverages : [];
|
|
4694
4919
|
const incomingCoverages = Array.isArray(incoming.coverages) ? incoming.coverages : [];
|
|
4695
|
-
const
|
|
4920
|
+
const coverageKey2 = (coverage) => keyFromParts(
|
|
4696
4921
|
coverage.name,
|
|
4697
4922
|
coverage.limitType,
|
|
4698
4923
|
coverage.limit,
|
|
@@ -4701,7 +4926,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4701
4926
|
);
|
|
4702
4927
|
const byKey = /* @__PURE__ */ new Map();
|
|
4703
4928
|
for (const coverage of [...existingCoverages, ...incomingCoverages]) {
|
|
4704
|
-
const key =
|
|
4929
|
+
const key = coverageKey2(coverage);
|
|
4705
4930
|
const current = byKey.get(key);
|
|
4706
4931
|
byKey.set(key, current ? mergeShallowPreferPresent(current, coverage) : coverage);
|
|
4707
4932
|
}
|
|
@@ -7048,6 +7273,190 @@ async function resolveReferentialCoverages(params) {
|
|
|
7048
7273
|
};
|
|
7049
7274
|
}
|
|
7050
7275
|
|
|
7276
|
+
// src/extraction/coverage-schedule-recovery.ts
|
|
7277
|
+
function textValue(value) {
|
|
7278
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
7279
|
+
}
|
|
7280
|
+
function numberValue(value) {
|
|
7281
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
7282
|
+
}
|
|
7283
|
+
function normalize(value) {
|
|
7284
|
+
return value.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, " ").trim();
|
|
7285
|
+
}
|
|
7286
|
+
function sourceUnit3(span) {
|
|
7287
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
7288
|
+
}
|
|
7289
|
+
function pageNumber(span) {
|
|
7290
|
+
return span.pageStart ?? span.location?.page ?? span.location?.startPage;
|
|
7291
|
+
}
|
|
7292
|
+
function coveragePages(pageAssignments) {
|
|
7293
|
+
const pages = /* @__PURE__ */ new Set();
|
|
7294
|
+
for (const assignment of pageAssignments) {
|
|
7295
|
+
if (assignment.extractorNames.includes("coverage_limits") || assignment.hasScheduleValues) {
|
|
7296
|
+
pages.add(assignment.localPageNumber);
|
|
7297
|
+
}
|
|
7298
|
+
}
|
|
7299
|
+
return pages;
|
|
7300
|
+
}
|
|
7301
|
+
function parseCurrencyAmount(value) {
|
|
7302
|
+
const match = value.match(/(?:CAD|USD|US)?\s*\$?\s*([0-9][0-9,]*(?:\.\d+)?)/i);
|
|
7303
|
+
if (!match) return void 0;
|
|
7304
|
+
const amount = Number(match[1].replace(/,/g, ""));
|
|
7305
|
+
return Number.isFinite(amount) ? amount : void 0;
|
|
7306
|
+
}
|
|
7307
|
+
function limitTypeFrom(value) {
|
|
7308
|
+
const normalized = normalize(value);
|
|
7309
|
+
if (normalized.includes("/") || normalized.includes("each claim") && normalized.includes("aggregate")) return "scheduled";
|
|
7310
|
+
if (normalized.includes("each claim") || normalized.includes("per claim")) return "per_claim";
|
|
7311
|
+
if (normalized.includes("each occurrence") || normalized.includes("per occurrence")) return "per_occurrence";
|
|
7312
|
+
if (normalized.includes("aggregate")) return "aggregate";
|
|
7313
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage")) return "scheduled";
|
|
7314
|
+
if (normalized.includes("/")) return "scheduled";
|
|
7315
|
+
return void 0;
|
|
7316
|
+
}
|
|
7317
|
+
function limitValueTypeFrom(value) {
|
|
7318
|
+
const normalized = normalize(value);
|
|
7319
|
+
if (parseCurrencyAmount(value) !== void 0) return "numeric";
|
|
7320
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage") || normalized.includes("shown above")) {
|
|
7321
|
+
return "referential";
|
|
7322
|
+
}
|
|
7323
|
+
if (normalized.includes("included")) return "included";
|
|
7324
|
+
if (normalized.includes("not included") || normalized === "nil" || normalized === "none") return "not_included";
|
|
7325
|
+
if (normalized.includes("as stated")) return "as_stated";
|
|
7326
|
+
return "other";
|
|
7327
|
+
}
|
|
7328
|
+
function cleanName(value) {
|
|
7329
|
+
return value.replace(/\s*\([^)]*part of and not in addition to[^)]*\)\s*/gi, " ").replace(/\s+/g, " ").trim().replace(/[.:;-]+$/, "").trim();
|
|
7330
|
+
}
|
|
7331
|
+
function isDeductibleOnly(name, rowText) {
|
|
7332
|
+
const normalizedName = normalize(name);
|
|
7333
|
+
const normalizedRow = normalize(rowText);
|
|
7334
|
+
if (!/\b(deductible|retention|sir)\b/.test(`${normalizedName} ${normalizedRow}`)) return false;
|
|
7335
|
+
if (/\b(coverage|sub limit|sublimit|limit)\b/.test(normalizedName) && !/\b(enhanced|standard)\s+deductible\b/.test(normalizedName)) {
|
|
7336
|
+
return false;
|
|
7337
|
+
}
|
|
7338
|
+
return /\b(enhanced|standard)?\s*(deductible|retention|sir)\b/.test(normalizedName) || /^\s*(deductible|retention|sir)\b/.test(normalizedRow);
|
|
7339
|
+
}
|
|
7340
|
+
function splitRowFields(rowText) {
|
|
7341
|
+
return rowText.split(/\s+\|\s+/).map((part) => part.trim()).filter(Boolean).map((part) => {
|
|
7342
|
+
const match = part.match(/^([^:]{1,100}):\s*(.*)$/);
|
|
7343
|
+
if (!match) return { value: part };
|
|
7344
|
+
return { key: match[1].trim(), value: match[2].trim() };
|
|
7345
|
+
});
|
|
7346
|
+
}
|
|
7347
|
+
function firstField(fields, patterns) {
|
|
7348
|
+
for (const field of fields) {
|
|
7349
|
+
const target = `${field.key ?? ""} ${field.value}`;
|
|
7350
|
+
if (patterns.some((pattern) => pattern.test(target))) return field.value || field.key;
|
|
7351
|
+
}
|
|
7352
|
+
return void 0;
|
|
7353
|
+
}
|
|
7354
|
+
function coverageFromRow(span) {
|
|
7355
|
+
const rowText = span.text.trim();
|
|
7356
|
+
if (!rowText || sourceUnit3(span) !== "table_row") return void 0;
|
|
7357
|
+
if (span.table?.isHeader || span.metadata?.isHeader === "true") return void 0;
|
|
7358
|
+
const fields = splitRowFields(rowText);
|
|
7359
|
+
let name;
|
|
7360
|
+
let limit;
|
|
7361
|
+
for (const field of fields) {
|
|
7362
|
+
const key = field.key?.trim();
|
|
7363
|
+
const value = field.value.trim();
|
|
7364
|
+
if (!key || !value) continue;
|
|
7365
|
+
if (!name && /^coverage$/i.test(key)) name = cleanName(value);
|
|
7366
|
+
if (!limit && /\blimit\b/i.test(key)) limit = value;
|
|
7367
|
+
if (!name && /\b(sub[-\s]?limit|aggregate|each claim)\b/i.test(key) && parseCurrencyAmount(value) !== void 0) {
|
|
7368
|
+
name = cleanName(key);
|
|
7369
|
+
limit = limit ?? value;
|
|
7370
|
+
}
|
|
7371
|
+
}
|
|
7372
|
+
if (!name || !limit) {
|
|
7373
|
+
for (const field of fields) {
|
|
7374
|
+
const key = field.key?.trim();
|
|
7375
|
+
const value = field.value.trim();
|
|
7376
|
+
if (!key || !value) continue;
|
|
7377
|
+
if (!name && /\b(sub[-\s]?limit|coverage|aggregate|each claim)\b/i.test(key) && !parseCurrencyAmount(key)) {
|
|
7378
|
+
name = cleanName(key);
|
|
7379
|
+
limit = limit ?? value;
|
|
7380
|
+
}
|
|
7381
|
+
}
|
|
7382
|
+
}
|
|
7383
|
+
if (!name || !limit) return void 0;
|
|
7384
|
+
if (isDeductibleOnly(name, rowText)) return void 0;
|
|
7385
|
+
const normalizedLimit = normalize(limit);
|
|
7386
|
+
const hasUsableLimit = parseCurrencyAmount(limit) !== void 0 || /\b(shared within|within coverage|as stated|included|not included)\b/i.test(normalizedLimit);
|
|
7387
|
+
if (!hasUsableLimit) return void 0;
|
|
7388
|
+
const deductible = firstField(fields, [/\bdeductible\b/i, /\bretention\b/i, /\bsir\b/i]);
|
|
7389
|
+
const basis = firstField(fields, [/\bbasis\b/i]);
|
|
7390
|
+
const retroactiveDate = firstField(fields, [/\bretroactive date\b/i, /\bretro date\b/i]);
|
|
7391
|
+
const page = pageNumber(span);
|
|
7392
|
+
const limitAmount = parseCurrencyAmount(limit);
|
|
7393
|
+
const deductibleAmount = deductible ? parseCurrencyAmount(deductible) : void 0;
|
|
7394
|
+
return {
|
|
7395
|
+
name,
|
|
7396
|
+
limit,
|
|
7397
|
+
...limitAmount !== void 0 ? { limitAmount } : {},
|
|
7398
|
+
...limitTypeFrom(`${name} ${limit}`) ? { limitType: limitTypeFrom(`${name} ${limit}`) } : {},
|
|
7399
|
+
limitValueType: limitValueTypeFrom(limit),
|
|
7400
|
+
...deductible && !/^nil|none$/i.test(deductible.trim()) ? { deductible } : {},
|
|
7401
|
+
...deductibleAmount !== void 0 ? { deductibleAmount } : {},
|
|
7402
|
+
...deductible ? { deductibleValueType: limitValueTypeFrom(deductible) } : {},
|
|
7403
|
+
...basis && /claims[- ]made/i.test(basis) ? { trigger: "claims_made" } : {},
|
|
7404
|
+
...retroactiveDate ? { retroactiveDate } : {},
|
|
7405
|
+
...span.formNumber ? { formNumber: span.formNumber } : {},
|
|
7406
|
+
...page ? { pageNumber: page } : {},
|
|
7407
|
+
sectionRef: span.sectionId ?? "SCHEDULE",
|
|
7408
|
+
originalContent: rowText,
|
|
7409
|
+
sourceSpanIds: [span.id],
|
|
7410
|
+
sourceTextHash: span.textHash ?? span.hash
|
|
7411
|
+
};
|
|
7412
|
+
}
|
|
7413
|
+
function coverageKey(coverage) {
|
|
7414
|
+
return [
|
|
7415
|
+
textValue(coverage.name) ?? "",
|
|
7416
|
+
textValue(coverage.limit) ?? "",
|
|
7417
|
+
textValue(coverage.limitType) ?? "",
|
|
7418
|
+
numberValue(coverage.pageNumber) ?? ""
|
|
7419
|
+
].map((part) => normalize(String(part))).join("|");
|
|
7420
|
+
}
|
|
7421
|
+
function rowMatchesExisting(row, existing) {
|
|
7422
|
+
const rowKey = coverageKey(row);
|
|
7423
|
+
const rowName = normalize(textValue(row.name) ?? "");
|
|
7424
|
+
const rowLimit = normalize(textValue(row.limit) ?? "");
|
|
7425
|
+
return existing.some((coverage) => {
|
|
7426
|
+
if (coverageKey(coverage) === rowKey) return true;
|
|
7427
|
+
const name = normalize(textValue(coverage.name) ?? "");
|
|
7428
|
+
const limit = normalize(textValue(coverage.limit) ?? "");
|
|
7429
|
+
return Boolean(rowName && rowLimit && name === rowName && limit === rowLimit);
|
|
7430
|
+
});
|
|
7431
|
+
}
|
|
7432
|
+
function recoverCoverageScheduleRows(params) {
|
|
7433
|
+
const payload = params.memory.get("coverage_limits");
|
|
7434
|
+
const existing = Array.isArray(payload?.coverages) ? payload.coverages : [];
|
|
7435
|
+
const pages = coveragePages(params.pageAssignments);
|
|
7436
|
+
const candidates = params.sourceSpans.filter((span) => sourceUnit3(span) === "table_row").filter((span) => {
|
|
7437
|
+
const page = pageNumber(span);
|
|
7438
|
+
return page !== void 0 && pages.has(page);
|
|
7439
|
+
}).map(coverageFromRow).filter((coverage) => Boolean(coverage));
|
|
7440
|
+
const recovered = [];
|
|
7441
|
+
for (const coverage of candidates) {
|
|
7442
|
+
if (rowMatchesExisting(coverage, [...existing, ...recovered])) continue;
|
|
7443
|
+
recovered.push(coverage);
|
|
7444
|
+
}
|
|
7445
|
+
if (recovered.length > 0) {
|
|
7446
|
+
params.memory.set("coverage_limits", {
|
|
7447
|
+
...payload ?? {},
|
|
7448
|
+
coverages: [...existing, ...recovered]
|
|
7449
|
+
});
|
|
7450
|
+
}
|
|
7451
|
+
return {
|
|
7452
|
+
recovered,
|
|
7453
|
+
missingSourceRows: recovered.map((coverage) => {
|
|
7454
|
+
const id = Array.isArray(coverage.sourceSpanIds) ? coverage.sourceSpanIds[0] : void 0;
|
|
7455
|
+
return params.sourceSpans.find((span) => span.id === id);
|
|
7456
|
+
}).filter((span) => Boolean(span))
|
|
7457
|
+
};
|
|
7458
|
+
}
|
|
7459
|
+
|
|
7051
7460
|
// src/extraction/focused-dispatch.ts
|
|
7052
7461
|
async function runFocusedExtractorWithFallback(params) {
|
|
7053
7462
|
const {
|
|
@@ -7211,6 +7620,20 @@ function sourcePrecedence(sectionRef) {
|
|
|
7211
7620
|
if (normalized.includes("coverage form") || normalized.includes("policy form")) return 1;
|
|
7212
7621
|
return 0;
|
|
7213
7622
|
}
|
|
7623
|
+
function looksDeductibleOnlyCoverageRow(coverage) {
|
|
7624
|
+
const name = typeof coverage.name === "string" ? coverage.name : "";
|
|
7625
|
+
const originalContent = typeof coverage.originalContent === "string" ? coverage.originalContent : "";
|
|
7626
|
+
const evidenceText = originalContent || name;
|
|
7627
|
+
const normalizedEvidence = evidenceText.toLowerCase();
|
|
7628
|
+
const normalizedName = name.toLowerCase();
|
|
7629
|
+
if (!/\b(deductible|retention|self[-\s]?insured retention|sir)\b/.test(`${normalizedName} ${normalizedEvidence}`)) return false;
|
|
7630
|
+
if (coverage.deductible || coverage.deductibleAmount !== void 0) return false;
|
|
7631
|
+
if (!coverage.limit && coverage.limitAmount === void 0) return false;
|
|
7632
|
+
if (originalContent && /\b(deductible|retention|self[-\s]?insured retention|sir)\b/.test(normalizedEvidence) && !/\b(?:sub[-\s]?limit|limit|coverage)\b/.test(normalizedEvidence)) {
|
|
7633
|
+
return true;
|
|
7634
|
+
}
|
|
7635
|
+
return /[-–—]\s*(?:enhanced\s+|standard\s+)?(?:deductible|retention|sir)\b/.test(normalizedName);
|
|
7636
|
+
}
|
|
7214
7637
|
function buildExtractionReviewReport(params) {
|
|
7215
7638
|
const { memory, reviewRounds } = params;
|
|
7216
7639
|
const deterministicIssues = [];
|
|
@@ -7236,6 +7659,29 @@ function buildExtractionReviewReport(params) {
|
|
|
7236
7659
|
addMissingSourceGroundingIssues(deterministicIssues, "definitions", "definitions", definitions, "term");
|
|
7237
7660
|
addMissingSourceGroundingIssues(deterministicIssues, "covered_reasons", "coveredReasons", coveredReasons, "name");
|
|
7238
7661
|
}
|
|
7662
|
+
if (params.sourceSpans?.length) {
|
|
7663
|
+
const tempCoveragePayload = {
|
|
7664
|
+
...memory.get("coverage_limits") ?? {},
|
|
7665
|
+
coverages: coverages.map((coverage) => ({ ...coverage }))
|
|
7666
|
+
};
|
|
7667
|
+
const tempMemory = new Map(memory);
|
|
7668
|
+
tempMemory.set("coverage_limits", tempCoveragePayload);
|
|
7669
|
+
const scheduleRecovery = recoverCoverageScheduleRows({
|
|
7670
|
+
memory: tempMemory,
|
|
7671
|
+
sourceSpans: params.sourceSpans,
|
|
7672
|
+
pageAssignments: params.pageAssignments
|
|
7673
|
+
});
|
|
7674
|
+
for (const recovered of scheduleRecovery.recovered) {
|
|
7675
|
+
deterministicIssues.push({
|
|
7676
|
+
code: "coverage_schedule_row_missing",
|
|
7677
|
+
severity: "blocking",
|
|
7678
|
+
message: `Coverage schedule row "${String(recovered.name ?? "unknown")}" is present in source table evidence but missing from extracted coverages.`,
|
|
7679
|
+
extractorName: "coverage_limits",
|
|
7680
|
+
pageNumber: typeof recovered.pageNumber === "number" ? recovered.pageNumber : void 0,
|
|
7681
|
+
itemName: typeof recovered.name === "string" ? recovered.name : void 0
|
|
7682
|
+
});
|
|
7683
|
+
}
|
|
7684
|
+
}
|
|
7239
7685
|
if (mappedDefinitions && definitions.length === 0) {
|
|
7240
7686
|
deterministicIssues.push({
|
|
7241
7687
|
code: "definitions_mapped_but_empty",
|
|
@@ -7316,6 +7762,17 @@ function buildExtractionReviewReport(params) {
|
|
|
7316
7762
|
itemName: coverage.name
|
|
7317
7763
|
});
|
|
7318
7764
|
}
|
|
7765
|
+
if (looksDeductibleOnlyCoverageRow(coverage)) {
|
|
7766
|
+
deterministicIssues.push({
|
|
7767
|
+
code: "deductible_row_as_coverage_limit",
|
|
7768
|
+
severity: "blocking",
|
|
7769
|
+
message: `Coverage "${String(coverage.name ?? "unknown")}" appears to store a deductible or retention value as a coverage limit.`,
|
|
7770
|
+
extractorName: "coverage_limits",
|
|
7771
|
+
formNumber,
|
|
7772
|
+
pageNumber: typeof coverage.pageNumber === "number" ? coverage.pageNumber : void 0,
|
|
7773
|
+
itemName: typeof coverage.name === "string" ? coverage.name : void 0
|
|
7774
|
+
});
|
|
7775
|
+
}
|
|
7319
7776
|
if (typeof coverage.pageNumber !== "number") {
|
|
7320
7777
|
deterministicIssues.push({
|
|
7321
7778
|
code: "coverage_missing_page_number",
|
|
@@ -7653,17 +8110,17 @@ var ARRAY_PATHS = [
|
|
|
7653
8110
|
{ memoryKey: "covered_reasons", arrayKeys: ["coveredReasons", "covered_reasons"] },
|
|
7654
8111
|
{ memoryKey: "declarations", arrayKeys: ["fields"] }
|
|
7655
8112
|
];
|
|
7656
|
-
function
|
|
8113
|
+
function normalize2(value) {
|
|
7657
8114
|
return value.replace(/\s+/g, " ").trim().toLowerCase();
|
|
7658
8115
|
}
|
|
7659
|
-
function
|
|
8116
|
+
function textValue2(record, ...keys) {
|
|
7660
8117
|
for (const key of keys) {
|
|
7661
8118
|
const value = record[key];
|
|
7662
8119
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
7663
8120
|
}
|
|
7664
8121
|
return void 0;
|
|
7665
8122
|
}
|
|
7666
|
-
function
|
|
8123
|
+
function numberValue2(record, ...keys) {
|
|
7667
8124
|
for (const key of keys) {
|
|
7668
8125
|
const value = record[key];
|
|
7669
8126
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
@@ -7680,37 +8137,73 @@ function pageOverlaps(recordStart, recordEnd, span) {
|
|
|
7680
8137
|
return start <= (spanEnd ?? spanStart) && end >= spanStart;
|
|
7681
8138
|
}
|
|
7682
8139
|
function formMatches(record, span) {
|
|
7683
|
-
const formNumber =
|
|
8140
|
+
const formNumber = textValue2(record, "formNumber");
|
|
7684
8141
|
if (!formNumber || !span.formNumber) return false;
|
|
7685
|
-
return
|
|
8142
|
+
return normalize2(formNumber) === normalize2(span.formNumber);
|
|
7686
8143
|
}
|
|
7687
8144
|
function textMatches(record, span) {
|
|
7688
|
-
const spanText =
|
|
8145
|
+
const spanText = normalize2(span.text);
|
|
7689
8146
|
const candidates = [
|
|
7690
|
-
|
|
7691
|
-
|
|
7692
|
-
|
|
8147
|
+
textValue2(record, "originalContent", "content", "definition", "value"),
|
|
8148
|
+
textValue2(record, "name", "title", "term", "field", "coverageName"),
|
|
8149
|
+
textValue2(record, "limit", "deductible", "premium")
|
|
7693
8150
|
].filter((value) => !!value && value.length >= 3);
|
|
7694
|
-
return candidates.some((candidate) => spanText.includes(
|
|
8151
|
+
return candidates.some((candidate) => spanText.includes(normalize2(candidate)));
|
|
7695
8152
|
}
|
|
7696
8153
|
function sourceHashFor(spans) {
|
|
7697
8154
|
return spans.map((span) => span.textHash ?? span.hash).filter(Boolean).join(":") || void 0;
|
|
7698
8155
|
}
|
|
8156
|
+
function sourceUnit4(span) {
|
|
8157
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
8158
|
+
}
|
|
8159
|
+
function hierarchyScore(span) {
|
|
8160
|
+
switch (sourceUnit4(span)) {
|
|
8161
|
+
case "table_row":
|
|
8162
|
+
return 5;
|
|
8163
|
+
case "table":
|
|
8164
|
+
return 3;
|
|
8165
|
+
case "section":
|
|
8166
|
+
case "page":
|
|
8167
|
+
return 2;
|
|
8168
|
+
case "table_cell":
|
|
8169
|
+
return -2;
|
|
8170
|
+
default:
|
|
8171
|
+
return 0;
|
|
8172
|
+
}
|
|
8173
|
+
}
|
|
8174
|
+
function parentRowId(span) {
|
|
8175
|
+
return span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
8176
|
+
}
|
|
8177
|
+
function preferParentRows(matches, sourceSpans) {
|
|
8178
|
+
if (matches.length === 0) return matches;
|
|
8179
|
+
const byId = new Map(sourceSpans.map((span) => [span.id, span]));
|
|
8180
|
+
const expanded = [];
|
|
8181
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8182
|
+
for (const match of matches) {
|
|
8183
|
+
const parent = sourceUnit4(match) === "table_cell" ? byId.get(parentRowId(match) ?? "") : void 0;
|
|
8184
|
+
const preferred = parent && sourceUnit4(parent) === "table_row" ? parent : match;
|
|
8185
|
+
if (seen.has(preferred.id)) continue;
|
|
8186
|
+
seen.add(preferred.id);
|
|
8187
|
+
expanded.push(preferred);
|
|
8188
|
+
}
|
|
8189
|
+
return expanded;
|
|
8190
|
+
}
|
|
7699
8191
|
function findSourceSpansForRecord(record, sourceSpans) {
|
|
7700
8192
|
if (sourceSpans.length === 0) return [];
|
|
7701
|
-
const pageStart =
|
|
7702
|
-
const pageEnd =
|
|
8193
|
+
const pageStart = numberValue2(record, "pageNumber", "pageStart");
|
|
8194
|
+
const pageEnd = numberValue2(record, "pageNumber", "pageEnd");
|
|
7703
8195
|
const scored = sourceSpans.map((span) => {
|
|
7704
8196
|
let score = 0;
|
|
7705
8197
|
if (pageOverlaps(pageStart, pageEnd, span)) score += 4;
|
|
7706
8198
|
if (formMatches(record, span)) score += 3;
|
|
7707
8199
|
if (textMatches(record, span)) score += 2;
|
|
8200
|
+
if (score > 0) score += hierarchyScore(span);
|
|
7708
8201
|
return { span, score };
|
|
7709
8202
|
}).filter((item) => item.score >= 2).sort((left, right) => {
|
|
7710
8203
|
if (right.score !== left.score) return right.score - left.score;
|
|
7711
8204
|
return left.span.id.localeCompare(right.span.id);
|
|
7712
8205
|
});
|
|
7713
|
-
return scored.slice(0,
|
|
8206
|
+
return preferParentRows(scored.slice(0, 5).map((item) => item.span), sourceSpans).slice(0, 3);
|
|
7714
8207
|
}
|
|
7715
8208
|
function groundRecord(record, sourceSpans) {
|
|
7716
8209
|
if (Array.isArray(record.sourceSpanIds) && record.sourceSpanIds.length > 0 && record.sourceTextHash) {
|
|
@@ -8418,6 +8911,14 @@ ${pageText}`;
|
|
|
8418
8911
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8419
8912
|
}
|
|
8420
8913
|
}
|
|
8914
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
8915
|
+
memory,
|
|
8916
|
+
sourceSpans,
|
|
8917
|
+
pageAssignments
|
|
8918
|
+
});
|
|
8919
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
8920
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from table evidence`);
|
|
8921
|
+
}
|
|
8421
8922
|
const planIncludesSupplementary = tasks.some((task) => task.extractorName === "supplementary");
|
|
8422
8923
|
if (!planIncludesSupplementary && hasSupplementaryExtractionSignal(pageAssignments, formInventory, memory)) {
|
|
8423
8924
|
onProgress?.("Extracting supplementary retrieval facts...");
|
|
@@ -8514,7 +9015,8 @@ ${pageText}`;
|
|
|
8514
9015
|
memory,
|
|
8515
9016
|
pageAssignments,
|
|
8516
9017
|
reviewRounds,
|
|
8517
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9018
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9019
|
+
sourceSpans
|
|
8518
9020
|
});
|
|
8519
9021
|
if (shouldRunLlmReview(reviewMode, preReviewReport, sourceSpans.length > 0)) {
|
|
8520
9022
|
for (let round = 0; round < maxReviewRounds; round++) {
|
|
@@ -8583,6 +9085,14 @@ ${pageText}`;
|
|
|
8583
9085
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8584
9086
|
}
|
|
8585
9087
|
}
|
|
9088
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
9089
|
+
memory,
|
|
9090
|
+
sourceSpans,
|
|
9091
|
+
pageAssignments
|
|
9092
|
+
});
|
|
9093
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
9094
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from follow-up table evidence`);
|
|
9095
|
+
}
|
|
8586
9096
|
}
|
|
8587
9097
|
} else {
|
|
8588
9098
|
onProgress?.("Skipping LLM extraction review; deterministic checks passed.");
|
|
@@ -8592,7 +9102,8 @@ ${pageText}`;
|
|
|
8592
9102
|
memory,
|
|
8593
9103
|
pageAssignments,
|
|
8594
9104
|
reviewRounds,
|
|
8595
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9105
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9106
|
+
sourceSpans
|
|
8596
9107
|
});
|
|
8597
9108
|
if (reviewReport.issues.length > 0) {
|
|
8598
9109
|
await log?.(
|
|
@@ -8618,7 +9129,8 @@ ${pageText}`;
|
|
|
8618
9129
|
memory,
|
|
8619
9130
|
pageAssignments,
|
|
8620
9131
|
reviewRounds,
|
|
8621
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9132
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9133
|
+
sourceSpans
|
|
8622
9134
|
}));
|
|
8623
9135
|
onProgress?.("Assembling document...");
|
|
8624
9136
|
const document = assembleDocument(id, documentType, memory);
|
|
@@ -12541,6 +13053,8 @@ var AGENT_TOOLS = [
|
|
|
12541
13053
|
SourceSpanLocationSchema,
|
|
12542
13054
|
SourceSpanRefSchema,
|
|
12543
13055
|
SourceSpanSchema,
|
|
13056
|
+
SourceSpanTableLocationSchema,
|
|
13057
|
+
SourceSpanUnitSchema,
|
|
12544
13058
|
SubAnswerSchema,
|
|
12545
13059
|
SubQuestionSchema,
|
|
12546
13060
|
SubjectivityCategorySchema,
|