@claritylabs/cl-sdk 1.3.7 → 1.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.d.mts +185 -47
- package/dist/index.d.ts +185 -47
- package/dist/index.js +544 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +542 -55
- package/dist/index.mjs.map +1 -1
- package/dist/storage-sqlite.d.mts +89 -20
- package/dist/storage-sqlite.d.ts +89 -20
- package/dist/storage-sqlite.js +33 -2
- package/dist/storage-sqlite.js.map +1 -1
- package/dist/storage-sqlite.mjs +33 -2
- package/dist/storage-sqlite.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -224,6 +224,8 @@ __export(index_exports, {
|
|
|
224
224
|
SourceSpanLocationSchema: () => SourceSpanLocationSchema,
|
|
225
225
|
SourceSpanRefSchema: () => SourceSpanRefSchema,
|
|
226
226
|
SourceSpanSchema: () => SourceSpanSchema,
|
|
227
|
+
SourceSpanTableLocationSchema: () => SourceSpanTableLocationSchema,
|
|
228
|
+
SourceSpanUnitSchema: () => SourceSpanUnitSchema,
|
|
227
229
|
SubAnswerSchema: () => SubAnswerSchema,
|
|
228
230
|
SubQuestionSchema: () => SubQuestionSchema,
|
|
229
231
|
SubjectivityCategorySchema: () => SubjectivityCategorySchema,
|
|
@@ -959,6 +961,8 @@ var CoverageSchema = import_zod4.z.object({
|
|
|
959
961
|
deductible: import_zod4.z.string().optional(),
|
|
960
962
|
deductibleAmount: import_zod4.z.number().optional(),
|
|
961
963
|
deductibleValueType: CoverageValueTypeSchema.optional(),
|
|
964
|
+
trigger: CoverageTriggerSchema.optional(),
|
|
965
|
+
retroactiveDate: import_zod4.z.string().optional(),
|
|
962
966
|
formNumber: import_zod4.z.string().optional(),
|
|
963
967
|
pageNumber: import_zod4.z.number().optional(),
|
|
964
968
|
sectionRef: import_zod4.z.string().optional(),
|
|
@@ -2187,6 +2191,15 @@ var SourceSpanKindSchema = import_zod20.z.enum([
|
|
|
2187
2191
|
"plain_text",
|
|
2188
2192
|
"structured_field"
|
|
2189
2193
|
]);
|
|
2194
|
+
var SourceSpanUnitSchema = import_zod20.z.enum([
|
|
2195
|
+
"page",
|
|
2196
|
+
"section",
|
|
2197
|
+
"table",
|
|
2198
|
+
"table_row",
|
|
2199
|
+
"table_cell",
|
|
2200
|
+
"key_value",
|
|
2201
|
+
"text"
|
|
2202
|
+
]);
|
|
2190
2203
|
var SourceKindSchema = import_zod20.z.enum([
|
|
2191
2204
|
"policy_pdf",
|
|
2192
2205
|
"application_pdf",
|
|
@@ -2211,6 +2224,15 @@ var SourceSpanLocationSchema = import_zod20.z.object({
|
|
|
2211
2224
|
lineEnd: import_zod20.z.number().int().positive().optional(),
|
|
2212
2225
|
fieldPath: import_zod20.z.string().optional()
|
|
2213
2226
|
});
|
|
2227
|
+
var SourceSpanTableLocationSchema = import_zod20.z.object({
|
|
2228
|
+
tableId: import_zod20.z.string().optional(),
|
|
2229
|
+
rowIndex: import_zod20.z.number().int().nonnegative().optional(),
|
|
2230
|
+
columnIndex: import_zod20.z.number().int().nonnegative().optional(),
|
|
2231
|
+
columnName: import_zod20.z.string().optional(),
|
|
2232
|
+
rowSpanId: import_zod20.z.string().optional(),
|
|
2233
|
+
tableSpanId: import_zod20.z.string().optional(),
|
|
2234
|
+
isHeader: import_zod20.z.boolean().optional()
|
|
2235
|
+
});
|
|
2214
2236
|
var SourceSpanSchema = import_zod20.z.object({
|
|
2215
2237
|
id: import_zod20.z.string().min(1),
|
|
2216
2238
|
documentId: import_zod20.z.string().min(1),
|
|
@@ -2224,6 +2246,9 @@ var SourceSpanSchema = import_zod20.z.object({
|
|
|
2224
2246
|
pageEnd: import_zod20.z.number().int().positive().optional(),
|
|
2225
2247
|
sectionId: import_zod20.z.string().optional(),
|
|
2226
2248
|
formNumber: import_zod20.z.string().optional(),
|
|
2249
|
+
sourceUnit: SourceSpanUnitSchema.optional(),
|
|
2250
|
+
parentSpanId: import_zod20.z.string().optional(),
|
|
2251
|
+
table: SourceSpanTableLocationSchema.optional(),
|
|
2227
2252
|
bbox: import_zod20.z.array(SourceSpanBBoxSchema).optional(),
|
|
2228
2253
|
location: SourceSpanLocationSchema.optional(),
|
|
2229
2254
|
metadata: import_zod20.z.record(import_zod20.z.string(), import_zod20.z.string()).optional()
|
|
@@ -2342,6 +2367,9 @@ function buildSourceSpan(input, localIndex = 0) {
|
|
|
2342
2367
|
pageEnd: input.pageEnd,
|
|
2343
2368
|
sectionId: input.sectionId,
|
|
2344
2369
|
formNumber: input.formNumber,
|
|
2370
|
+
sourceUnit: input.sourceUnit,
|
|
2371
|
+
parentSpanId: input.parentSpanId,
|
|
2372
|
+
table: input.table,
|
|
2345
2373
|
location: {
|
|
2346
2374
|
page: input.pageStart === input.pageEnd ? input.pageStart : void 0,
|
|
2347
2375
|
startPage: input.pageStart,
|
|
@@ -2362,7 +2390,11 @@ function buildPageSourceSpans(pages) {
|
|
|
2362
2390
|
pageEnd: page.pageNumber,
|
|
2363
2391
|
sectionId: page.sectionId,
|
|
2364
2392
|
formNumber: page.formNumber,
|
|
2365
|
-
|
|
2393
|
+
sourceUnit: "page",
|
|
2394
|
+
metadata: {
|
|
2395
|
+
...page.metadata ?? {},
|
|
2396
|
+
sourceUnit: page.metadata?.sourceUnit ?? "page"
|
|
2397
|
+
}
|
|
2366
2398
|
},
|
|
2367
2399
|
index
|
|
2368
2400
|
)
|
|
@@ -2384,6 +2416,7 @@ function buildSectionSourceSpans(pages, options = {}) {
|
|
|
2384
2416
|
pageEnd: page.pageNumber,
|
|
2385
2417
|
sectionId: section.title,
|
|
2386
2418
|
formNumber: inferFormNumber(section.text),
|
|
2419
|
+
sourceUnit: "section",
|
|
2387
2420
|
metadata: {
|
|
2388
2421
|
...page.metadata ?? {},
|
|
2389
2422
|
sourceUnit: "section_candidate"
|
|
@@ -2416,6 +2449,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2416
2449
|
const chunks = [];
|
|
2417
2450
|
let current = [];
|
|
2418
2451
|
let currentLength = 0;
|
|
2452
|
+
const spansForChunking = filterChunkableSourceSpans(spans);
|
|
2419
2453
|
const flush = () => {
|
|
2420
2454
|
if (current.length === 0) return;
|
|
2421
2455
|
const text = current.map((span) => span.text).join("\n\n");
|
|
@@ -2439,7 +2473,7 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2439
2473
|
current = [];
|
|
2440
2474
|
currentLength = 0;
|
|
2441
2475
|
};
|
|
2442
|
-
for (const span of
|
|
2476
|
+
for (const span of spansForChunking) {
|
|
2443
2477
|
const nextLength = currentLength + span.text.length + (current.length > 0 ? 2 : 0);
|
|
2444
2478
|
if (current.length > 0 && nextLength > maxChars) {
|
|
2445
2479
|
flush();
|
|
@@ -2450,6 +2484,20 @@ function chunkSourceSpans(spans, options = {}) {
|
|
|
2450
2484
|
flush();
|
|
2451
2485
|
return chunks;
|
|
2452
2486
|
}
|
|
2487
|
+
function sourceUnit(span) {
|
|
2488
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
2489
|
+
}
|
|
2490
|
+
function filterChunkableSourceSpans(spans) {
|
|
2491
|
+
const rowIds = new Set(
|
|
2492
|
+
spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
|
|
2493
|
+
);
|
|
2494
|
+
if (rowIds.size === 0) return spans;
|
|
2495
|
+
return spans.filter((span) => {
|
|
2496
|
+
if (sourceUnit(span) !== "table_cell") return true;
|
|
2497
|
+
const rowId = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
2498
|
+
return !rowId || !rowIds.has(rowId);
|
|
2499
|
+
});
|
|
2500
|
+
}
|
|
2453
2501
|
function splitPageIntoSections(text, headingPattern, minSectionChars) {
|
|
2454
2502
|
const lines = text.split(/\r?\n/);
|
|
2455
2503
|
const sections = [];
|
|
@@ -2834,36 +2882,18 @@ function normalizeDoclingDocument(document, options) {
|
|
|
2834
2882
|
pageTexts.set(page, appendText(pageTexts.get(page), unit.text));
|
|
2835
2883
|
}
|
|
2836
2884
|
const fullText = Array.from({ length: pageCount }, (_, index) => {
|
|
2837
|
-
const
|
|
2838
|
-
const text = pageTexts.get(
|
|
2839
|
-
return text ? `Page ${
|
|
2885
|
+
const pageNumber2 = index + 1;
|
|
2886
|
+
const text = pageTexts.get(pageNumber2)?.trim();
|
|
2887
|
+
return text ? `Page ${pageNumber2}
|
|
2840
2888
|
${text}` : "";
|
|
2841
2889
|
}).filter(Boolean).join("\n\n");
|
|
2842
2890
|
const sourceKind = options.sourceKind ?? "policy_pdf";
|
|
2843
|
-
const sourceSpans = units.
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
pageStart: unit.pageStart,
|
|
2850
|
-
pageEnd: unit.pageEnd,
|
|
2851
|
-
sectionId: unit.label,
|
|
2852
|
-
metadata: {
|
|
2853
|
-
sourceSystem: "docling",
|
|
2854
|
-
sourceUnit: "docling_item",
|
|
2855
|
-
doclingRef: unit.ref,
|
|
2856
|
-
...unit.label ? { doclingLabel: unit.label } : {}
|
|
2857
|
-
}
|
|
2858
|
-
},
|
|
2859
|
-
index
|
|
2860
|
-
);
|
|
2861
|
-
return {
|
|
2862
|
-
...span,
|
|
2863
|
-
kind: "plain_text",
|
|
2864
|
-
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
2865
|
-
};
|
|
2866
|
-
});
|
|
2891
|
+
const sourceSpans = units.flatMap(
|
|
2892
|
+
(unit, index) => buildSourceSpansForUnit(unit, index, {
|
|
2893
|
+
documentId: options.documentId,
|
|
2894
|
+
sourceKind
|
|
2895
|
+
})
|
|
2896
|
+
);
|
|
2867
2897
|
return {
|
|
2868
2898
|
pageCount,
|
|
2869
2899
|
fullText,
|
|
@@ -2901,6 +2931,11 @@ function mergeSourceSpans(spans) {
|
|
|
2901
2931
|
span.pageStart ?? span.location?.startPage ?? span.location?.page ?? "na",
|
|
2902
2932
|
span.pageEnd ?? span.location?.endPage ?? span.pageStart ?? "na",
|
|
2903
2933
|
span.sectionId ?? span.location?.fieldPath ?? "na",
|
|
2934
|
+
span.sourceUnit ?? span.metadata?.sourceUnit ?? "na",
|
|
2935
|
+
span.parentSpanId ?? "na",
|
|
2936
|
+
span.table?.tableId ?? span.metadata?.tableId ?? "na",
|
|
2937
|
+
span.table?.rowIndex ?? span.metadata?.rowIndex ?? "na",
|
|
2938
|
+
span.table?.columnIndex ?? span.metadata?.columnIndex ?? "na",
|
|
2904
2939
|
span.textHash ?? sourceSpanTextHash(span.text)
|
|
2905
2940
|
].join(":");
|
|
2906
2941
|
if (seen.has(key)) continue;
|
|
@@ -2963,6 +2998,7 @@ function getOrderedBodyRefs(document, itemMap) {
|
|
|
2963
2998
|
function normalizeItem(ref, item) {
|
|
2964
2999
|
const text = getItemText(item).trim();
|
|
2965
3000
|
if (!text) return void 0;
|
|
3001
|
+
const table = getItemTable(item);
|
|
2966
3002
|
const pages = (item.prov ?? []).map((prov) => getPageNumber(prov)).filter((page) => typeof page === "number" && page > 0);
|
|
2967
3003
|
const pageStart = pages.length ? Math.min(...pages) : void 0;
|
|
2968
3004
|
const pageEnd = pages.length ? Math.max(...pages) : pageStart;
|
|
@@ -2973,17 +3009,143 @@ function normalizeItem(ref, item) {
|
|
|
2973
3009
|
text,
|
|
2974
3010
|
pageStart,
|
|
2975
3011
|
pageEnd,
|
|
2976
|
-
bboxes: bboxes.length ? bboxes : void 0
|
|
3012
|
+
bboxes: bboxes.length ? bboxes : void 0,
|
|
3013
|
+
table
|
|
2977
3014
|
};
|
|
2978
3015
|
}
|
|
3016
|
+
function buildSourceSpansForUnit(unit, index, options) {
|
|
3017
|
+
const baseMetadata = {
|
|
3018
|
+
sourceSystem: "docling",
|
|
3019
|
+
sourceUnit: unit.table ? "table" : "docling_item",
|
|
3020
|
+
doclingRef: unit.ref,
|
|
3021
|
+
...unit.label ? { doclingLabel: unit.label } : {}
|
|
3022
|
+
};
|
|
3023
|
+
const tableId = unit.table ? `${unit.ref}:table` : void 0;
|
|
3024
|
+
const tableSpan = withDoclingKind(buildSourceSpan(
|
|
3025
|
+
{
|
|
3026
|
+
documentId: options.documentId,
|
|
3027
|
+
sourceKind: options.sourceKind,
|
|
3028
|
+
text: unit.text,
|
|
3029
|
+
pageStart: unit.pageStart,
|
|
3030
|
+
pageEnd: unit.pageEnd,
|
|
3031
|
+
sectionId: unit.label,
|
|
3032
|
+
sourceUnit: unit.table ? "table" : labelToSourceUnit(unit.label),
|
|
3033
|
+
table: tableId ? { tableId } : void 0,
|
|
3034
|
+
metadata: baseMetadata
|
|
3035
|
+
},
|
|
3036
|
+
index * 1e3
|
|
3037
|
+
), unit);
|
|
3038
|
+
if (!unit.table) return [tableSpan];
|
|
3039
|
+
const spans = [tableSpan];
|
|
3040
|
+
const table = unit.table;
|
|
3041
|
+
for (let rowIndex = 0; rowIndex < table.rows.length; rowIndex += 1) {
|
|
3042
|
+
const row = table.rows[rowIndex];
|
|
3043
|
+
const isHeader = rowIndex === 0 && table.headers.length > 0;
|
|
3044
|
+
const rowText = tableRowText(table, rowIndex);
|
|
3045
|
+
if (!rowText) continue;
|
|
3046
|
+
const rowSpan = withDoclingKind(buildSourceSpan(
|
|
3047
|
+
{
|
|
3048
|
+
documentId: options.documentId,
|
|
3049
|
+
sourceKind: options.sourceKind,
|
|
3050
|
+
text: rowText,
|
|
3051
|
+
pageStart: unit.pageStart,
|
|
3052
|
+
pageEnd: unit.pageEnd,
|
|
3053
|
+
sectionId: unit.label,
|
|
3054
|
+
sourceUnit: "table_row",
|
|
3055
|
+
parentSpanId: tableSpan.id,
|
|
3056
|
+
table: {
|
|
3057
|
+
tableId,
|
|
3058
|
+
tableSpanId: tableSpan.id,
|
|
3059
|
+
rowIndex,
|
|
3060
|
+
isHeader
|
|
3061
|
+
},
|
|
3062
|
+
metadata: {
|
|
3063
|
+
...baseMetadata,
|
|
3064
|
+
sourceUnit: "table_row",
|
|
3065
|
+
tableId: tableId ?? "",
|
|
3066
|
+
tableSpanId: tableSpan.id,
|
|
3067
|
+
rowIndex: String(rowIndex),
|
|
3068
|
+
isHeader: String(isHeader)
|
|
3069
|
+
}
|
|
3070
|
+
},
|
|
3071
|
+
index * 1e3 + 1 + rowIndex
|
|
3072
|
+
), unit);
|
|
3073
|
+
spans.push(rowSpan);
|
|
3074
|
+
for (let columnIndex = 0; columnIndex < row.length; columnIndex += 1) {
|
|
3075
|
+
const text = row[columnIndex]?.trim();
|
|
3076
|
+
if (!text) continue;
|
|
3077
|
+
const columnName = table.headers[columnIndex]?.trim() || void 0;
|
|
3078
|
+
const cellSpan = withDoclingKind(buildSourceSpan(
|
|
3079
|
+
{
|
|
3080
|
+
documentId: options.documentId,
|
|
3081
|
+
sourceKind: options.sourceKind,
|
|
3082
|
+
text,
|
|
3083
|
+
pageStart: unit.pageStart,
|
|
3084
|
+
pageEnd: unit.pageEnd,
|
|
3085
|
+
sectionId: unit.label,
|
|
3086
|
+
sourceUnit: "table_cell",
|
|
3087
|
+
parentSpanId: rowSpan.id,
|
|
3088
|
+
table: {
|
|
3089
|
+
tableId,
|
|
3090
|
+
tableSpanId: tableSpan.id,
|
|
3091
|
+
rowSpanId: rowSpan.id,
|
|
3092
|
+
rowIndex,
|
|
3093
|
+
columnIndex,
|
|
3094
|
+
columnName,
|
|
3095
|
+
isHeader
|
|
3096
|
+
},
|
|
3097
|
+
metadata: {
|
|
3098
|
+
...baseMetadata,
|
|
3099
|
+
sourceUnit: "table_cell",
|
|
3100
|
+
tableId: tableId ?? "",
|
|
3101
|
+
tableSpanId: tableSpan.id,
|
|
3102
|
+
rowSpanId: rowSpan.id,
|
|
3103
|
+
rowIndex: String(rowIndex),
|
|
3104
|
+
columnIndex: String(columnIndex),
|
|
3105
|
+
...columnName ? { columnName } : {},
|
|
3106
|
+
isHeader: String(isHeader)
|
|
3107
|
+
}
|
|
3108
|
+
},
|
|
3109
|
+
index * 1e3 + 100 + rowIndex * 100 + columnIndex
|
|
3110
|
+
), unit);
|
|
3111
|
+
spans.push(cellSpan);
|
|
3112
|
+
}
|
|
3113
|
+
}
|
|
3114
|
+
return spans;
|
|
3115
|
+
}
|
|
3116
|
+
function withDoclingKind(span, unit) {
|
|
3117
|
+
return {
|
|
3118
|
+
...span,
|
|
3119
|
+
kind: "plain_text",
|
|
3120
|
+
bbox: unit.bboxes?.length ? unit.bboxes : void 0
|
|
3121
|
+
};
|
|
3122
|
+
}
|
|
3123
|
+
function labelToSourceUnit(label) {
|
|
3124
|
+
const normalized = label?.toLowerCase() ?? "";
|
|
3125
|
+
if (normalized.includes("table")) return "table";
|
|
3126
|
+
if (normalized.includes("key")) return "key_value";
|
|
3127
|
+
if (normalized.includes("section") || normalized.includes("header")) return "section";
|
|
3128
|
+
return "text";
|
|
3129
|
+
}
|
|
3130
|
+
function tableRowText(table, rowIndex) {
|
|
3131
|
+
const row = table.rows[rowIndex] ?? [];
|
|
3132
|
+
const headers = table.headers;
|
|
3133
|
+
if (rowIndex === 0 && headers.length > 0) return row.filter(Boolean).join(" | ");
|
|
3134
|
+
return row.map((value, columnIndex) => {
|
|
3135
|
+
const trimmed = value.trim();
|
|
3136
|
+
if (!trimmed) return "";
|
|
3137
|
+
const header = headers[columnIndex]?.trim();
|
|
3138
|
+
return header ? `${header}: ${trimmed}` : trimmed;
|
|
3139
|
+
}).filter(Boolean).join(" | ");
|
|
3140
|
+
}
|
|
2979
3141
|
function getItemText(item) {
|
|
2980
3142
|
if (typeof item.text === "string" && item.text.trim()) return item.text;
|
|
2981
3143
|
if (typeof item.orig === "string" && item.orig.trim()) return item.orig;
|
|
2982
|
-
const table =
|
|
2983
|
-
if (table) return table;
|
|
3144
|
+
const table = parseTableData(item.data);
|
|
3145
|
+
if (table) return table.markdown;
|
|
2984
3146
|
return "";
|
|
2985
3147
|
}
|
|
2986
|
-
function
|
|
3148
|
+
function parseTableData(data) {
|
|
2987
3149
|
const record = asRecord(data);
|
|
2988
3150
|
const cells = Array.isArray(record?.table_cells) ? record.table_cells : Array.isArray(record?.tableCells) ? record.tableCells : void 0;
|
|
2989
3151
|
if (!cells) return void 0;
|
|
@@ -2999,10 +3161,20 @@ function tableToMarkdown(data) {
|
|
|
2999
3161
|
for (const cell of parsedCells) {
|
|
3000
3162
|
rows[cell.row][cell.col] = cell.text;
|
|
3001
3163
|
}
|
|
3002
|
-
if (rows.length === 1)
|
|
3164
|
+
if (rows.length === 1) {
|
|
3165
|
+
const markdown2 = rows[0].filter(Boolean).join(" | ");
|
|
3166
|
+
return { markdown: markdown2, headers: [], rows, cells: parsedCells };
|
|
3167
|
+
}
|
|
3003
3168
|
const header = rows[0];
|
|
3004
3169
|
const separator = header.map(() => "---");
|
|
3005
|
-
|
|
3170
|
+
const markdown = [header, separator, ...rows.slice(1)].map((row) => `| ${row.map((value) => value.trim()).join(" | ")} |`).join("\n");
|
|
3171
|
+
return { markdown, headers: header, rows, cells: parsedCells };
|
|
3172
|
+
}
|
|
3173
|
+
function getItemTable(item) {
|
|
3174
|
+
if (typeof item.text === "string" && item.text.trim() || typeof item.orig === "string" && item.orig.trim()) {
|
|
3175
|
+
return void 0;
|
|
3176
|
+
}
|
|
3177
|
+
return parseTableData(item.data);
|
|
3006
3178
|
}
|
|
3007
3179
|
function inferPageCount(document, units) {
|
|
3008
3180
|
const pages = document.pages;
|
|
@@ -3079,8 +3251,18 @@ function buildSourceContext(spans, maxChars = 12e3) {
|
|
|
3079
3251
|
if (spans.length === 0) return "";
|
|
3080
3252
|
const lines = [];
|
|
3081
3253
|
let length = 0;
|
|
3082
|
-
for (const span of spans) {
|
|
3083
|
-
const
|
|
3254
|
+
for (const span of orderSourceSpansForContext(spans)) {
|
|
3255
|
+
const unit = sourceUnit2(span);
|
|
3256
|
+
const table = span.table;
|
|
3257
|
+
const tableContext = [
|
|
3258
|
+
unit ? ` unit:${unit}` : "",
|
|
3259
|
+
table?.tableId ? ` table:${table.tableId}` : "",
|
|
3260
|
+
table?.rowIndex != null ? ` row:${table.rowIndex}` : "",
|
|
3261
|
+
table?.columnIndex != null ? ` col:${table.columnIndex}` : "",
|
|
3262
|
+
table?.columnName ? ` column:${table.columnName}` : "",
|
|
3263
|
+
span.parentSpanId ? ` parent:${span.parentSpanId}` : ""
|
|
3264
|
+
].join("");
|
|
3265
|
+
const header = `[sourceSpan:${span.id}${span.pageStart ? ` page:${span.pageStart}${span.pageEnd && span.pageEnd !== span.pageStart ? `-${span.pageEnd}` : ""}` : ""}${span.sectionId ? ` section:${span.sectionId}` : ""}${span.formNumber ? ` form:${span.formNumber}` : ""}${tableContext}]`;
|
|
3084
3266
|
const text = `${header}
|
|
3085
3267
|
${span.text}`;
|
|
3086
3268
|
if (length + text.length > maxChars && lines.length > 0) break;
|
|
@@ -3094,6 +3276,49 @@ ${lines.join("\n\n")}
|
|
|
3094
3276
|
|
|
3095
3277
|
Use sourceSpan IDs when grounding extracted contractual values.`;
|
|
3096
3278
|
}
|
|
3279
|
+
function sourceUnit2(span) {
|
|
3280
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
3281
|
+
}
|
|
3282
|
+
function sourceContextRank(span) {
|
|
3283
|
+
switch (sourceUnit2(span)) {
|
|
3284
|
+
case "table_row":
|
|
3285
|
+
return 0;
|
|
3286
|
+
case "table":
|
|
3287
|
+
return 1;
|
|
3288
|
+
case "section":
|
|
3289
|
+
return 2;
|
|
3290
|
+
case "page":
|
|
3291
|
+
return 3;
|
|
3292
|
+
case "key_value":
|
|
3293
|
+
return 4;
|
|
3294
|
+
case "table_cell":
|
|
3295
|
+
return 8;
|
|
3296
|
+
default:
|
|
3297
|
+
return 5;
|
|
3298
|
+
}
|
|
3299
|
+
}
|
|
3300
|
+
function orderSourceSpansForContext(spans) {
|
|
3301
|
+
const parentRows = new Set(
|
|
3302
|
+
spans.filter((span) => sourceUnit2(span) === "table_row").map((span) => span.id)
|
|
3303
|
+
);
|
|
3304
|
+
const filtered = spans.filter((span) => {
|
|
3305
|
+
if (sourceUnit2(span) !== "table_cell") return true;
|
|
3306
|
+
const parent = span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
3307
|
+
return !parent || !parentRows.has(parent);
|
|
3308
|
+
});
|
|
3309
|
+
return [...filtered].sort((left, right) => {
|
|
3310
|
+
const leftPage = left.pageStart ?? left.location?.startPage ?? left.location?.page ?? 0;
|
|
3311
|
+
const rightPage = right.pageStart ?? right.location?.startPage ?? right.location?.page ?? 0;
|
|
3312
|
+
if (leftPage !== rightPage) return leftPage - rightPage;
|
|
3313
|
+
const leftRank = sourceContextRank(left);
|
|
3314
|
+
const rightRank = sourceContextRank(right);
|
|
3315
|
+
if (leftRank !== rightRank) return leftRank - rightRank;
|
|
3316
|
+
const leftRow = left.table?.rowIndex ?? Number(left.metadata?.rowIndex ?? Number.NaN);
|
|
3317
|
+
const rightRow = right.table?.rowIndex ?? Number(right.metadata?.rowIndex ?? Number.NaN);
|
|
3318
|
+
if (Number.isFinite(leftRow) && Number.isFinite(rightRow) && leftRow !== rightRow) return leftRow - rightRow;
|
|
3319
|
+
return left.id.localeCompare(right.id);
|
|
3320
|
+
});
|
|
3321
|
+
}
|
|
3097
3322
|
async function runExtractor(params) {
|
|
3098
3323
|
const {
|
|
3099
3324
|
name,
|
|
@@ -4692,7 +4917,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4692
4917
|
const merged = mergeShallowPreferPresent(existing, incoming);
|
|
4693
4918
|
const existingCoverages = Array.isArray(existing.coverages) ? existing.coverages : [];
|
|
4694
4919
|
const incomingCoverages = Array.isArray(incoming.coverages) ? incoming.coverages : [];
|
|
4695
|
-
const
|
|
4920
|
+
const coverageKey2 = (coverage) => keyFromParts(
|
|
4696
4921
|
coverage.name,
|
|
4697
4922
|
coverage.limitType,
|
|
4698
4923
|
coverage.limit,
|
|
@@ -4701,7 +4926,7 @@ function mergeCoverageLimits(existing, incoming) {
|
|
|
4701
4926
|
);
|
|
4702
4927
|
const byKey = /* @__PURE__ */ new Map();
|
|
4703
4928
|
for (const coverage of [...existingCoverages, ...incomingCoverages]) {
|
|
4704
|
-
const key =
|
|
4929
|
+
const key = coverageKey2(coverage);
|
|
4705
4930
|
const current = byKey.get(key);
|
|
4706
4931
|
byKey.set(key, current ? mergeShallowPreferPresent(current, coverage) : coverage);
|
|
4707
4932
|
}
|
|
@@ -7048,6 +7273,190 @@ async function resolveReferentialCoverages(params) {
|
|
|
7048
7273
|
};
|
|
7049
7274
|
}
|
|
7050
7275
|
|
|
7276
|
+
// src/extraction/coverage-schedule-recovery.ts
|
|
7277
|
+
function textValue(value) {
|
|
7278
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
7279
|
+
}
|
|
7280
|
+
function numberValue(value) {
|
|
7281
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
7282
|
+
}
|
|
7283
|
+
function normalize(value) {
|
|
7284
|
+
return value.toLowerCase().replace(/&/g, "and").replace(/[^a-z0-9]+/g, " ").trim();
|
|
7285
|
+
}
|
|
7286
|
+
function sourceUnit3(span) {
|
|
7287
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
7288
|
+
}
|
|
7289
|
+
function pageNumber(span) {
|
|
7290
|
+
return span.pageStart ?? span.location?.page ?? span.location?.startPage;
|
|
7291
|
+
}
|
|
7292
|
+
function coveragePages(pageAssignments) {
|
|
7293
|
+
const pages = /* @__PURE__ */ new Set();
|
|
7294
|
+
for (const assignment of pageAssignments) {
|
|
7295
|
+
if (assignment.extractorNames.includes("coverage_limits") || assignment.hasScheduleValues) {
|
|
7296
|
+
pages.add(assignment.localPageNumber);
|
|
7297
|
+
}
|
|
7298
|
+
}
|
|
7299
|
+
return pages;
|
|
7300
|
+
}
|
|
7301
|
+
function parseCurrencyAmount(value) {
|
|
7302
|
+
const match = value.match(/(?:CAD|USD|US)?\s*\$?\s*([0-9][0-9,]*(?:\.\d+)?)/i);
|
|
7303
|
+
if (!match) return void 0;
|
|
7304
|
+
const amount = Number(match[1].replace(/,/g, ""));
|
|
7305
|
+
return Number.isFinite(amount) ? amount : void 0;
|
|
7306
|
+
}
|
|
7307
|
+
function limitTypeFrom(value) {
|
|
7308
|
+
const normalized = normalize(value);
|
|
7309
|
+
if (normalized.includes("/") || normalized.includes("each claim") && normalized.includes("aggregate")) return "scheduled";
|
|
7310
|
+
if (normalized.includes("each claim") || normalized.includes("per claim")) return "per_claim";
|
|
7311
|
+
if (normalized.includes("each occurrence") || normalized.includes("per occurrence")) return "per_occurrence";
|
|
7312
|
+
if (normalized.includes("aggregate")) return "aggregate";
|
|
7313
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage")) return "scheduled";
|
|
7314
|
+
if (normalized.includes("/")) return "scheduled";
|
|
7315
|
+
return void 0;
|
|
7316
|
+
}
|
|
7317
|
+
function limitValueTypeFrom(value) {
|
|
7318
|
+
const normalized = normalize(value);
|
|
7319
|
+
if (parseCurrencyAmount(value) !== void 0) return "numeric";
|
|
7320
|
+
if (normalized.includes("shared within") || normalized.includes("within coverage") || normalized.includes("shown above")) {
|
|
7321
|
+
return "referential";
|
|
7322
|
+
}
|
|
7323
|
+
if (normalized.includes("included")) return "included";
|
|
7324
|
+
if (normalized.includes("not included") || normalized === "nil" || normalized === "none") return "not_included";
|
|
7325
|
+
if (normalized.includes("as stated")) return "as_stated";
|
|
7326
|
+
return "other";
|
|
7327
|
+
}
|
|
7328
|
+
function cleanName(value) {
|
|
7329
|
+
return value.replace(/\s*\([^)]*part of and not in addition to[^)]*\)\s*/gi, " ").replace(/\s+/g, " ").trim().replace(/[.:;-]+$/, "").trim();
|
|
7330
|
+
}
|
|
7331
|
+
function isDeductibleOnly(name, rowText) {
|
|
7332
|
+
const normalizedName = normalize(name);
|
|
7333
|
+
const normalizedRow = normalize(rowText);
|
|
7334
|
+
if (!/\b(deductible|retention|sir)\b/.test(`${normalizedName} ${normalizedRow}`)) return false;
|
|
7335
|
+
if (/\b(coverage|sub limit|sublimit|limit)\b/.test(normalizedName) && !/\b(enhanced|standard)\s+deductible\b/.test(normalizedName)) {
|
|
7336
|
+
return false;
|
|
7337
|
+
}
|
|
7338
|
+
return /\b(enhanced|standard)?\s*(deductible|retention|sir)\b/.test(normalizedName) || /^\s*(deductible|retention|sir)\b/.test(normalizedRow);
|
|
7339
|
+
}
|
|
7340
|
+
function splitRowFields(rowText) {
|
|
7341
|
+
return rowText.split(/\s+\|\s+/).map((part) => part.trim()).filter(Boolean).map((part) => {
|
|
7342
|
+
const match = part.match(/^([^:]{1,100}):\s*(.*)$/);
|
|
7343
|
+
if (!match) return { value: part };
|
|
7344
|
+
return { key: match[1].trim(), value: match[2].trim() };
|
|
7345
|
+
});
|
|
7346
|
+
}
|
|
7347
|
+
function firstField(fields, patterns) {
|
|
7348
|
+
for (const field of fields) {
|
|
7349
|
+
const target = `${field.key ?? ""} ${field.value}`;
|
|
7350
|
+
if (patterns.some((pattern) => pattern.test(target))) return field.value || field.key;
|
|
7351
|
+
}
|
|
7352
|
+
return void 0;
|
|
7353
|
+
}
|
|
7354
|
+
function coverageFromRow(span) {
|
|
7355
|
+
const rowText = span.text.trim();
|
|
7356
|
+
if (!rowText || sourceUnit3(span) !== "table_row") return void 0;
|
|
7357
|
+
if (span.table?.isHeader || span.metadata?.isHeader === "true") return void 0;
|
|
7358
|
+
const fields = splitRowFields(rowText);
|
|
7359
|
+
let name;
|
|
7360
|
+
let limit;
|
|
7361
|
+
for (const field of fields) {
|
|
7362
|
+
const key = field.key?.trim();
|
|
7363
|
+
const value = field.value.trim();
|
|
7364
|
+
if (!key || !value) continue;
|
|
7365
|
+
if (!name && /^coverage$/i.test(key)) name = cleanName(value);
|
|
7366
|
+
if (!limit && /\blimit\b/i.test(key)) limit = value;
|
|
7367
|
+
if (!name && /\b(sub[-\s]?limit|aggregate|each claim)\b/i.test(key) && parseCurrencyAmount(value) !== void 0) {
|
|
7368
|
+
name = cleanName(key);
|
|
7369
|
+
limit = limit ?? value;
|
|
7370
|
+
}
|
|
7371
|
+
}
|
|
7372
|
+
if (!name || !limit) {
|
|
7373
|
+
for (const field of fields) {
|
|
7374
|
+
const key = field.key?.trim();
|
|
7375
|
+
const value = field.value.trim();
|
|
7376
|
+
if (!key || !value) continue;
|
|
7377
|
+
if (!name && /\b(sub[-\s]?limit|coverage|aggregate|each claim)\b/i.test(key) && !parseCurrencyAmount(key)) {
|
|
7378
|
+
name = cleanName(key);
|
|
7379
|
+
limit = limit ?? value;
|
|
7380
|
+
}
|
|
7381
|
+
}
|
|
7382
|
+
}
|
|
7383
|
+
if (!name || !limit) return void 0;
|
|
7384
|
+
if (isDeductibleOnly(name, rowText)) return void 0;
|
|
7385
|
+
const normalizedLimit = normalize(limit);
|
|
7386
|
+
const hasUsableLimit = parseCurrencyAmount(limit) !== void 0 || /\b(shared within|within coverage|as stated|included|not included)\b/i.test(normalizedLimit);
|
|
7387
|
+
if (!hasUsableLimit) return void 0;
|
|
7388
|
+
const deductible = firstField(fields, [/\bdeductible\b/i, /\bretention\b/i, /\bsir\b/i]);
|
|
7389
|
+
const basis = firstField(fields, [/\bbasis\b/i]);
|
|
7390
|
+
const retroactiveDate = firstField(fields, [/\bretroactive date\b/i, /\bretro date\b/i]);
|
|
7391
|
+
const page = pageNumber(span);
|
|
7392
|
+
const limitAmount = parseCurrencyAmount(limit);
|
|
7393
|
+
const deductibleAmount = deductible ? parseCurrencyAmount(deductible) : void 0;
|
|
7394
|
+
return {
|
|
7395
|
+
name,
|
|
7396
|
+
limit,
|
|
7397
|
+
...limitAmount !== void 0 ? { limitAmount } : {},
|
|
7398
|
+
...limitTypeFrom(`${name} ${limit}`) ? { limitType: limitTypeFrom(`${name} ${limit}`) } : {},
|
|
7399
|
+
limitValueType: limitValueTypeFrom(limit),
|
|
7400
|
+
...deductible && !/^nil|none$/i.test(deductible.trim()) ? { deductible } : {},
|
|
7401
|
+
...deductibleAmount !== void 0 ? { deductibleAmount } : {},
|
|
7402
|
+
...deductible ? { deductibleValueType: limitValueTypeFrom(deductible) } : {},
|
|
7403
|
+
...basis && /claims[- ]made/i.test(basis) ? { trigger: "claims_made" } : {},
|
|
7404
|
+
...retroactiveDate ? { retroactiveDate } : {},
|
|
7405
|
+
...span.formNumber ? { formNumber: span.formNumber } : {},
|
|
7406
|
+
...page ? { pageNumber: page } : {},
|
|
7407
|
+
sectionRef: span.sectionId ?? "SCHEDULE",
|
|
7408
|
+
originalContent: rowText,
|
|
7409
|
+
sourceSpanIds: [span.id],
|
|
7410
|
+
sourceTextHash: span.textHash ?? span.hash
|
|
7411
|
+
};
|
|
7412
|
+
}
|
|
7413
|
+
function coverageKey(coverage) {
|
|
7414
|
+
return [
|
|
7415
|
+
textValue(coverage.name) ?? "",
|
|
7416
|
+
textValue(coverage.limit) ?? "",
|
|
7417
|
+
textValue(coverage.limitType) ?? "",
|
|
7418
|
+
numberValue(coverage.pageNumber) ?? ""
|
|
7419
|
+
].map((part) => normalize(String(part))).join("|");
|
|
7420
|
+
}
|
|
7421
|
+
function rowMatchesExisting(row, existing) {
|
|
7422
|
+
const rowKey = coverageKey(row);
|
|
7423
|
+
const rowName = normalize(textValue(row.name) ?? "");
|
|
7424
|
+
const rowLimit = normalize(textValue(row.limit) ?? "");
|
|
7425
|
+
return existing.some((coverage) => {
|
|
7426
|
+
if (coverageKey(coverage) === rowKey) return true;
|
|
7427
|
+
const name = normalize(textValue(coverage.name) ?? "");
|
|
7428
|
+
const limit = normalize(textValue(coverage.limit) ?? "");
|
|
7429
|
+
return Boolean(rowName && rowLimit && name === rowName && limit === rowLimit);
|
|
7430
|
+
});
|
|
7431
|
+
}
|
|
7432
|
+
function recoverCoverageScheduleRows(params) {
|
|
7433
|
+
const payload = params.memory.get("coverage_limits");
|
|
7434
|
+
const existing = Array.isArray(payload?.coverages) ? payload.coverages : [];
|
|
7435
|
+
const pages = coveragePages(params.pageAssignments);
|
|
7436
|
+
const candidates = params.sourceSpans.filter((span) => sourceUnit3(span) === "table_row").filter((span) => {
|
|
7437
|
+
const page = pageNumber(span);
|
|
7438
|
+
return page !== void 0 && pages.has(page);
|
|
7439
|
+
}).map(coverageFromRow).filter((coverage) => Boolean(coverage));
|
|
7440
|
+
const recovered = [];
|
|
7441
|
+
for (const coverage of candidates) {
|
|
7442
|
+
if (rowMatchesExisting(coverage, [...existing, ...recovered])) continue;
|
|
7443
|
+
recovered.push(coverage);
|
|
7444
|
+
}
|
|
7445
|
+
if (recovered.length > 0) {
|
|
7446
|
+
params.memory.set("coverage_limits", {
|
|
7447
|
+
...payload ?? {},
|
|
7448
|
+
coverages: [...existing, ...recovered]
|
|
7449
|
+
});
|
|
7450
|
+
}
|
|
7451
|
+
return {
|
|
7452
|
+
recovered,
|
|
7453
|
+
missingSourceRows: recovered.map((coverage) => {
|
|
7454
|
+
const id = Array.isArray(coverage.sourceSpanIds) ? coverage.sourceSpanIds[0] : void 0;
|
|
7455
|
+
return params.sourceSpans.find((span) => span.id === id);
|
|
7456
|
+
}).filter((span) => Boolean(span))
|
|
7457
|
+
};
|
|
7458
|
+
}
|
|
7459
|
+
|
|
7051
7460
|
// src/extraction/focused-dispatch.ts
|
|
7052
7461
|
async function runFocusedExtractorWithFallback(params) {
|
|
7053
7462
|
const {
|
|
@@ -7250,6 +7659,29 @@ function buildExtractionReviewReport(params) {
|
|
|
7250
7659
|
addMissingSourceGroundingIssues(deterministicIssues, "definitions", "definitions", definitions, "term");
|
|
7251
7660
|
addMissingSourceGroundingIssues(deterministicIssues, "covered_reasons", "coveredReasons", coveredReasons, "name");
|
|
7252
7661
|
}
|
|
7662
|
+
if (params.sourceSpans?.length) {
|
|
7663
|
+
const tempCoveragePayload = {
|
|
7664
|
+
...memory.get("coverage_limits") ?? {},
|
|
7665
|
+
coverages: coverages.map((coverage) => ({ ...coverage }))
|
|
7666
|
+
};
|
|
7667
|
+
const tempMemory = new Map(memory);
|
|
7668
|
+
tempMemory.set("coverage_limits", tempCoveragePayload);
|
|
7669
|
+
const scheduleRecovery = recoverCoverageScheduleRows({
|
|
7670
|
+
memory: tempMemory,
|
|
7671
|
+
sourceSpans: params.sourceSpans,
|
|
7672
|
+
pageAssignments: params.pageAssignments
|
|
7673
|
+
});
|
|
7674
|
+
for (const recovered of scheduleRecovery.recovered) {
|
|
7675
|
+
deterministicIssues.push({
|
|
7676
|
+
code: "coverage_schedule_row_missing",
|
|
7677
|
+
severity: "blocking",
|
|
7678
|
+
message: `Coverage schedule row "${String(recovered.name ?? "unknown")}" is present in source table evidence but missing from extracted coverages.`,
|
|
7679
|
+
extractorName: "coverage_limits",
|
|
7680
|
+
pageNumber: typeof recovered.pageNumber === "number" ? recovered.pageNumber : void 0,
|
|
7681
|
+
itemName: typeof recovered.name === "string" ? recovered.name : void 0
|
|
7682
|
+
});
|
|
7683
|
+
}
|
|
7684
|
+
}
|
|
7253
7685
|
if (mappedDefinitions && definitions.length === 0) {
|
|
7254
7686
|
deterministicIssues.push({
|
|
7255
7687
|
code: "definitions_mapped_but_empty",
|
|
@@ -7678,17 +8110,17 @@ var ARRAY_PATHS = [
|
|
|
7678
8110
|
{ memoryKey: "covered_reasons", arrayKeys: ["coveredReasons", "covered_reasons"] },
|
|
7679
8111
|
{ memoryKey: "declarations", arrayKeys: ["fields"] }
|
|
7680
8112
|
];
|
|
7681
|
-
function
|
|
8113
|
+
function normalize2(value) {
|
|
7682
8114
|
return value.replace(/\s+/g, " ").trim().toLowerCase();
|
|
7683
8115
|
}
|
|
7684
|
-
function
|
|
8116
|
+
function textValue2(record, ...keys) {
|
|
7685
8117
|
for (const key of keys) {
|
|
7686
8118
|
const value = record[key];
|
|
7687
8119
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
7688
8120
|
}
|
|
7689
8121
|
return void 0;
|
|
7690
8122
|
}
|
|
7691
|
-
function
|
|
8123
|
+
function numberValue2(record, ...keys) {
|
|
7692
8124
|
for (const key of keys) {
|
|
7693
8125
|
const value = record[key];
|
|
7694
8126
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
@@ -7705,37 +8137,73 @@ function pageOverlaps(recordStart, recordEnd, span) {
|
|
|
7705
8137
|
return start <= (spanEnd ?? spanStart) && end >= spanStart;
|
|
7706
8138
|
}
|
|
7707
8139
|
function formMatches(record, span) {
|
|
7708
|
-
const formNumber =
|
|
8140
|
+
const formNumber = textValue2(record, "formNumber");
|
|
7709
8141
|
if (!formNumber || !span.formNumber) return false;
|
|
7710
|
-
return
|
|
8142
|
+
return normalize2(formNumber) === normalize2(span.formNumber);
|
|
7711
8143
|
}
|
|
7712
8144
|
function textMatches(record, span) {
|
|
7713
|
-
const spanText =
|
|
8145
|
+
const spanText = normalize2(span.text);
|
|
7714
8146
|
const candidates = [
|
|
7715
|
-
|
|
7716
|
-
|
|
7717
|
-
|
|
8147
|
+
textValue2(record, "originalContent", "content", "definition", "value"),
|
|
8148
|
+
textValue2(record, "name", "title", "term", "field", "coverageName"),
|
|
8149
|
+
textValue2(record, "limit", "deductible", "premium")
|
|
7718
8150
|
].filter((value) => !!value && value.length >= 3);
|
|
7719
|
-
return candidates.some((candidate) => spanText.includes(
|
|
8151
|
+
return candidates.some((candidate) => spanText.includes(normalize2(candidate)));
|
|
7720
8152
|
}
|
|
7721
8153
|
function sourceHashFor(spans) {
|
|
7722
8154
|
return spans.map((span) => span.textHash ?? span.hash).filter(Boolean).join(":") || void 0;
|
|
7723
8155
|
}
|
|
8156
|
+
function sourceUnit4(span) {
|
|
8157
|
+
return span.sourceUnit ?? span.metadata?.sourceUnit;
|
|
8158
|
+
}
|
|
8159
|
+
function hierarchyScore(span) {
|
|
8160
|
+
switch (sourceUnit4(span)) {
|
|
8161
|
+
case "table_row":
|
|
8162
|
+
return 5;
|
|
8163
|
+
case "table":
|
|
8164
|
+
return 3;
|
|
8165
|
+
case "section":
|
|
8166
|
+
case "page":
|
|
8167
|
+
return 2;
|
|
8168
|
+
case "table_cell":
|
|
8169
|
+
return -2;
|
|
8170
|
+
default:
|
|
8171
|
+
return 0;
|
|
8172
|
+
}
|
|
8173
|
+
}
|
|
8174
|
+
function parentRowId(span) {
|
|
8175
|
+
return span.parentSpanId ?? span.table?.rowSpanId ?? span.metadata?.rowSpanId;
|
|
8176
|
+
}
|
|
8177
|
+
function preferParentRows(matches, sourceSpans) {
|
|
8178
|
+
if (matches.length === 0) return matches;
|
|
8179
|
+
const byId = new Map(sourceSpans.map((span) => [span.id, span]));
|
|
8180
|
+
const expanded = [];
|
|
8181
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8182
|
+
for (const match of matches) {
|
|
8183
|
+
const parent = sourceUnit4(match) === "table_cell" ? byId.get(parentRowId(match) ?? "") : void 0;
|
|
8184
|
+
const preferred = parent && sourceUnit4(parent) === "table_row" ? parent : match;
|
|
8185
|
+
if (seen.has(preferred.id)) continue;
|
|
8186
|
+
seen.add(preferred.id);
|
|
8187
|
+
expanded.push(preferred);
|
|
8188
|
+
}
|
|
8189
|
+
return expanded;
|
|
8190
|
+
}
|
|
7724
8191
|
function findSourceSpansForRecord(record, sourceSpans) {
|
|
7725
8192
|
if (sourceSpans.length === 0) return [];
|
|
7726
|
-
const pageStart =
|
|
7727
|
-
const pageEnd =
|
|
8193
|
+
const pageStart = numberValue2(record, "pageNumber", "pageStart");
|
|
8194
|
+
const pageEnd = numberValue2(record, "pageNumber", "pageEnd");
|
|
7728
8195
|
const scored = sourceSpans.map((span) => {
|
|
7729
8196
|
let score = 0;
|
|
7730
8197
|
if (pageOverlaps(pageStart, pageEnd, span)) score += 4;
|
|
7731
8198
|
if (formMatches(record, span)) score += 3;
|
|
7732
8199
|
if (textMatches(record, span)) score += 2;
|
|
8200
|
+
if (score > 0) score += hierarchyScore(span);
|
|
7733
8201
|
return { span, score };
|
|
7734
8202
|
}).filter((item) => item.score >= 2).sort((left, right) => {
|
|
7735
8203
|
if (right.score !== left.score) return right.score - left.score;
|
|
7736
8204
|
return left.span.id.localeCompare(right.span.id);
|
|
7737
8205
|
});
|
|
7738
|
-
return scored.slice(0,
|
|
8206
|
+
return preferParentRows(scored.slice(0, 5).map((item) => item.span), sourceSpans).slice(0, 3);
|
|
7739
8207
|
}
|
|
7740
8208
|
function groundRecord(record, sourceSpans) {
|
|
7741
8209
|
if (Array.isArray(record.sourceSpanIds) && record.sourceSpanIds.length > 0 && record.sourceTextHash) {
|
|
@@ -8443,6 +8911,14 @@ ${pageText}`;
|
|
|
8443
8911
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8444
8912
|
}
|
|
8445
8913
|
}
|
|
8914
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
8915
|
+
memory,
|
|
8916
|
+
sourceSpans,
|
|
8917
|
+
pageAssignments
|
|
8918
|
+
});
|
|
8919
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
8920
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from table evidence`);
|
|
8921
|
+
}
|
|
8446
8922
|
const planIncludesSupplementary = tasks.some((task) => task.extractorName === "supplementary");
|
|
8447
8923
|
if (!planIncludesSupplementary && hasSupplementaryExtractionSignal(pageAssignments, formInventory, memory)) {
|
|
8448
8924
|
onProgress?.("Extracting supplementary retrieval facts...");
|
|
@@ -8539,7 +9015,8 @@ ${pageText}`;
|
|
|
8539
9015
|
memory,
|
|
8540
9016
|
pageAssignments,
|
|
8541
9017
|
reviewRounds,
|
|
8542
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9018
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9019
|
+
sourceSpans
|
|
8543
9020
|
});
|
|
8544
9021
|
if (shouldRunLlmReview(reviewMode, preReviewReport, sourceSpans.length > 0)) {
|
|
8545
9022
|
for (let round = 0; round < maxReviewRounds; round++) {
|
|
@@ -8608,6 +9085,14 @@ ${pageText}`;
|
|
|
8608
9085
|
mergeMemoryResult(result.name, result.data, memory);
|
|
8609
9086
|
}
|
|
8610
9087
|
}
|
|
9088
|
+
const recoveredCoverages = recoverCoverageScheduleRows({
|
|
9089
|
+
memory,
|
|
9090
|
+
sourceSpans,
|
|
9091
|
+
pageAssignments
|
|
9092
|
+
});
|
|
9093
|
+
if (recoveredCoverages.recovered.length > 0) {
|
|
9094
|
+
await log?.(`Recovered ${recoveredCoverages.recovered.length} source-backed coverage schedule row(s) from follow-up table evidence`);
|
|
9095
|
+
}
|
|
8611
9096
|
}
|
|
8612
9097
|
} else {
|
|
8613
9098
|
onProgress?.("Skipping LLM extraction review; deterministic checks passed.");
|
|
@@ -8617,7 +9102,8 @@ ${pageText}`;
|
|
|
8617
9102
|
memory,
|
|
8618
9103
|
pageAssignments,
|
|
8619
9104
|
reviewRounds,
|
|
8620
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9105
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9106
|
+
sourceSpans
|
|
8621
9107
|
});
|
|
8622
9108
|
if (reviewReport.issues.length > 0) {
|
|
8623
9109
|
await log?.(
|
|
@@ -8643,7 +9129,8 @@ ${pageText}`;
|
|
|
8643
9129
|
memory,
|
|
8644
9130
|
pageAssignments,
|
|
8645
9131
|
reviewRounds,
|
|
8646
|
-
sourceSpansAvailable: sourceSpans.length > 0
|
|
9132
|
+
sourceSpansAvailable: sourceSpans.length > 0,
|
|
9133
|
+
sourceSpans
|
|
8647
9134
|
}));
|
|
8648
9135
|
onProgress?.("Assembling document...");
|
|
8649
9136
|
const document = assembleDocument(id, documentType, memory);
|
|
@@ -12566,6 +13053,8 @@ var AGENT_TOOLS = [
|
|
|
12566
13053
|
SourceSpanLocationSchema,
|
|
12567
13054
|
SourceSpanRefSchema,
|
|
12568
13055
|
SourceSpanSchema,
|
|
13056
|
+
SourceSpanTableLocationSchema,
|
|
13057
|
+
SourceSpanUnitSchema,
|
|
12569
13058
|
SubAnswerSchema,
|
|
12570
13059
|
SubQuestionSchema,
|
|
12571
13060
|
SubjectivityCategorySchema,
|