@claritylabs/cl-sdk 3.1.23 → 3.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2 -459
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -459
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -9380,46 +9380,6 @@ function applyOperationalProfileCleanup(profile, cleanup, validNodeIds, validSpa
|
|
|
9380
9380
|
}
|
|
9381
9381
|
|
|
9382
9382
|
// src/extraction/source-tree-extractor.ts
|
|
9383
|
-
var ORGANIZABLE_KINDS = [
|
|
9384
|
-
"page_group",
|
|
9385
|
-
"form",
|
|
9386
|
-
"endorsement",
|
|
9387
|
-
"section",
|
|
9388
|
-
"schedule",
|
|
9389
|
-
"clause"
|
|
9390
|
-
];
|
|
9391
|
-
var ORGANIZATION_TOP_LEVEL_BATCH_SIZE = 1;
|
|
9392
|
-
var ORGANIZER_MAX_TOP_LEVEL_NODES = 120;
|
|
9393
|
-
var ORGANIZER_MAX_BATCH_NODES = 80;
|
|
9394
|
-
var OUTLINE_CLEANUP_MAX_TOP_LEVEL_NODES = 80;
|
|
9395
|
-
var SourceTreeOrganizationSchema = z42.object({
|
|
9396
|
-
labels: z42.array(z42.object({
|
|
9397
|
-
nodeId: z42.string(),
|
|
9398
|
-
kind: z42.enum([
|
|
9399
|
-
"document",
|
|
9400
|
-
"page_group",
|
|
9401
|
-
"page",
|
|
9402
|
-
"form",
|
|
9403
|
-
"endorsement",
|
|
9404
|
-
"section",
|
|
9405
|
-
"schedule",
|
|
9406
|
-
"clause",
|
|
9407
|
-
"table",
|
|
9408
|
-
"table_row",
|
|
9409
|
-
"table_cell",
|
|
9410
|
-
"text"
|
|
9411
|
-
]).optional(),
|
|
9412
|
-
title: z42.string().optional(),
|
|
9413
|
-
description: z42.string().optional()
|
|
9414
|
-
})),
|
|
9415
|
-
groups: z42.array(z42.object({
|
|
9416
|
-
kind: z42.enum(ORGANIZABLE_KINDS),
|
|
9417
|
-
title: z42.string(),
|
|
9418
|
-
description: z42.string().optional(),
|
|
9419
|
-
parentNodeId: z42.string().optional(),
|
|
9420
|
-
childNodeIds: z42.array(z42.string()).min(1)
|
|
9421
|
-
}))
|
|
9422
|
-
});
|
|
9423
9383
|
var SourceBackedValueForPromptSchema = z42.object({
|
|
9424
9384
|
value: z42.string(),
|
|
9425
9385
|
normalizedValue: z42.string().optional(),
|
|
@@ -9463,17 +9423,6 @@ var OperationalProfilePromptSchema = z42.object({
|
|
|
9463
9423
|
sourceNodeIds: z42.array(z42.string()).optional(),
|
|
9464
9424
|
sourceSpanIds: z42.array(z42.string()).optional()
|
|
9465
9425
|
});
|
|
9466
|
-
function formatFormHintsForPrompt(forms) {
|
|
9467
|
-
const usable = forms.filter((form) => typeof form.pageStart === "number" && typeof form.pageEnd === "number").slice(0, 120).map((form) => ({
|
|
9468
|
-
title: form.title,
|
|
9469
|
-
formType: form.formType,
|
|
9470
|
-
formNumber: form.formNumber,
|
|
9471
|
-
editionDate: form.editionDate,
|
|
9472
|
-
pageStart: form.pageStart,
|
|
9473
|
-
pageEnd: form.pageEnd
|
|
9474
|
-
}));
|
|
9475
|
-
return usable.length ? JSON.stringify(usable, null, 2) : "[]";
|
|
9476
|
-
}
|
|
9477
9426
|
function cleanText(value, fallback) {
|
|
9478
9427
|
const text = value?.replace(/\s+/g, " ").trim();
|
|
9479
9428
|
return text || fallback;
|
|
@@ -10016,15 +9965,6 @@ function applySemanticPageGrouping(sourceTree) {
|
|
|
10016
9965
|
}
|
|
10017
9966
|
return applyEndorsementGrouping(normalizeDocumentSourceTreePaths(nextTree));
|
|
10018
9967
|
}
|
|
10019
|
-
function rejectsOrganizerGroup(group, children) {
|
|
10020
|
-
if (/^endorsements?\b/i.test(group.title) && group.kind !== "page_group") return true;
|
|
10021
|
-
if (group.kind !== "endorsement") return false;
|
|
10022
|
-
if (/^endorsements?\s+\d+\s*[–-]\s*\d+\b/i.test(group.title)) return true;
|
|
10023
|
-
const childNumbers = new Set(
|
|
10024
|
-
children.map((child) => endorsementReference([child.title, child.description, child.textExcerpt].filter(Boolean).join(" "))).filter((value) => Boolean(value))
|
|
10025
|
-
);
|
|
10026
|
-
return childNumbers.size > 1;
|
|
10027
|
-
}
|
|
10028
9968
|
function isEndorsementGroup(node) {
|
|
10029
9969
|
return node.kind === "page_group" && /^endorsements?\b/i.test(node.title);
|
|
10030
9970
|
}
|
|
@@ -10088,10 +10028,10 @@ function rootSemanticRank(node) {
|
|
|
10088
10028
|
function normalizeRootSemanticOrder(sourceTree) {
|
|
10089
10029
|
const rootId = sourceTreeRootId(sourceTree);
|
|
10090
10030
|
if (!rootId) return sourceTree;
|
|
10091
|
-
const
|
|
10031
|
+
const rootChildren = (nodesByParent(sourceTree).get(rootId) ?? []).filter((node) => node.kind !== "document").sort(
|
|
10092
10032
|
(left, right) => rootSemanticRank(left) - rootSemanticRank(right) || (left.pageStart ?? Number.MAX_SAFE_INTEGER) - (right.pageStart ?? Number.MAX_SAFE_INTEGER) || left.order - right.order || left.id.localeCompare(right.id)
|
|
10093
10033
|
);
|
|
10094
|
-
const orderById = new Map(
|
|
10034
|
+
const orderById = new Map(rootChildren.map((node, index) => [node.id, index + 1]));
|
|
10095
10035
|
return sourceTree.map((node) => {
|
|
10096
10036
|
const order = orderById.get(node.id);
|
|
10097
10037
|
return order === void 0 ? node : { ...node, order };
|
|
@@ -10561,174 +10501,6 @@ function nodesByParent(sourceTree) {
|
|
|
10561
10501
|
function sourceTreeRootId(sourceTree) {
|
|
10562
10502
|
return sourceTree.find((node) => node.kind === "document")?.id;
|
|
10563
10503
|
}
|
|
10564
|
-
function rootChildren(sourceTree) {
|
|
10565
|
-
const byParent = nodesByParent(sourceTree);
|
|
10566
|
-
const rootId = sourceTreeRootId(sourceTree);
|
|
10567
|
-
return (byParent.get(rootId) ?? []).filter((node) => node.kind !== "document");
|
|
10568
|
-
}
|
|
10569
|
-
function shouldRunSourceTreeOrganizer(sourceTree, _sourceSpans) {
|
|
10570
|
-
const topLevelCount = rootChildren(sourceTree).length;
|
|
10571
|
-
if (topLevelCount > ORGANIZER_MAX_TOP_LEVEL_NODES) return false;
|
|
10572
|
-
return true;
|
|
10573
|
-
}
|
|
10574
|
-
function organizationCandidateText(node) {
|
|
10575
|
-
return [node.title, node.description, node.textExcerpt].filter(Boolean).join(" ");
|
|
10576
|
-
}
|
|
10577
|
-
function isHighSignalOrganizationNode(node) {
|
|
10578
|
-
if (node.kind === "document") return false;
|
|
10579
|
-
if (["page", "page_group", "form", "endorsement", "section", "schedule", "clause", "table"].includes(node.kind)) {
|
|
10580
|
-
return true;
|
|
10581
|
-
}
|
|
10582
|
-
if (node.kind !== "text" && node.kind !== "table_cell" && node.kind !== "table_row") return false;
|
|
10583
|
-
const text = organizationCandidateText(node);
|
|
10584
|
-
return /\b(SECTION|PART|ARTICLE|SCHEDULE)\s+[IVXLCDM0-9]+/i.test(text) || /\bItem\s+\d+[\.:]/i.test(text) || /^[A-Z]\.\s+\S/.test(cleanText(node.textExcerpt ?? node.title, "")) || /\b(forms? and endorsements?|coverage parts?|limits? of liability|extended reporting period|producer|premium|aggregate|retroactive date|endorsement no\.?)\b/i.test(text);
|
|
10585
|
-
}
|
|
10586
|
-
function organizationBatchNodes(topLevelBatch, byParent) {
|
|
10587
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
10588
|
-
const queue = [...topLevelBatch];
|
|
10589
|
-
while (queue.length > 0 && nodes.size < ORGANIZER_MAX_BATCH_NODES) {
|
|
10590
|
-
const node = queue.shift();
|
|
10591
|
-
if (!node || nodes.has(node.id) || !isHighSignalOrganizationNode(node)) continue;
|
|
10592
|
-
nodes.set(node.id, node);
|
|
10593
|
-
for (const child of byParent.get(node.id) ?? []) {
|
|
10594
|
-
if (nodes.size + queue.length >= ORGANIZER_MAX_BATCH_NODES) break;
|
|
10595
|
-
if (isHighSignalOrganizationNode(child)) queue.push(child);
|
|
10596
|
-
}
|
|
10597
|
-
}
|
|
10598
|
-
return [...nodes.values()];
|
|
10599
|
-
}
|
|
10600
|
-
function organizationBatches(sourceTree) {
|
|
10601
|
-
const byParent = nodesByParent(sourceTree);
|
|
10602
|
-
const rootId = sourceTreeRootId(sourceTree);
|
|
10603
|
-
const topLevelNodes = (byParent.get(rootId) ?? []).filter((node) => node.kind !== "document");
|
|
10604
|
-
if (topLevelNodes.length === 0) {
|
|
10605
|
-
const nodes = sourceTree.filter((node) => node.kind !== "document").slice(0, ORGANIZER_MAX_BATCH_NODES);
|
|
10606
|
-
return [{
|
|
10607
|
-
label: "fallback node prefix because no document root children were found",
|
|
10608
|
-
topLevelNodeIds: nodes.map((node) => node.id),
|
|
10609
|
-
nodes
|
|
10610
|
-
}];
|
|
10611
|
-
}
|
|
10612
|
-
const batches = [];
|
|
10613
|
-
for (let index = 0; index < topLevelNodes.length; index += ORGANIZATION_TOP_LEVEL_BATCH_SIZE) {
|
|
10614
|
-
const topLevelBatch = topLevelNodes.slice(index, index + ORGANIZATION_TOP_LEVEL_BATCH_SIZE);
|
|
10615
|
-
batches.push({
|
|
10616
|
-
label: `top-level nodes ${index + 1}-${index + topLevelBatch.length} of ${topLevelNodes.length}`,
|
|
10617
|
-
topLevelNodeIds: topLevelBatch.map((node) => node.id),
|
|
10618
|
-
nodes: organizationBatchNodes(topLevelBatch, byParent)
|
|
10619
|
-
});
|
|
10620
|
-
}
|
|
10621
|
-
return batches;
|
|
10622
|
-
}
|
|
10623
|
-
function pageRangeOverlaps(leftStart, leftEnd, rightStart, rightEnd) {
|
|
10624
|
-
if (leftStart === void 0 || rightStart === void 0) return true;
|
|
10625
|
-
const leftLast = leftEnd ?? leftStart;
|
|
10626
|
-
const rightLast = rightEnd ?? rightStart;
|
|
10627
|
-
return leftStart <= rightLast && rightStart <= leftLast;
|
|
10628
|
-
}
|
|
10629
|
-
function formHintsForBatch(batch, formHints) {
|
|
10630
|
-
const batchStart = batch.nodes.map((node) => node.pageStart).filter((page) => typeof page === "number").reduce((min, page) => min === void 0 ? page : Math.min(min, page), void 0);
|
|
10631
|
-
const batchEnd = batch.nodes.map((node) => node.pageEnd ?? node.pageStart).filter((page) => typeof page === "number").reduce((max, page) => max === void 0 ? page : Math.max(max, page), void 0);
|
|
10632
|
-
return formHints.filter((form) => pageRangeOverlaps(batchStart, batchEnd, form.pageStart, form.pageEnd)).slice(0, 24);
|
|
10633
|
-
}
|
|
10634
|
-
function mergeOrganizationResults(results) {
|
|
10635
|
-
const labels = /* @__PURE__ */ new Map();
|
|
10636
|
-
const groups = /* @__PURE__ */ new Map();
|
|
10637
|
-
for (const result of results) {
|
|
10638
|
-
for (const label of result.labels) {
|
|
10639
|
-
labels.set(label.nodeId, { ...labels.get(label.nodeId), ...label });
|
|
10640
|
-
}
|
|
10641
|
-
for (const group of result.groups) {
|
|
10642
|
-
const key = `${group.parentNodeId ?? ""}:${group.kind}:${group.childNodeIds.join("|")}`;
|
|
10643
|
-
groups.set(key, group);
|
|
10644
|
-
}
|
|
10645
|
-
}
|
|
10646
|
-
return {
|
|
10647
|
-
labels: [...labels.values()],
|
|
10648
|
-
groups: [...groups.values()]
|
|
10649
|
-
};
|
|
10650
|
-
}
|
|
10651
|
-
function buildOrganizationPrompt(batch, formHints) {
|
|
10652
|
-
const nodes = batch.nodes.map((node) => compactNode2(node, node.kind === "page" ? 520 : 260));
|
|
10653
|
-
const localFormHints = formHintsForBatch(batch, formHints);
|
|
10654
|
-
return `You organize an insurance document source tree.
|
|
10655
|
-
|
|
10656
|
-
Scope:
|
|
10657
|
-
- ${batch.label}
|
|
10658
|
-
- The provided list is a bounded extraction-time batch. It is not necessarily the whole document.
|
|
10659
|
-
- Top-level page/form candidates in this batch: ${JSON.stringify(batch.topLevelNodeIds)}
|
|
10660
|
-
|
|
10661
|
-
Expected form inventory / page ranges:
|
|
10662
|
-
${formatFormHintsForPrompt(localFormHints)}
|
|
10663
|
-
|
|
10664
|
-
Rules:
|
|
10665
|
-
- Use only node IDs from the provided list.
|
|
10666
|
-
- Do not invent text, page numbers, source spans, limits, or policy facts.
|
|
10667
|
-
- You may relabel existing nodes and group adjacent sibling nodes from this batch only when they are clearly one continuous form, one declarations item, one policy section, one schedule, or one clause family.
|
|
10668
|
-
- Treat the form inventory as a page-range hint for the expected order: front matter/notices, declarations, policy form, then endorsements.
|
|
10669
|
-
- Prefer section hierarchy from printed title elements inside a form over page-by-page grouping.
|
|
10670
|
-
- Printed section markers are hard hierarchy boundaries. If source text moves from "SECTION XI" to "SECTION XII", "Section 12", "PART III", or similar sequential headings, create a new sibling section at that marker even when the marker appears mid-page after prior section text.
|
|
10671
|
-
- Declarations pages often use "Item 1.", "Item 2.", ... as the real section structure. Preserve each item as its own section or schedule under "Declarations"; do not bury item labels inside a single table when they mark new declaration fields.
|
|
10672
|
-
- For declaration item tables, a table belongs only to the current item until the next "Item N" marker. Start a new declaration item section at the next marker, even if the parser presents the marker as a table row or cell.
|
|
10673
|
-
- Keep forms-and-endorsements schedules attached to declarations when they appear as "Item 10", "Forms and Endorsements Attached at Inception", or a continuation page listing form numbers. Do not discard that page just because it also contains countersignature or authorized-representative language.
|
|
10674
|
-
- Prefer sections/schedules over table grouping when a table-like node mixes structural labels such as "Item 7", "Item 8", "SECTION XII", "Producer", or "Forms and Endorsements" with values.
|
|
10675
|
-
- Use group.parentNodeId only when the source clearly shows a child was nested under the wrong earlier section/table. Example: a "SECTION XII" marker nested below "SECTION XI" should be grouped with parentNodeId set to the Policy Form node so Section XII becomes a sibling of Section XI.
|
|
10676
|
-
- Use group.parentNodeId to place "Item N" groups under the Declarations node when parser table structure would otherwise leave those items trapped inside one table.
|
|
10677
|
-
- Group adjacent separately numbered endorsements under a single generic "Endorsements" page_group parent, with each individual endorsement preserved as its own child node.
|
|
10678
|
-
- Never create rollup titles such as "Endorsements 1-3 (...)" or merge multiple endorsements into one endorsement node.
|
|
10679
|
-
- Add concise, human-readable titles to generic text, table, row, and cell nodes when the text makes their role clear.
|
|
10680
|
-
- Keep organizer titles terse. Use the printed heading or a compact canonical title such as "Declarations", "Policy Form", "Definitions", or "Endorsement No. 3"; do not add parenthetical summaries.
|
|
10681
|
-
- Groups must list existing childNodeIds only.
|
|
10682
|
-
- Keep descriptions short and useful for search.
|
|
10683
|
-
- Prefer the document's own form titles, endorsement titles, schedules, declarations headings, and page order over keyword-only guessing.
|
|
10684
|
-
|
|
10685
|
-
Source nodes:
|
|
10686
|
-
${JSON.stringify(nodes, null, 2)}
|
|
10687
|
-
|
|
10688
|
-
Return JSON with labels and groups only.`;
|
|
10689
|
-
}
|
|
10690
|
-
function shouldRunOutlineCleanup(sourceTree) {
|
|
10691
|
-
const topLevel = rootChildren(sourceTree);
|
|
10692
|
-
if (topLevel.length === 0 || topLevel.length > OUTLINE_CLEANUP_MAX_TOP_LEVEL_NODES) return false;
|
|
10693
|
-
const genericPages = topLevel.filter((node) => node.kind === "page" && /^Page\s+\d+$/i.test(node.title));
|
|
10694
|
-
const hasDeclarations = topLevel.some((node) => node.title === "Declarations");
|
|
10695
|
-
const hasPolicyForm = topLevel.some((node) => node.title === "Policy Form");
|
|
10696
|
-
const hasEndorsements = topLevel.some((node) => node.title === "Endorsements");
|
|
10697
|
-
if (hasDeclarations && hasPolicyForm && hasEndorsements) return false;
|
|
10698
|
-
return genericPages.length > 0 || topLevel.length > 6 || !hasDeclarations || !hasPolicyForm || !hasEndorsements;
|
|
10699
|
-
}
|
|
10700
|
-
function buildOutlineCleanupPrompt(sourceTree, formHints) {
|
|
10701
|
-
const topLevel = rootChildren(sourceTree).slice(0, OUTLINE_CLEANUP_MAX_TOP_LEVEL_NODES);
|
|
10702
|
-
const nodes = topLevel.map((node) => compactNode2(node, 900));
|
|
10703
|
-
return `You clean a top-level source outline for an insurance policy.
|
|
10704
|
-
|
|
10705
|
-
Expected product-facing order:
|
|
10706
|
-
1. Optional front matter: policy jacket, important notices, privacy notices, OFAC notices, TRIA/terrorism notices, marketing/admin pages, signatures, countersignatures, or other pages that are not the declarations, policy wording, or endorsements.
|
|
10707
|
-
2. Declarations: declarations page(s), schedules, itemized declaration fields, named insured/policy period/premium rows, coverage limit schedules, forms-and-endorsements schedules.
|
|
10708
|
-
3. Policy Form: the main policy wording, insuring agreements, definitions, exclusions, conditions, claim provisions, and general policy terms.
|
|
10709
|
-
4. Endorsements: one generic "Endorsements" page_group containing each separately numbered endorsement as its own child.
|
|
10710
|
-
|
|
10711
|
-
Expected form inventory / page ranges:
|
|
10712
|
-
${formatFormHintsForPrompt(formHints)}
|
|
10713
|
-
|
|
10714
|
-
Rules:
|
|
10715
|
-
- Use only node IDs from this top-level list: ${JSON.stringify(topLevel.map((node) => node.id))}
|
|
10716
|
-
- Group only adjacent top-level nodes.
|
|
10717
|
-
- Do not invent text, pages, source spans, limits, or policy facts.
|
|
10718
|
-
- Do not merge individually numbered endorsements into one endorsement node; use the generic "Endorsements" parent for the series.
|
|
10719
|
-
- Use canonical terse titles: "Notices and Jacket", "Declarations", "Policy Form", "Endorsements", or the printed endorsement number.
|
|
10720
|
-
- Keep page_group descriptions short and include the page range when pages are known, for example "Declarations pages 6-8".
|
|
10721
|
-
- If a page is only an OFAC, privacy, terrorism/TRIA, claim-reporting notice, signature page, or jacket, do not label it as declarations or policy form.
|
|
10722
|
-
- If a page contains declaration items, coverage schedules, premium rows, producer rows, or a forms-and-endorsements schedule, keep it in Declarations even if it also contains countersignature or authorized-representative text.
|
|
10723
|
-
- If a policy-form page contains a later printed section marker such as "SECTION XII \u2014 EXTENDED REPORTING PERIOD", preserve that marker as the start of a new sibling section rather than a paragraph under the previous section.
|
|
10724
|
-
- If the form inventory provides page ranges, keep groups aligned to those ranges unless the source node text clearly contradicts them.
|
|
10725
|
-
- If the existing deterministic outline is already correct, return empty labels and groups.
|
|
10726
|
-
|
|
10727
|
-
Top-level source nodes:
|
|
10728
|
-
${JSON.stringify(nodes, null, 2)}
|
|
10729
|
-
|
|
10730
|
-
Return JSON with labels and groups only.`;
|
|
10731
|
-
}
|
|
10732
10504
|
function operationalProfilePromptNodes(sourceTree) {
|
|
10733
10505
|
return sourceTree.filter((node) => node.kind !== "document").filter((node) => {
|
|
10734
10506
|
if (["page_group", "form", "endorsement", "schedule", "table", "table_row", "table_cell"].includes(node.kind)) {
|
|
@@ -10784,147 +10556,6 @@ ${JSON.stringify(nodes, null, 2)}
|
|
|
10784
10556
|
|
|
10785
10557
|
Return JSON for the operational profile.`;
|
|
10786
10558
|
}
|
|
10787
|
-
function tableCellValueText(cell) {
|
|
10788
|
-
return cleanText(cell.textExcerpt ?? cell.description ?? "", "");
|
|
10789
|
-
}
|
|
10790
|
-
function tableRowTextFromCells(cells) {
|
|
10791
|
-
const text = cells.map((cell) => {
|
|
10792
|
-
const label = cleanText(cell.title, "");
|
|
10793
|
-
const value = tableCellValueText(cell);
|
|
10794
|
-
if (!value) return label;
|
|
10795
|
-
if (!label || value.toLowerCase() === label.toLowerCase()) return value;
|
|
10796
|
-
return `${label}: ${value}`;
|
|
10797
|
-
}).filter(Boolean).join(" | ");
|
|
10798
|
-
return text ? cleanText(text, text) : void 0;
|
|
10799
|
-
}
|
|
10800
|
-
function normalizeTableBoundaryDelimiters(value) {
|
|
10801
|
-
const text = cleanText(value, "");
|
|
10802
|
-
if (!text) return void 0;
|
|
10803
|
-
const normalized = text.replace(/\s+\|\s+\|/g, " | ").replace(/\s+\/\s+\|/g, " |").replace(/\s+[|/]\s*$/g, "").replace(/\s{2,}/g, " ").trim();
|
|
10804
|
-
return normalized || void 0;
|
|
10805
|
-
}
|
|
10806
|
-
function normalizeSourceTreeTableDisplayText(sourceTree) {
|
|
10807
|
-
const byParent = nodesByParent(sourceTree);
|
|
10808
|
-
const updates = /* @__PURE__ */ new Map();
|
|
10809
|
-
const currentNode = (node) => updates.get(node.id) ?? node;
|
|
10810
|
-
for (const node of sourceTree) {
|
|
10811
|
-
if (node.kind !== "table_cell") continue;
|
|
10812
|
-
const textExcerpt = normalizeTableBoundaryDelimiters(node.textExcerpt);
|
|
10813
|
-
const textChanged = textExcerpt !== void 0 && textExcerpt !== node.textExcerpt;
|
|
10814
|
-
const description = textChanged && textExcerpt ? normalizeTableBoundaryDelimiters([node.title, textExcerpt].filter(Boolean).join(" | ")) : normalizeTableBoundaryDelimiters(node.description);
|
|
10815
|
-
if (textExcerpt === node.textExcerpt && description === node.description) continue;
|
|
10816
|
-
updates.set(node.id, {
|
|
10817
|
-
...node,
|
|
10818
|
-
...textExcerpt !== void 0 ? { textExcerpt } : {},
|
|
10819
|
-
...description !== void 0 ? { description } : {}
|
|
10820
|
-
});
|
|
10821
|
-
}
|
|
10822
|
-
for (const node of sourceTree) {
|
|
10823
|
-
if (node.kind !== "table_row") continue;
|
|
10824
|
-
const cells = (byParent.get(node.id) ?? []).filter((child) => child.kind === "table_cell").map(currentNode).sort(
|
|
10825
|
-
(left, right) => tableCellColumnIndex(left, 0) - tableCellColumnIndex(right, 0) || left.order - right.order || left.id.localeCompare(right.id)
|
|
10826
|
-
);
|
|
10827
|
-
const textExcerpt = cells.length ? tableRowTextFromCells(cells) : normalizeTableBoundaryDelimiters(node.textExcerpt);
|
|
10828
|
-
const textChanged = textExcerpt !== void 0 && textExcerpt !== node.textExcerpt;
|
|
10829
|
-
const description = textChanged && textExcerpt ? normalizeTableBoundaryDelimiters([node.title, textExcerpt].filter(Boolean).join(" | ")) : normalizeTableBoundaryDelimiters(node.description);
|
|
10830
|
-
if (textExcerpt === node.textExcerpt && description === node.description) continue;
|
|
10831
|
-
updates.set(node.id, {
|
|
10832
|
-
...node,
|
|
10833
|
-
...textExcerpt !== void 0 ? { textExcerpt } : {},
|
|
10834
|
-
...description !== void 0 ? { description } : {}
|
|
10835
|
-
});
|
|
10836
|
-
}
|
|
10837
|
-
if (updates.size === 0) return sourceTree;
|
|
10838
|
-
return sourceTree.map((node) => updates.get(node.id) ?? node);
|
|
10839
|
-
}
|
|
10840
|
-
function tableCellColumnIndex(cell, fallbackIndex) {
|
|
10841
|
-
const metadataIndex = cell.metadata?.columnIndex;
|
|
10842
|
-
return typeof metadataIndex === "number" && Number.isInteger(metadataIndex) ? metadataIndex : fallbackIndex;
|
|
10843
|
-
}
|
|
10844
|
-
function groupNodeId(documentId, group) {
|
|
10845
|
-
return [
|
|
10846
|
-
documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
|
|
10847
|
-
"source_node",
|
|
10848
|
-
group.kind,
|
|
10849
|
-
group.childNodeIds.join("_").replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 80)
|
|
10850
|
-
].join(":");
|
|
10851
|
-
}
|
|
10852
|
-
function isDescendantOf(nodeId2, ancestorId, byId) {
|
|
10853
|
-
let parentId = nodeId2 ? byId.get(nodeId2)?.parentId : void 0;
|
|
10854
|
-
const seen = /* @__PURE__ */ new Set();
|
|
10855
|
-
while (parentId) {
|
|
10856
|
-
if (parentId === ancestorId) return true;
|
|
10857
|
-
if (seen.has(parentId)) return false;
|
|
10858
|
-
seen.add(parentId);
|
|
10859
|
-
parentId = byId.get(parentId)?.parentId;
|
|
10860
|
-
}
|
|
10861
|
-
return false;
|
|
10862
|
-
}
|
|
10863
|
-
function applyOrganization(sourceTree, organization) {
|
|
10864
|
-
const byId = new Map(sourceTree.map((node) => [node.id, node]));
|
|
10865
|
-
const labels = new Map(organization.labels.map((label) => [label.nodeId, label]));
|
|
10866
|
-
let nextTree = sourceTree.map((node) => {
|
|
10867
|
-
const label = labels.get(node.id);
|
|
10868
|
-
if (!label) return node;
|
|
10869
|
-
return {
|
|
10870
|
-
...node,
|
|
10871
|
-
kind: label.kind ?? node.kind,
|
|
10872
|
-
title: simplifyOrganizerTitle(label.title, node.title, label.kind ?? node.kind),
|
|
10873
|
-
description: cleanText(label.description, node.description)
|
|
10874
|
-
};
|
|
10875
|
-
});
|
|
10876
|
-
for (const group of organization.groups) {
|
|
10877
|
-
const children = group.childNodeIds.map((id2) => byId.get(id2)).filter((node2) => Boolean(node2));
|
|
10878
|
-
if (children.length === 0) continue;
|
|
10879
|
-
if (rejectsOrganizerGroup(group, children)) continue;
|
|
10880
|
-
const originalParentId = children[0].parentId;
|
|
10881
|
-
const requestedParent = group.parentNodeId ? byId.get(group.parentNodeId) : void 0;
|
|
10882
|
-
if (requestedParent && children.some((child) => child.id === requestedParent.id || isDescendantOf(requestedParent.id, child.id, byId))) {
|
|
10883
|
-
continue;
|
|
10884
|
-
}
|
|
10885
|
-
if (requestedParent) {
|
|
10886
|
-
if (!children.every((child) => child.parentId === requestedParent.id || hasAncestor(child, requestedParent.id, byId))) {
|
|
10887
|
-
continue;
|
|
10888
|
-
}
|
|
10889
|
-
} else if (!children.every((child) => child.parentId === originalParentId)) {
|
|
10890
|
-
continue;
|
|
10891
|
-
}
|
|
10892
|
-
const parentId = requestedParent?.id ?? originalParentId;
|
|
10893
|
-
const documentId = children[0].documentId;
|
|
10894
|
-
const title = simplifyOrganizerTitle(group.title, group.title, group.kind);
|
|
10895
|
-
const description = descriptionWithPages(cleanText(group.description, title), children);
|
|
10896
|
-
const id = groupNodeId(documentId, { ...group, title });
|
|
10897
|
-
if (byId.has(id)) continue;
|
|
10898
|
-
const sourceSpanIds = [...new Set(children.flatMap((child) => child.sourceSpanIds))];
|
|
10899
|
-
const pageStarts = children.map((child) => child.pageStart).filter((page) => typeof page === "number");
|
|
10900
|
-
const pageEnds = children.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
|
|
10901
|
-
const order = Math.min(...children.map((child) => child.order));
|
|
10902
|
-
const node = {
|
|
10903
|
-
id,
|
|
10904
|
-
documentId,
|
|
10905
|
-
parentId,
|
|
10906
|
-
kind: group.kind,
|
|
10907
|
-
title,
|
|
10908
|
-
description,
|
|
10909
|
-
textExcerpt: children.map((child) => child.textExcerpt ?? child.description).filter(Boolean).join("\n\n").slice(0, 1600),
|
|
10910
|
-
sourceSpanIds,
|
|
10911
|
-
pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
|
|
10912
|
-
pageEnd: pageEnds.length ? Math.max(...pageEnds) : void 0,
|
|
10913
|
-
bbox: children.flatMap((child) => child.bbox ?? []).slice(0, 12),
|
|
10914
|
-
order,
|
|
10915
|
-
path: "",
|
|
10916
|
-
metadata: { sourceTreeVersion: "v3", organizer: "llm_group" }
|
|
10917
|
-
};
|
|
10918
|
-
nextTree = [
|
|
10919
|
-
...nextTree.map(
|
|
10920
|
-
(child) => group.childNodeIds.includes(child.id) ? { ...child, parentId: id, order: child.order + 1e-3 } : child
|
|
10921
|
-
),
|
|
10922
|
-
node
|
|
10923
|
-
];
|
|
10924
|
-
byId.set(id, node);
|
|
10925
|
-
}
|
|
10926
|
-
return applyEndorsementGrouping(normalizeDocumentSourceTreePaths(nextTree));
|
|
10927
|
-
}
|
|
10928
10559
|
function sourceTreeToOutline(sourceTree) {
|
|
10929
10560
|
const byParent = /* @__PURE__ */ new Map();
|
|
10930
10561
|
for (const node of sourceTree.filter((item) => item.kind !== "document")) {
|
|
@@ -11171,94 +10802,6 @@ async function runSourceTreeExtraction(params) {
|
|
|
11171
10802
|
}
|
|
11172
10803
|
params.trackUsage(usage, report);
|
|
11173
10804
|
};
|
|
11174
|
-
if (shouldRunSourceTreeOrganizer(sourceTree, sourceSpans)) {
|
|
11175
|
-
try {
|
|
11176
|
-
const batches = organizationBatches(sourceTree);
|
|
11177
|
-
const organizationResults = await Promise.all(batches.map(async (batch, batchIndex) => {
|
|
11178
|
-
const label = batches.length > 1 ? `source_tree_organizer_${batchIndex + 1}` : "source_tree_organizer";
|
|
11179
|
-
const budget = params.resolveBudget("extraction_source_tree", 4096);
|
|
11180
|
-
const startedAt = Date.now();
|
|
11181
|
-
const response = await safeGenerateObject(
|
|
11182
|
-
params.generateObject,
|
|
11183
|
-
{
|
|
11184
|
-
prompt: buildOrganizationPrompt(batch, formHints),
|
|
11185
|
-
schema: SourceTreeOrganizationSchema,
|
|
11186
|
-
maxTokens: budget.maxTokens,
|
|
11187
|
-
taskKind: "extraction_source_tree",
|
|
11188
|
-
budgetDiagnostics: budget,
|
|
11189
|
-
trace: {
|
|
11190
|
-
label,
|
|
11191
|
-
batchIndex: batchIndex + 1,
|
|
11192
|
-
batchCount: batches.length,
|
|
11193
|
-
sourceBacked: true
|
|
11194
|
-
}
|
|
11195
|
-
},
|
|
11196
|
-
{
|
|
11197
|
-
fallback: { labels: [], groups: [] },
|
|
11198
|
-
maxRetries: 0,
|
|
11199
|
-
log: params.log,
|
|
11200
|
-
retry: false
|
|
11201
|
-
}
|
|
11202
|
-
);
|
|
11203
|
-
return {
|
|
11204
|
-
batchIndex,
|
|
11205
|
-
label,
|
|
11206
|
-
budget,
|
|
11207
|
-
durationMs: Date.now() - startedAt,
|
|
11208
|
-
usage: response.usage,
|
|
11209
|
-
organization: response.object
|
|
11210
|
-
};
|
|
11211
|
-
}));
|
|
11212
|
-
const organizations = [];
|
|
11213
|
-
for (const result of organizationResults.sort((left, right) => left.batchIndex - right.batchIndex)) {
|
|
11214
|
-
localTrack(result.usage, {
|
|
11215
|
-
taskKind: "extraction_source_tree",
|
|
11216
|
-
label: result.label,
|
|
11217
|
-
maxTokens: result.budget.maxTokens,
|
|
11218
|
-
durationMs: result.durationMs
|
|
11219
|
-
});
|
|
11220
|
-
organizations.push(result.organization);
|
|
11221
|
-
}
|
|
11222
|
-
sourceTree = applyOrganization(sourceTree, mergeOrganizationResults(organizations));
|
|
11223
|
-
} catch (error) {
|
|
11224
|
-
warnings.push(`Source-tree organizer failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
|
|
11225
|
-
}
|
|
11226
|
-
} else {
|
|
11227
|
-
await params.log?.("Deterministic source tree ready; skipped model organizer");
|
|
11228
|
-
}
|
|
11229
|
-
if (shouldRunOutlineCleanup(sourceTree)) {
|
|
11230
|
-
try {
|
|
11231
|
-
const budget = params.resolveBudget("extraction_source_tree", 1600);
|
|
11232
|
-
const maxTokens = Math.min(budget.maxTokens, 1600);
|
|
11233
|
-
const startedAt = Date.now();
|
|
11234
|
-
const response = await safeGenerateObject(
|
|
11235
|
-
params.generateObject,
|
|
11236
|
-
{
|
|
11237
|
-
prompt: buildOutlineCleanupPrompt(sourceTree, formHints),
|
|
11238
|
-
schema: SourceTreeOrganizationSchema,
|
|
11239
|
-
maxTokens,
|
|
11240
|
-
taskKind: "extraction_source_tree",
|
|
11241
|
-
budgetDiagnostics: { ...budget, maxTokens }
|
|
11242
|
-
},
|
|
11243
|
-
{
|
|
11244
|
-
fallback: { labels: [], groups: [] },
|
|
11245
|
-
maxRetries: 0,
|
|
11246
|
-
log: params.log,
|
|
11247
|
-
retry: false
|
|
11248
|
-
}
|
|
11249
|
-
);
|
|
11250
|
-
localTrack(response.usage, {
|
|
11251
|
-
taskKind: "extraction_source_tree",
|
|
11252
|
-
label: "source_tree_outline_cleanup",
|
|
11253
|
-
maxTokens,
|
|
11254
|
-
durationMs: Date.now() - startedAt
|
|
11255
|
-
});
|
|
11256
|
-
sourceTree = applyOrganization(sourceTree, response.object);
|
|
11257
|
-
} catch (error) {
|
|
11258
|
-
warnings.push(`Source-tree outline cleanup failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
|
|
11259
|
-
}
|
|
11260
|
-
}
|
|
11261
|
-
sourceTree = normalizeSourceTreeTableDisplayText(sourceTree);
|
|
11262
10805
|
const emptyProfile = emptyOperationalProfile();
|
|
11263
10806
|
let operationalProfile = emptyProfile;
|
|
11264
10807
|
try {
|