@claritylabs/cl-sdk 3.0.2 → 3.0.4

package/dist/index.mjs

@@ -2301,9 +2301,117 @@ function chunkSourceSpans(spans, options = {}) {
   flush();
   return chunks;
 }
+function normalizeSourceSpans(spans) {
+  const droppedParentSpanIds = /* @__PURE__ */ new Set();
+  const cleaned = [];
+  for (const [index, span] of spans.entries()) {
+    if (span.parentSpanId && droppedParentSpanIds.has(span.parentSpanId)) continue;
+    const normalized = normalizeSourceSpanText(span);
+    if (!normalized) {
+      droppedParentSpanIds.add(span.id);
+      continue;
+    }
+    cleaned.push({ ...normalized, __originalIndex: index });
+  }
+  return mergeTextRuns(cleaned).map(({ __originalIndex: _index, ...span }) => span);
+}
 function sourceUnit(span) {
   return span.sourceUnit ?? span.metadata?.sourceUnit;
 }
+function spanPage(span) {
+  return span.pageStart ?? span.location?.page ?? span.location?.startPage;
+}
+function normalizeSourceSpanText(span) {
+  const unit = sourceUnit(span);
+  const text = normalizeWhitespace(span.text);
+  if (!text) return void 0;
+  if (isDiscardableBoilerplate(text, unit)) return void 0;
+  const cleanedText = cleanBoilerplateLines(text);
+  if (!cleanedText) return void 0;
+  if (cleanedText === text) return span;
+  return retextSpan(span, cleanedText, {
+    boilerplateRemoved: "true",
+    removedBoilerplateText: removedBoilerplateLines(text).join(" | ").slice(0, 500)
+  });
+}
+function isDiscardableBoilerplate(text, unit) {
+  const cleaned = normalizeWhitespace(text.replace(/\bColumn\s+\d+:\s*/gi, ""));
+  if (/^SPECIMEN POLICY\s+[-—]\s+FOR TESTING ONLY$/i.test(cleaned)) return true;
+  if (/^Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  if (/^[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}$/i.test(cleaned)) return true;
+  if (/^[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}\s*\|\s*Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  if (unit === "table_row" && /^[^|]{0,40}\|\s*Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  return false;
+}
+function isBoilerplateLine(line) {
+  const cleaned = normalizeWhitespace(line.replace(/\bColumn\s+\d+:\s*/gi, ""));
+  return isDiscardableBoilerplate(cleaned) || /^IMPORTANT NOTICE\s*[-—]\s*/i.test(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
+}
+function removedBoilerplateLines(text) {
+  return text.split(/\s{2,}|\r?\n/).map(normalizeWhitespace).filter((line) => line && isBoilerplateLine(line));
+}
+function cleanBoilerplateLines(text) {
+  const withoutInlineBoilerplate = text.replace(/\b(?:Column\s+\d+:\s*)?[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}\s+(?:\|\s*)?(?:Column\s+\d+:\s*)?Page\s+\d+\s+of\s+\d+\b/gi, " ").replace(/\bSPECIMEN POLICY\s+[-—]\s+FOR TESTING ONLY\b/gi, " ").replace(/\bPage\s+\d+\s+of\s+\d+\b/gi, " ");
+  const lines = withoutInlineBoilerplate.split(/\r?\n/);
+  const filtered = lines.map(normalizeWhitespace).filter((line) => line && !isBoilerplateLine(line));
+  return normalizeWhitespace(filtered.join(" "));
+}
+function shouldMergeTextSpan(left, right) {
+  if (sourceUnit(left) !== "text" || sourceUnit(right) !== "text") return false;
+  if (spanPage(left) !== spanPage(right)) return false;
+  if (left.metadata?.elementType === "title" || right.metadata?.elementType === "title") return false;
+  const leftText = normalizeWhitespace(left.text);
+  const rightText = normalizeWhitespace(right.text);
+  if (!leftText || !rightText) return false;
+  if (/[:.;!?)]$/.test(leftText)) return false;
+  if (/^(?:[A-Z][A-Z0-9 &/(),.-]{8,}|Item\s+\d+|Section\s+\d+|Part\s+[A-Z]\b)/.test(rightText)) return false;
+  return /^[a-z(]/.test(rightText) || /\b(?:a|an|and|any|as|at|by|for|from|in|into|may|must|of|or|that|the|this|to|with|within|you|your)$/i.test(leftText);
+}
+function mergeTextRuns(spans) {
+  const result = [];
+  let current;
+  for (const span of spans) {
+    if (current && shouldMergeTextSpan(current, span)) {
+      current = mergeTextSpanPair(current, span);
+      continue;
+    }
+    if (current) result.push(current);
+    current = span;
+  }
+  if (current) result.push(current);
+  return result;
+}
+function mergeTextSpanPair(left, right) {
+  const text = normalizeWhitespace(`${left.text} ${right.text}`);
+  const merged = retextSpan(left, text, {
+    mergedSourceSpanIds: [left.metadata?.mergedSourceSpanIds, left.id, right.id, right.metadata?.mergedSourceSpanIds].filter(Boolean).join(","),
+    sourceSpanNormalization: "merged_text_run"
+  });
+  return {
+    ...merged,
+    bbox: [...left.bbox ?? [], ...right.bbox ?? []],
+    pageEnd: right.pageEnd ?? left.pageEnd,
+    location: {
+      ...left.location,
+      endPage: right.location?.endPage ?? right.pageEnd ?? left.location?.endPage
+    },
+    __originalIndex: left.__originalIndex
+  };
+}
+function retextSpan(span, text, metadata) {
+  const textHash = sourceSpanTextHash(text);
+  return SourceSpanSchema.parse({
+    ...span,
+    id: `${span.id.split(":").slice(0, -1).join(":")}:${textHash.slice(0, 12)}`,
+    text,
+    hash: textHash,
+    textHash,
+    metadata: {
+      ...span.metadata ?? {},
+      ...metadata
+    }
+  });
+}
 function filterChunkableSourceSpans(spans) {
   const rowIds = new Set(
     spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
@@ -4060,14 +4168,14 @@ function stringArray(value) {
 function sourceUnit4(span) {
   return span.sourceUnit ?? span.metadata?.sourceUnit;
 }
-function spanPage(span) {
+function spanPage2(span) {
   return span.pageStart ?? span.location?.page ?? span.location?.startPage;
 }
 function spanPageEnd(span) {
-  return span.pageEnd ?? span.location?.endPage ?? spanPage(span);
+  return span.pageEnd ?? span.location?.endPage ?? spanPage2(span);
 }
 function sourceSpansForPage(sourceSpans, page) {
-  return sourceSpans.filter((span) => spanPage(span) === page && sourceUnit4(span) === "page").map((span) => span.id);
+  return sourceSpans.filter((span) => spanPage2(span) === page && sourceUnit4(span) === "page").map((span) => span.id);
 }
 function nodePageOverlaps(node, record) {
   const recordStart = numberValue(record, "pageNumber", "pageStart", "resolvedFromPage");
@@ -4168,8 +4276,8 @@ function buildNodesFromSourceSpans(sourceSpans) {
     const unit = sourceUnit4(span);
     return unit === "section" || unit === "section_candidate" || unit === "page";
   });
-  return candidates.sort((left, right) => (spanPage(left) ?? 0) - (spanPage(right) ?? 0) || left.id.localeCompare(right.id)).map((span, index) => {
-    const title = span.sectionId ?? span.formNumber ?? (sourceUnit4(span) === "page" && spanPage(span) ? `Page ${spanPage(span)}` : `Source unit ${index + 1}`);
+  return candidates.sort((left, right) => (spanPage2(left) ?? 0) - (spanPage2(right) ?? 0) || left.id.localeCompare(right.id)).map((span, index) => {
+    const title = span.sectionId ?? span.formNumber ?? (sourceUnit4(span) === "page" && spanPage2(span) ? `Page ${spanPage2(span)}` : `Source unit ${index + 1}`);
     return {
       id: `source:${index}:${slugPart(span.id)}`,
       title,
@@ -4177,7 +4285,7 @@ function buildNodesFromSourceSpans(sourceSpans) {
       type: sourceUnit4(span),
       label: sourceUnit4(span),
       level: 1,
-      pageStart: spanPage(span),
+      pageStart: spanPage2(span),
       pageEnd: spanPageEnd(span),
       formNumber: span.formNumber,
       excerpt: span.text.slice(0, 500),
@@ -8890,6 +8998,8 @@ var ORGANIZABLE_KINDS = [
   "schedule",
   "clause"
 ];
+var ORGANIZATION_TOP_LEVEL_BATCH_SIZE = 80;
+var ORGANIZATION_CHILD_CONTEXT_LIMIT = 4;
 var SourceTreeOrganizationSchema = z41.object({
   labels: z41.array(z41.object({
     nodeId: z41.string(),
@@ -8954,7 +9064,7 @@ function cleanText(value, fallback) {
   const text = value?.replace(/\s+/g, " ").trim();
   return text || fallback;
 }
-function compactNode(node) {
+function compactNode(node, maxText = 700) {
   return {
     id: node.id,
     kind: node.kind,
@@ -8963,20 +9073,89 @@ function compactNode(node) {
     pageStart: node.pageStart,
     pageEnd: node.pageEnd,
     sourceSpanIds: node.sourceSpanIds.slice(0, 8),
-    text: (node.textExcerpt ?? node.description).slice(0, 700)
+    text: (node.textExcerpt ?? node.description).slice(0, maxText)
   };
 }
-function buildOrganizationPrompt(sourceTree) {
-  const nodes = sourceTree.filter((node) => node.kind !== "document").slice(0, 240).map(compactNode);
+function nodesByParent(sourceTree) {
+  const byParent = /* @__PURE__ */ new Map();
+  for (const node of sourceTree) {
+    const children = byParent.get(node.parentId) ?? [];
+    children.push(node);
+    byParent.set(node.parentId, children);
+  }
+  for (const children of byParent.values()) {
+    children.sort((left, right) => left.order - right.order || left.id.localeCompare(right.id));
+  }
+  return byParent;
+}
+function sourceTreeRootId(sourceTree) {
+  return sourceTree.find((node) => node.kind === "document")?.id;
+}
+function organizationBatches(sourceTree) {
+  const byParent = nodesByParent(sourceTree);
+  const rootId = sourceTreeRootId(sourceTree);
+  const topLevelNodes = (byParent.get(rootId) ?? []).filter((node) => node.kind !== "document");
+  if (topLevelNodes.length === 0) {
+    const nodes = sourceTree.filter((node) => node.kind !== "document").slice(0, 240);
+    return [{
+      label: "fallback node prefix because no document root children were found",
+      topLevelNodeIds: nodes.map((node) => node.id),
+      nodes
+    }];
+  }
+  const batches = [];
+  for (let index = 0; index < topLevelNodes.length; index += ORGANIZATION_TOP_LEVEL_BATCH_SIZE) {
+    const topLevelBatch = topLevelNodes.slice(index, index + ORGANIZATION_TOP_LEVEL_BATCH_SIZE);
+    const candidates = /* @__PURE__ */ new Map();
+    for (const node of topLevelBatch) {
+      candidates.set(node.id, node);
+      const childContext = (byParent.get(node.id) ?? []).filter((child) => child.kind !== "text" && child.kind !== "table_cell").slice(0, ORGANIZATION_CHILD_CONTEXT_LIMIT);
+      for (const child of childContext) {
+        candidates.set(child.id, child);
+      }
+    }
+    batches.push({
+      label: `top-level nodes ${index + 1}-${index + topLevelBatch.length} of ${topLevelNodes.length}`,
+      topLevelNodeIds: topLevelBatch.map((node) => node.id),
+      nodes: [...candidates.values()]
+    });
+  }
+  return batches;
+}
+function mergeOrganizationResults(results) {
+  const labels = /* @__PURE__ */ new Map();
+  const groups = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    for (const label of result.labels) {
+      labels.set(label.nodeId, { ...labels.get(label.nodeId), ...label });
+    }
+    for (const group of result.groups) {
+      const key = `${group.kind}:${group.childNodeIds.join("|")}`;
+      groups.set(key, group);
+    }
+  }
+  return {
+    labels: [...labels.values()],
+    groups: [...groups.values()]
+  };
+}
+function buildOrganizationPrompt(batch) {
+  const nodes = batch.nodes.map((node) => compactNode(node, node.kind === "page" ? 900 : 320));
   return `You organize an insurance document source tree.
 
+Scope:
+- ${batch.label}
+- The provided list is a bounded extraction-time batch. It is not necessarily the whole document.
+- Top-level page/form candidates in this batch: ${JSON.stringify(batch.topLevelNodeIds)}
+
 Rules:
 - Use only node IDs from the provided list.
 - Do not invent text, page numbers, source spans, limits, or policy facts.
-- You may relabel existing nodes and group adjacent top-level/page nodes when they are clearly one form, endorsement, declarations set, schedule, or clause family.
+- You may relabel existing nodes and group adjacent top-level/page nodes from this batch when they are clearly one form, endorsement, declarations set, schedule, or clause family.
 - Add concise, human-readable titles to generic text, table, row, and cell nodes when the text makes their role clear.
 - Groups must list existing childNodeIds only.
 - Keep descriptions short and useful for search.
+- Prefer the document's own form titles, endorsement titles, schedules, declarations headings, and page order over keyword-only guessing.
 
 Source nodes:
 ${JSON.stringify(nodes, null, 2)}
@@ -9016,8 +9195,9 @@ function groupNodeId(documentId, group) {
 }
 function applyOrganization(sourceTree, organization) {
   const byId = new Map(sourceTree.map((node) => [node.id, node]));
+  const labels = new Map(organization.labels.map((label) => [label.nodeId, label]));
   let nextTree = sourceTree.map((node) => {
-    const label = organization.labels.find((item) => item.nodeId === node.id);
+    const label = labels.get(node.id);
     if (!label) return node;
     return {
       ...node,
@@ -9026,7 +9206,7 @@ function applyOrganization(sourceTree, organization) {
       description: cleanText(label.description, node.description)
     };
   });
-  for (const group of organization.groups.slice(0, 40)) {
+  for (const group of organization.groups) {
     const children = group.childNodeIds.map((id2) => byId.get(id2)).filter((node2) => Boolean(node2));
     if (children.length === 0) continue;
     const parentId = children[0].parentId;
@@ -9185,7 +9365,8 @@ function materializeDocument(params) {
   };
 }
 async function runSourceTreeExtraction(params) {
-  let sourceTree = buildDocumentSourceTree(params.sourceSpans, params.id);
+  const sourceSpans = normalizeSourceSpans(params.sourceSpans);
+  let sourceTree = buildDocumentSourceTree(sourceSpans, params.id);
   const warnings = [];
   let modelCalls = 0;
   let callsWithUsage = 0;
@@ -9208,41 +9389,46 @@ async function runSourceTreeExtraction(params) {
     params.trackUsage(usage, report);
   };
   try {
-    const budget = params.resolveBudget("extraction_source_tree", 4096);
-    const startedAt = Date.now();
-    const response = await safeGenerateObject(
-      params.generateObject,
-      {
-        prompt: buildOrganizationPrompt(sourceTree),
-        schema: SourceTreeOrganizationSchema,
-        maxTokens: budget.maxTokens,
+    const organizations = [];
+    const batches = organizationBatches(sourceTree);
+    for (const [batchIndex, batch] of batches.entries()) {
+      const budget = params.resolveBudget("extraction_source_tree", 4096);
+      const startedAt = Date.now();
+      const response = await safeGenerateObject(
+        params.generateObject,
+        {
+          prompt: buildOrganizationPrompt(batch),
+          schema: SourceTreeOrganizationSchema,
+          maxTokens: budget.maxTokens,
+          taskKind: "extraction_source_tree",
+          budgetDiagnostics: budget,
+          providerOptions: { ...params.providerOptions, sourceSpans }
+        },
+        {
+          fallback: { labels: [], groups: [] },
+          log: params.log
+        }
+      );
+      localTrack(response.usage, {
         taskKind: "extraction_source_tree",
-        budgetDiagnostics: budget,
-        providerOptions: { ...params.providerOptions, sourceSpans: params.sourceSpans }
-      },
-      {
-        fallback: { labels: [], groups: [] },
-        log: params.log
-      }
-    );
-    localTrack(response.usage, {
-      taskKind: "extraction_source_tree",
-      label: "source_tree_organizer",
-      maxTokens: budget.maxTokens,
-      durationMs: Date.now() - startedAt
-    });
-    sourceTree = applyOrganization(sourceTree, response.object);
+        label: batches.length > 1 ? `source_tree_organizer_${batchIndex + 1}` : "source_tree_organizer",
+        maxTokens: budget.maxTokens,
+        durationMs: Date.now() - startedAt
+      });
+      organizations.push(response.object);
+    }
+    sourceTree = applyOrganization(sourceTree, mergeOrganizationResults(organizations));
   } catch (error) {
     warnings.push(`Source-tree organizer failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
   }
   const deterministicProfile = buildDeterministicOperationalProfile({
     sourceTree,
-    sourceSpans: params.sourceSpans
+    sourceSpans
   });
   let operationalProfile = deterministicProfile;
   try {
     const validNodeIds = new Set(sourceTree.map((node) => node.id));
-    const validSpanIds = new Set(params.sourceSpans.map((span) => span.id));
+    const validSpanIds = new Set(sourceSpans.map((span) => span.id));
     const budget = params.resolveBudget("extraction_operational_profile", 8192);
     const startedAt = Date.now();
     const response = await safeGenerateObject(
@@ -9253,7 +9439,7 @@ async function runSourceTreeExtraction(params) {
         maxTokens: budget.maxTokens,
         taskKind: "extraction_operational_profile",
         budgetDiagnostics: budget,
-        providerOptions: { ...params.providerOptions, sourceSpans: params.sourceSpans, sourceTree }
+        providerOptions: { ...params.providerOptions, sourceSpans, sourceTree }
       },
       {
         fallback: deterministicProfile,
@@ -9282,8 +9468,8 @@ async function runSourceTreeExtraction(params) {
   });
   return {
     sourceTree,
-    sourceSpans: params.sourceSpans,
-    sourceChunks: chunkSourceSpans(params.sourceSpans),
+    sourceSpans,
+    sourceChunks: chunkSourceSpans(sourceSpans),
     operationalProfile,
     document,
     chunks: [],
@@ -14297,6 +14483,7 @@ export {
   normalizeDoclingDocument,
   normalizeDocumentSourceTreePaths,
   normalizeForMatch,
+  normalizeSourceSpans,
   orderSourceEvidence,
   overlayTextOnPdf,
   pLimit,