@claritylabs/cl-sdk 3.0.3 → 3.0.4

package/dist/index.mjs

@@ -2301,9 +2301,117 @@ function chunkSourceSpans(spans, options = {}) {
   flush();
   return chunks;
 }
+function normalizeSourceSpans(spans) {
+  const droppedParentSpanIds = /* @__PURE__ */ new Set();
+  const cleaned = [];
+  for (const [index, span] of spans.entries()) {
+    if (span.parentSpanId && droppedParentSpanIds.has(span.parentSpanId)) continue;
+    const normalized = normalizeSourceSpanText(span);
+    if (!normalized) {
+      droppedParentSpanIds.add(span.id);
+      continue;
+    }
+    cleaned.push({ ...normalized, __originalIndex: index });
+  }
+  return mergeTextRuns(cleaned).map(({ __originalIndex: _index, ...span }) => span);
+}
 function sourceUnit(span) {
   return span.sourceUnit ?? span.metadata?.sourceUnit;
 }
+function spanPage(span) {
+  return span.pageStart ?? span.location?.page ?? span.location?.startPage;
+}
+function normalizeSourceSpanText(span) {
+  const unit = sourceUnit(span);
+  const text = normalizeWhitespace(span.text);
+  if (!text) return void 0;
+  if (isDiscardableBoilerplate(text, unit)) return void 0;
+  const cleanedText = cleanBoilerplateLines(text);
+  if (!cleanedText) return void 0;
+  if (cleanedText === text) return span;
+  return retextSpan(span, cleanedText, {
+    boilerplateRemoved: "true",
+    removedBoilerplateText: removedBoilerplateLines(text).join(" | ").slice(0, 500)
+  });
+}
+function isDiscardableBoilerplate(text, unit) {
+  const cleaned = normalizeWhitespace(text.replace(/\bColumn\s+\d+:\s*/gi, ""));
+  if (/^SPECIMEN POLICY\s+[-—]\s+FOR TESTING ONLY$/i.test(cleaned)) return true;
+  if (/^Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  if (/^[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}$/i.test(cleaned)) return true;
+  if (/^[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}\s*\|\s*Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  if (unit === "table_row" && /^[^|]{0,40}\|\s*Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  return false;
+}
+function isBoilerplateLine(line) {
+  const cleaned = normalizeWhitespace(line.replace(/\bColumn\s+\d+:\s*/gi, ""));
+  return isDiscardableBoilerplate(cleaned) || /^IMPORTANT NOTICE\s*[-—]\s*/i.test(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
+}
+function removedBoilerplateLines(text) {
+  return text.split(/\s{2,}|\r?\n/).map(normalizeWhitespace).filter((line) => line && isBoilerplateLine(line));
+}
+function cleanBoilerplateLines(text) {
+  const withoutInlineBoilerplate = text.replace(/\b(?:Column\s+\d+:\s*)?[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}\s+(?:\|\s*)?(?:Column\s+\d+:\s*)?Page\s+\d+\s+of\s+\d+\b/gi, " ").replace(/\bSPECIMEN POLICY\s+[-—]\s+FOR TESTING ONLY\b/gi, " ").replace(/\bPage\s+\d+\s+of\s+\d+\b/gi, " ");
+  const lines = withoutInlineBoilerplate.split(/\r?\n/);
+  const filtered = lines.map(normalizeWhitespace).filter((line) => line && !isBoilerplateLine(line));
+  return normalizeWhitespace(filtered.join(" "));
+}
+function shouldMergeTextSpan(left, right) {
+  if (sourceUnit(left) !== "text" || sourceUnit(right) !== "text") return false;
+  if (spanPage(left) !== spanPage(right)) return false;
+  if (left.metadata?.elementType === "title" || right.metadata?.elementType === "title") return false;
+  const leftText = normalizeWhitespace(left.text);
+  const rightText = normalizeWhitespace(right.text);
+  if (!leftText || !rightText) return false;
+  if (/[:.;!?)]$/.test(leftText)) return false;
+  if (/^(?:[A-Z][A-Z0-9 &/(),.-]{8,}|Item\s+\d+|Section\s+\d+|Part\s+[A-Z]\b)/.test(rightText)) return false;
+  return /^[a-z(]/.test(rightText) || /\b(?:a|an|and|any|as|at|by|for|from|in|into|may|must|of|or|that|the|this|to|with|within|you|your)$/i.test(leftText);
+}
+function mergeTextRuns(spans) {
+  const result = [];
+  let current;
+  for (const span of spans) {
+    if (current && shouldMergeTextSpan(current, span)) {
+      current = mergeTextSpanPair(current, span);
+      continue;
+    }
+    if (current) result.push(current);
+    current = span;
+  }
+  if (current) result.push(current);
+  return result;
+}
+function mergeTextSpanPair(left, right) {
+  const text = normalizeWhitespace(`${left.text} ${right.text}`);
+  const merged = retextSpan(left, text, {
+    mergedSourceSpanIds: [left.metadata?.mergedSourceSpanIds, left.id, right.id, right.metadata?.mergedSourceSpanIds].filter(Boolean).join(","),
+    sourceSpanNormalization: "merged_text_run"
+  });
+  return {
+    ...merged,
+    bbox: [...left.bbox ?? [], ...right.bbox ?? []],
+    pageEnd: right.pageEnd ?? left.pageEnd,
+    location: {
+      ...left.location,
+      endPage: right.location?.endPage ?? right.pageEnd ?? left.location?.endPage
+    },
+    __originalIndex: left.__originalIndex
+  };
+}
+function retextSpan(span, text, metadata) {
+  const textHash = sourceSpanTextHash(text);
+  return SourceSpanSchema.parse({
+    ...span,
+    id: `${span.id.split(":").slice(0, -1).join(":")}:${textHash.slice(0, 12)}`,
+    text,
+    hash: textHash,
+    textHash,
+    metadata: {
+      ...span.metadata ?? {},
+      ...metadata
+    }
+  });
+}
 function filterChunkableSourceSpans(spans) {
   const rowIds = new Set(
     spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
@@ -4060,14 +4168,14 @@ function stringArray(value) {
 function sourceUnit4(span) {
   return span.sourceUnit ?? span.metadata?.sourceUnit;
 }
-function spanPage(span) {
+function spanPage2(span) {
   return span.pageStart ?? span.location?.page ?? span.location?.startPage;
 }
 function spanPageEnd(span) {
-  return span.pageEnd ?? span.location?.endPage ?? spanPage(span);
+  return span.pageEnd ?? span.location?.endPage ?? spanPage2(span);
 }
 function sourceSpansForPage(sourceSpans, page) {
-  return sourceSpans.filter((span) => spanPage(span) === page && sourceUnit4(span) === "page").map((span) => span.id);
+  return sourceSpans.filter((span) => spanPage2(span) === page && sourceUnit4(span) === "page").map((span) => span.id);
 }
 function nodePageOverlaps(node, record) {
   const recordStart = numberValue(record, "pageNumber", "pageStart", "resolvedFromPage");
@@ -4168,8 +4276,8 @@ function buildNodesFromSourceSpans(sourceSpans) {
     const unit = sourceUnit4(span);
     return unit === "section" || unit === "section_candidate" || unit === "page";
   });
-  return candidates.sort((left, right) => (spanPage(left) ?? 0) - (spanPage(right) ?? 0) || left.id.localeCompare(right.id)).map((span, index) => {
-    const title = span.sectionId ?? span.formNumber ?? (sourceUnit4(span) === "page" && spanPage(span) ? `Page ${spanPage(span)}` : `Source unit ${index + 1}`);
+  return candidates.sort((left, right) => (spanPage2(left) ?? 0) - (spanPage2(right) ?? 0) || left.id.localeCompare(right.id)).map((span, index) => {
+    const title = span.sectionId ?? span.formNumber ?? (sourceUnit4(span) === "page" && spanPage2(span) ? `Page ${spanPage2(span)}` : `Source unit ${index + 1}`);
     return {
       id: `source:${index}:${slugPart(span.id)}`,
       title,
@@ -4177,7 +4285,7 @@ function buildNodesFromSourceSpans(sourceSpans) {
       type: sourceUnit4(span),
       label: sourceUnit4(span),
       level: 1,
-      pageStart: spanPage(span),
+      pageStart: spanPage2(span),
       pageEnd: spanPageEnd(span),
       formNumber: span.formNumber,
       excerpt: span.text.slice(0, 500),
@@ -9257,7 +9365,8 @@ function materializeDocument(params) {
   };
 }
 async function runSourceTreeExtraction(params) {
-  let sourceTree = buildDocumentSourceTree(params.sourceSpans, params.id);
+  const sourceSpans = normalizeSourceSpans(params.sourceSpans);
+  let sourceTree = buildDocumentSourceTree(sourceSpans, params.id);
   const warnings = [];
   let modelCalls = 0;
   let callsWithUsage = 0;
@@ -9293,7 +9402,7 @@ async function runSourceTreeExtraction(params) {
           maxTokens: budget.maxTokens,
           taskKind: "extraction_source_tree",
           budgetDiagnostics: budget,
-          providerOptions: { ...params.providerOptions, sourceSpans: params.sourceSpans }
+          providerOptions: { ...params.providerOptions, sourceSpans }
         },
         {
           fallback: { labels: [], groups: [] },
@@ -9314,12 +9423,12 @@ async function runSourceTreeExtraction(params) {
   }
   const deterministicProfile = buildDeterministicOperationalProfile({
     sourceTree,
-    sourceSpans: params.sourceSpans
+    sourceSpans
   });
   let operationalProfile = deterministicProfile;
   try {
     const validNodeIds = new Set(sourceTree.map((node) => node.id));
-    const validSpanIds = new Set(params.sourceSpans.map((span) => span.id));
+    const validSpanIds = new Set(sourceSpans.map((span) => span.id));
     const budget = params.resolveBudget("extraction_operational_profile", 8192);
     const startedAt = Date.now();
     const response = await safeGenerateObject(
@@ -9330,7 +9439,7 @@ async function runSourceTreeExtraction(params) {
         maxTokens: budget.maxTokens,
         taskKind: "extraction_operational_profile",
         budgetDiagnostics: budget,
-        providerOptions: { ...params.providerOptions, sourceSpans: params.sourceSpans, sourceTree }
+        providerOptions: { ...params.providerOptions, sourceSpans, sourceTree }
       },
       {
         fallback: deterministicProfile,
@@ -9359,8 +9468,8 @@ async function runSourceTreeExtraction(params) {
   });
   return {
     sourceTree,
-    sourceSpans: params.sourceSpans,
-    sourceChunks: chunkSourceSpans(params.sourceSpans),
+    sourceSpans,
+    sourceChunks: chunkSourceSpans(sourceSpans),
     operationalProfile,
     document,
     chunks: [],
@@ -14374,6 +14483,7 @@ export {
   normalizeDoclingDocument,
   normalizeDocumentSourceTreePaths,
   normalizeForMatch,
+  normalizeSourceSpans,
   orderSourceEvidence,
   overlayTextOnPdf,
   pLimit,