@claritylabs/cl-sdk 3.0.3 → 3.0.5

package/dist/index.mjs

@@ -2301,9 +2301,117 @@ function chunkSourceSpans(spans, options = {}) {
   flush();
   return chunks;
 }
+function normalizeSourceSpans(spans) {
+  const droppedParentSpanIds = /* @__PURE__ */ new Set();
+  const cleaned = [];
+  for (const [index, span] of spans.entries()) {
+    if (span.parentSpanId && droppedParentSpanIds.has(span.parentSpanId)) continue;
+    const normalized = normalizeSourceSpanText(span);
+    if (!normalized) {
+      droppedParentSpanIds.add(span.id);
+      continue;
+    }
+    cleaned.push({ ...normalized, __originalIndex: index });
+  }
+  return mergeTextRuns(cleaned).map(({ __originalIndex: _index, ...span }) => span);
+}
 function sourceUnit(span) {
   return span.sourceUnit ?? span.metadata?.sourceUnit;
 }
+function spanPage(span) {
+  return span.pageStart ?? span.location?.page ?? span.location?.startPage;
+}
+function normalizeSourceSpanText(span) {
+  const unit = sourceUnit(span);
+  const text = normalizeWhitespace(span.text);
+  if (!text) return void 0;
+  if (isDiscardableBoilerplate(text, unit)) return void 0;
+  const cleanedText = cleanBoilerplateLines(text);
+  if (!cleanedText) return void 0;
+  if (cleanedText === text) return span;
+  return retextSpan(span, cleanedText, {
+    boilerplateRemoved: "true",
+    removedBoilerplateText: removedBoilerplateLines(text).join(" | ").slice(0, 500)
+  });
+}
+function isDiscardableBoilerplate(text, unit) {
+  const cleaned = normalizeWhitespace(text.replace(/\bColumn\s+\d+:\s*/gi, ""));
+  if (/^SPECIMEN POLICY\s+[-—]\s+FOR TESTING ONLY$/i.test(cleaned)) return true;
+  if (/^Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  if (/^[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}$/i.test(cleaned)) return true;
+  if (/^[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}\s*\|\s*Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  if (unit === "table_row" && /^[^|]{0,40}\|\s*Page\s+\d+\s+of\s+\d+$/i.test(cleaned)) return true;
+  return false;
+}
+function isBoilerplateLine(line) {
+  const cleaned = normalizeWhitespace(line.replace(/\bColumn\s+\d+:\s*/gi, ""));
+  return isDiscardableBoilerplate(cleaned) || /^IMPORTANT NOTICE\s*[-—]\s*/i.test(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
+}
+function removedBoilerplateLines(text) {
+  return text.split(/\s{2,}|\r?\n/).map(normalizeWhitespace).filter((line) => line && isBoilerplateLine(line));
+}
+function cleanBoilerplateLines(text) {
+  const withoutInlineBoilerplate = text.replace(/\b(?:Column\s+\d+:\s*)?[A-Z]{2,}(?:-[A-Z0-9]{2,})+\s+\d{2}\s+\d{2}\s+(?:\|\s*)?(?:Column\s+\d+:\s*)?Page\s+\d+\s+of\s+\d+\b/gi, " ").replace(/\bSPECIMEN POLICY\s+[-—]\s+FOR TESTING ONLY\b/gi, " ").replace(/\bPage\s+\d+\s+of\s+\d+\b/gi, " ");
+  const lines = withoutInlineBoilerplate.split(/\r?\n/);
+  const filtered = lines.map(normalizeWhitespace).filter((line) => line && !isBoilerplateLine(line));
+  return normalizeWhitespace(filtered.join(" "));
+}
+function shouldMergeTextSpan(left, right) {
+  if (sourceUnit(left) !== "text" || sourceUnit(right) !== "text") return false;
+  if (spanPage(left) !== spanPage(right)) return false;
+  if (left.metadata?.elementType === "title" || right.metadata?.elementType === "title") return false;
+  const leftText = normalizeWhitespace(left.text);
+  const rightText = normalizeWhitespace(right.text);
+  if (!leftText || !rightText) return false;
+  if (/[:.;!?)]$/.test(leftText)) return false;
+  if (/^(?:[A-Z][A-Z0-9 &/(),.-]{8,}|Item\s+\d+|Section\s+\d+|Part\s+[A-Z]\b)/.test(rightText)) return false;
+  return /^[a-z(]/.test(rightText) || /\b(?:a|an|and|any|as|at|by|for|from|in|into|may|must|of|or|that|the|this|to|with|within|you|your)$/i.test(leftText);
+}
+function mergeTextRuns(spans) {
+  const result = [];
+  let current;
+  for (const span of spans) {
+    if (current && shouldMergeTextSpan(current, span)) {
+      current = mergeTextSpanPair(current, span);
+      continue;
+    }
+    if (current) result.push(current);
+    current = span;
+  }
+  if (current) result.push(current);
+  return result;
+}
+function mergeTextSpanPair(left, right) {
+  const text = normalizeWhitespace(`${left.text} ${right.text}`);
+  const merged = retextSpan(left, text, {
+    mergedSourceSpanIds: [left.metadata?.mergedSourceSpanIds, left.id, right.id, right.metadata?.mergedSourceSpanIds].filter(Boolean).join(","),
+    sourceSpanNormalization: "merged_text_run"
+  });
+  return {
+    ...merged,
+    bbox: [...left.bbox ?? [], ...right.bbox ?? []],
+    pageEnd: right.pageEnd ?? left.pageEnd,
+    location: {
+      ...left.location,
+      endPage: right.location?.endPage ?? right.pageEnd ?? left.location?.endPage
+    },
+    __originalIndex: left.__originalIndex
+  };
+}
+function retextSpan(span, text, metadata) {
+  const textHash = sourceSpanTextHash(text);
+  return SourceSpanSchema.parse({
+    ...span,
+    id: `${span.id.split(":").slice(0, -1).join(":")}:${textHash.slice(0, 12)}`,
+    text,
+    hash: textHash,
+    textHash,
+    metadata: {
+      ...span.metadata ?? {},
+      ...metadata
+    }
+  });
+}
 function filterChunkableSourceSpans(spans) {
   const rowIds = new Set(
     spans.filter((span) => sourceUnit(span) === "table_row").map((span) => span.id)
@@ -4060,14 +4168,14 @@ function stringArray(value) {
 function sourceUnit4(span) {
   return span.sourceUnit ?? span.metadata?.sourceUnit;
 }
-function spanPage(span) {
+function spanPage2(span) {
   return span.pageStart ?? span.location?.page ?? span.location?.startPage;
 }
 function spanPageEnd(span) {
-  return span.pageEnd ?? span.location?.endPage ?? spanPage(span);
+  return span.pageEnd ?? span.location?.endPage ?? spanPage2(span);
 }
 function sourceSpansForPage(sourceSpans, page) {
-  return sourceSpans.filter((span) => spanPage(span) === page && sourceUnit4(span) === "page").map((span) => span.id);
+  return sourceSpans.filter((span) => spanPage2(span) === page && sourceUnit4(span) === "page").map((span) => span.id);
 }
 function nodePageOverlaps(node, record) {
   const recordStart = numberValue(record, "pageNumber", "pageStart", "resolvedFromPage");
@@ -4168,8 +4276,8 @@ function buildNodesFromSourceSpans(sourceSpans) {
     const unit = sourceUnit4(span);
     return unit === "section" || unit === "section_candidate" || unit === "page";
   });
-  return candidates.sort((left, right) => (spanPage(left) ?? 0) - (spanPage(right) ?? 0) || left.id.localeCompare(right.id)).map((span, index) => {
-    const title = span.sectionId ?? span.formNumber ?? (sourceUnit4(span) === "page" && spanPage(span) ? `Page ${spanPage(span)}` : `Source unit ${index + 1}`);
+  return candidates.sort((left, right) => (spanPage2(left) ?? 0) - (spanPage2(right) ?? 0) || left.id.localeCompare(right.id)).map((span, index) => {
+    const title = span.sectionId ?? span.formNumber ?? (sourceUnit4(span) === "page" && spanPage2(span) ? `Page ${spanPage2(span)}` : `Source unit ${index + 1}`);
     return {
       id: `source:${index}:${slugPart(span.id)}`,
       title,
@@ -4177,7 +4285,7 @@ function buildNodesFromSourceSpans(sourceSpans) {
       type: sourceUnit4(span),
       label: sourceUnit4(span),
       level: 1,
-      pageStart: spanPage(span),
+      pageStart: spanPage2(span),
       pageEnd: spanPageEnd(span),
       formNumber: span.formNumber,
       excerpt: span.text.slice(0, 500),
@@ -8956,6 +9064,29 @@ function cleanText(value, fallback) {
   const text = value?.replace(/\s+/g, " ").trim();
   return text || fallback;
 }
+function simplifyOrganizerTitle(value, fallback, kind) {
+  const title = cleanText(value, fallback);
+  if (/^declarations\b/i.test(title)) return "Declarations";
+  if (/^policy\s+form\b/i.test(title)) return "Policy Form";
+  if (/^definitions\b/i.test(title)) return "Definitions";
+  const endorsementNumber = title.match(/^endorsement\s+(?:no\.?|number|#)?\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1];
+  if (endorsementNumber) return `Endorsement No. ${endorsementNumber}`;
+  if (kind === "endorsement" && /^endorsements?\s+\d+\s*[–-]\s*\d+\b/i.test(title)) {
+    return title.replace(/[–—]/g, "-").replace(/\s*\(.*/, "").trim();
+  }
+  return title;
+}
+function endorsementReference(value) {
+  return value?.match(/\bendorsement\s+(?:no\.?|number|#)?\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1]?.toUpperCase();
+}
+function rejectsOrganizerGroup(group, children) {
+  if (group.kind !== "endorsement") return false;
+  if (/^endorsements?\s+\d+\s*[–-]\s*\d+\b/i.test(group.title)) return true;
+  const childNumbers = new Set(
+    children.map((child) => endorsementReference([child.title, child.description, child.textExcerpt].filter(Boolean).join(" "))).filter((value) => Boolean(value))
+  );
+  return childNumbers.size > 1;
+}
 function compactNode(node, maxText = 700) {
   return {
     id: node.id,
@@ -9043,8 +9174,10 @@ Scope:
 Rules:
 - Use only node IDs from the provided list.
 - Do not invent text, page numbers, source spans, limits, or policy facts.
-- You may relabel existing nodes and group adjacent top-level/page nodes from this batch when they are clearly one form, endorsement, declarations set, schedule, or clause family.
+- You may relabel existing nodes and group adjacent top-level/page nodes from this batch only when they are clearly one continuous form, one declarations set, one schedule, or one clause family.
+- Do not group separately numbered endorsements into one parent. Never create rollup titles such as "Endorsements 1-3 (...)".
 - Add concise, human-readable titles to generic text, table, row, and cell nodes when the text makes their role clear.
+- Keep organizer titles terse. Use the printed heading or a compact canonical title such as "Declarations", "Policy Form", "Definitions", or "Endorsement No. 3"; do not add parenthetical summaries.
 - Groups must list existing childNodeIds only.
 - Keep descriptions short and useful for search.
 - Prefer the document's own form titles, endorsement titles, schedules, declarations headings, and page order over keyword-only guessing.
@@ -9094,17 +9227,20 @@ function applyOrganization(sourceTree, organization) {
     return {
       ...node,
       kind: label.kind ?? node.kind,
-      title: cleanText(label.title, node.title),
+      title: simplifyOrganizerTitle(label.title, node.title, label.kind ?? node.kind),
       description: cleanText(label.description, node.description)
     };
   });
   for (const group of organization.groups) {
     const children = group.childNodeIds.map((id2) => byId.get(id2)).filter((node2) => Boolean(node2));
     if (children.length === 0) continue;
+    if (rejectsOrganizerGroup(group, children)) continue;
     const parentId = children[0].parentId;
     if (!children.every((child) => child.parentId === parentId)) continue;
     const documentId = children[0].documentId;
-    const id = groupNodeId(documentId, group);
+    const title = simplifyOrganizerTitle(group.title, group.title, group.kind);
+    const description = cleanText(group.description, title);
+    const id = groupNodeId(documentId, { ...group, title });
     if (byId.has(id)) continue;
     const sourceSpanIds = [...new Set(children.flatMap((child) => child.sourceSpanIds))];
     const pageStarts = children.map((child) => child.pageStart).filter((page) => typeof page === "number");
@@ -9115,8 +9251,8 @@ function applyOrganization(sourceTree, organization) {
       documentId,
       parentId,
       kind: group.kind,
-      title: group.title,
-      description: group.description ?? group.title,
+      title,
+      description,
       textExcerpt: children.map((child) => child.textExcerpt ?? child.description).filter(Boolean).join("\n\n").slice(0, 1600),
       sourceSpanIds,
       pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
@@ -9257,7 +9393,8 @@ function materializeDocument(params) {
   };
 }
 async function runSourceTreeExtraction(params) {
-  let sourceTree = buildDocumentSourceTree(params.sourceSpans, params.id);
+  const sourceSpans = normalizeSourceSpans(params.sourceSpans);
+  let sourceTree = buildDocumentSourceTree(sourceSpans, params.id);
   const warnings = [];
   let modelCalls = 0;
   let callsWithUsage = 0;
@@ -9293,7 +9430,7 @@ async function runSourceTreeExtraction(params) {
           maxTokens: budget.maxTokens,
           taskKind: "extraction_source_tree",
           budgetDiagnostics: budget,
-          providerOptions: { ...params.providerOptions, sourceSpans: params.sourceSpans }
+          providerOptions: { ...params.providerOptions, sourceSpans }
         },
         {
           fallback: { labels: [], groups: [] },
@@ -9314,12 +9451,12 @@ async function runSourceTreeExtraction(params) {
   }
   const deterministicProfile = buildDeterministicOperationalProfile({
     sourceTree,
-    sourceSpans: params.sourceSpans
+    sourceSpans
   });
   let operationalProfile = deterministicProfile;
   try {
     const validNodeIds = new Set(sourceTree.map((node) => node.id));
-    const validSpanIds = new Set(params.sourceSpans.map((span) => span.id));
+    const validSpanIds = new Set(sourceSpans.map((span) => span.id));
     const budget = params.resolveBudget("extraction_operational_profile", 8192);
     const startedAt = Date.now();
     const response = await safeGenerateObject(
@@ -9330,7 +9467,7 @@ async function runSourceTreeExtraction(params) {
         maxTokens: budget.maxTokens,
         taskKind: "extraction_operational_profile",
         budgetDiagnostics: budget,
-        providerOptions: { ...params.providerOptions, sourceSpans: params.sourceSpans, sourceTree }
+        providerOptions: { ...params.providerOptions, sourceSpans, sourceTree }
       },
       {
         fallback: deterministicProfile,
@@ -9359,8 +9496,8 @@ async function runSourceTreeExtraction(params) {
   });
   return {
     sourceTree,
-    sourceSpans: params.sourceSpans,
-    sourceChunks: chunkSourceSpans(params.sourceSpans),
+    sourceSpans,
+    sourceChunks: chunkSourceSpans(sourceSpans),
     operationalProfile,
     document,
     chunks: [],
@@ -14374,6 +14511,7 @@ export {
   normalizeDoclingDocument,
   normalizeDocumentSourceTreePaths,
   normalizeForMatch,
+  normalizeSourceSpans,
   orderSourceEvidence,
   overlayTextOnPdf,
   pLimit,