@claritylabs/cl-sdk 3.0.11 → 3.0.13

package/dist/index.mjs

@@ -2602,7 +2602,7 @@ function normalizeNodeKind(span) {
   if (unit === "table_cell") return "table_cell";
   if (unit === "key_value") return "schedule";
   if (unit === "section") return "section";
-  if (element === "title" || element === "section_candidate") {
+  if (element === "section_candidate") {
     const text = span.text.toLowerCase();
     if (/endorsement/.test(text)) return "endorsement";
     if (/schedule|declarations?/.test(text)) return "schedule";
@@ -2638,6 +2638,90 @@ function makeNode(params) {
     metadata: params.metadata
   };
 }
+function metadataString(metadata, key) {
+  const value = metadata?.[key];
+  return typeof value === "string" && value.trim() ? value.trim() : void 0;
+}
+function isTitleContentNode(node) {
+  if (node.kind !== "text") return false;
+  return metadataString(node.metadata, "elementType") === "title" || metadataString(node.metadata, "sourceUnit") === "title";
+}
+function nodePageEnd(node) {
+  return node.pageEnd ?? node.pageStart;
+}
+function groupPageContentByTitles(nodes) {
+  const byParent = /* @__PURE__ */ new Map();
+  for (const node of nodes) {
+    const children = byParent.get(node.parentId) ?? [];
+    children.push(node);
+    byParent.set(node.parentId, children);
+  }
+  for (const children of byParent.values()) {
+    children.sort((left, right) => left.order - right.order || left.id.localeCompare(right.id));
+  }
+  const byId = new Map(nodes.map((node) => [node.id, node]));
+  for (const pageNode of nodes.filter((node) => node.kind === "page")) {
+    const children = (byParent.get(pageNode.id) ?? []).filter((child) => child.kind !== "table_row" && child.kind !== "table_cell");
+    let activeTitle;
+    let activeContent = [];
+    const flush = () => {
+      if (!activeTitle || activeContent.length === 0) {
+        activeTitle = void 0;
+        activeContent = [];
+        return;
+      }
+      const contentNodes = activeContent.map((node) => byId.get(node.id) ?? node).filter((node) => node.parentId === pageNode.id);
+      if (contentNodes.length === 0) {
+        activeTitle = void 0;
+        activeContent = [];
+        return;
+      }
+      const evidenceNodes = [activeTitle, ...contentNodes];
+      const title = truncate(activeTitle.textExcerpt ?? activeTitle.title, 120);
+      const pageStarts = evidenceNodes.map((node) => node.pageStart).filter((page) => typeof page === "number");
+      const pageEnds = evidenceNodes.map(nodePageEnd).filter((page) => typeof page === "number");
+      const sourceSpanIds = [...new Set(evidenceNodes.flatMap((node) => node.sourceSpanIds))];
+      const bbox = evidenceNodes.flatMap((node) => node.bbox ?? []).slice(0, 12);
+      byId.set(activeTitle.id, {
+        ...activeTitle,
+        kind: "section",
+        title,
+        description: nodeTextDescription({
+          kind: "section",
+          title,
+          text: evidenceNodes.map((node) => node.textExcerpt).filter(Boolean).join("\n\n"),
+          page: activeTitle.pageStart
+        }),
+        textExcerpt: evidenceNodes.map((node) => node.textExcerpt).filter(Boolean).join("\n\n").slice(0, 1600),
+        sourceSpanIds,
+        pageStart: pageStarts.length ? Math.min(...pageStarts) : activeTitle.pageStart,
+        pageEnd: pageEnds.length ? Math.max(...pageEnds) : activeTitle.pageEnd,
+        bbox,
+        metadata: {
+          ...activeTitle.metadata,
+          sourceTreeVersion: "v3",
+          organizer: "title_block"
+        }
+      });
+      for (const node of contentNodes) {
+        byId.set(node.id, { ...node, parentId: activeTitle.id });
+      }
+      activeTitle = void 0;
+      activeContent = [];
+    };
+    for (const child of children) {
+      if (isTitleContentNode(child)) {
+        flush();
+        activeTitle = child;
+        activeContent = [];
+        continue;
+      }
+      if (activeTitle) activeContent.push(child);
+    }
+    flush();
+  }
+  return nodes.map((node) => byId.get(node.id) ?? node);
+}
 function sortSpans(left, right) {
   const leftPage = pageStart(left) ?? 0;
   const rightPage = pageStart(right) ?? 0;
@@ -2813,7 +2897,7 @@ function buildDocumentSourceTree(sourceSpans, documentId) {
     }
     addStandaloneSpanNode(span, pageParentId);
   }
-  return normalizeDocumentSourceTreePaths([...nodes.values()]);
+  return normalizeDocumentSourceTreePaths(groupPageContentByTitles([...nodes.values()]));
 }
 
 // src/source/operational-profile.ts
@@ -9009,9 +9093,9 @@ var ORGANIZABLE_KINDS = [
   "clause"
 ];
 var ORGANIZATION_TOP_LEVEL_BATCH_SIZE = 80;
-var ORGANIZATION_CHILD_CONTEXT_LIMIT = 4;
 var ORGANIZER_MAX_SOURCE_SPANS = 400;
 var ORGANIZER_MAX_TOP_LEVEL_NODES = 18;
+var OUTLINE_CLEANUP_MAX_TOP_LEVEL_NODES = 80;
 var SourceTreeOrganizationSchema = z41.object({
   labels: z41.array(z41.object({
     nodeId: z41.string(),
@@ -9128,6 +9212,22 @@ function endorsementDescription(title, node) {
     title
   );
 }
+function nodePageEnd2(node) {
+  return node.pageEnd ?? node.pageStart;
+}
+function pageRangeForNodes(nodes) {
+  const pageStarts = nodes.map((node) => node.pageStart).filter((page) => typeof page === "number");
+  const pageEnds = nodes.map(nodePageEnd2).filter((page) => typeof page === "number");
+  if (pageStarts.length === 0 || pageEnds.length === 0) return void 0;
+  const start = Math.min(...pageStarts);
+  const end = Math.max(...pageEnds);
+  return start === end ? `page ${start}` : `pages ${start}-${end}`;
+}
+function descriptionWithPages(description, nodes) {
+  const range = pageRangeForNodes(nodes);
+  if (!range || new RegExp(`\\b${range.replace("-", "\\-")}\\b`, "i").test(description)) return description;
+  return `${description}; ${range}`;
+}
 function semanticGroupNodeId(documentId, kind, title, childNodeIds) {
   return [
     documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
@@ -9138,7 +9238,12 @@ function semanticGroupNodeId(documentId, kind, title, childNodeIds) {
   ].join(":");
 }
 function looksLikeDeclarationsStart(node) {
-  return /(^|\b)declarations?\b/i.test(sourceNodeText(node)) && !/\bforms?\s+and\s+endorsements?\b/i.test(sourceNodeText(node));
+  const title = cleanText(node.title, "");
+  const text = sourceNodeText(node);
+  if (/\b(important notice|privacy notice|ofac advisory|terrorism risk insurance act|how to report a claim)\b/i.test(text)) {
+    return false;
+  }
+  return /^declarations?$/i.test(title) || /\bdeclarations?\s+(page|schedule|section)\b/i.test(text) || /^declarations?\b/i.test(cleanText(node.textExcerpt, ""));
 }
 function looksLikeDeclarationsContinuation(node) {
   const text = sourceNodeText(node);
@@ -9173,7 +9278,7 @@ function groupAdjacentChildren(params) {
     parentId,
     kind: params.kind,
     title: params.title,
-    description: params.description,
+    description: descriptionWithPages(params.description, children),
     textExcerpt: children.map((child) => child.textExcerpt ?? child.description).filter(Boolean).join("\n\n").slice(0, 1600),
     sourceSpanIds,
     pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
@@ -9226,6 +9331,22 @@ function applySemanticPageGrouping(sourceTree) {
   const children = (nodesByParent(relabeled).get(rootId) ?? []).filter((node) => node.kind !== "document").sort((left, right) => left.order - right.order);
   let nextTree = relabeled;
   const declarationsStartIndex = children.findIndex(looksLikeDeclarationsStart);
+  const firstCoreIndex = children.findIndex(
+    (child) => looksLikeDeclarationsStart(child) || looksLikePolicyFormStart(child) || looksLikeEndorsementStart(child)
+  );
+  const frontMatterBoundary = declarationsStartIndex >= 0 ? declarationsStartIndex : firstCoreIndex;
+  if (frontMatterBoundary > 0) {
+    const frontMatterIds = children.slice(0, frontMatterBoundary).map((child) => child.id);
+    nextTree = groupAdjacentChildren({
+      sourceTree: nextTree,
+      children,
+      childIds: frontMatterIds,
+      kind: "page_group",
+      title: "Notices and Jacket",
+      description: "Policy jacket, notices, and administrative pages grouped by source order",
+      organizer: "semantic_front_matter_grouping"
+    });
+  }
   if (declarationsStartIndex >= 0) {
     const declarationIds = [];
     for (let index = declarationsStartIndex; index < children.length; index += 1) {
@@ -9394,17 +9515,19 @@ function normalizeContainerEvidenceFromChildren(sourceTree) {
   return sourceTree.map((node) => byId.get(node.id) ?? node);
 }
 function normalizeSemanticHierarchy(sourceTree) {
-  return normalizeDocumentSourceTreePaths(
-    normalizeContainerEvidenceFromChildren(
-      nestEndorsementContinuationPages(
-        normalizePolicyFormStructure(
-          normalizeDocumentSourceTreePaths(sourceTree)
-        )
-      )
+  const normalized = normalizeDocumentSourceTreePaths(
+    normalizePolicyFormStructure(
+      normalizeDocumentSourceTreePaths(sourceTree)
     )
   );
+  const nested = normalizeDocumentSourceTreePaths(nestEndorsementContinuationPages(normalized));
+  const withEvidence = normalizeContainerEvidenceFromChildren(nested);
+  return normalizeDocumentSourceTreePaths(
+    normalizeContainerEvidenceFromChildren(withEvidence)
+  );
 }
 function applyEndorsementGrouping(sourceTree) {
+  const rootId = sourceTreeRootId(sourceTree);
   const relabeledTree = sourceTree.map((node) => {
     if (node.kind === "document" || isEndorsementGroup(node)) return node;
     const title = endorsementStartTitle(node);
@@ -9442,7 +9565,7 @@ function applyEndorsementGrouping(sourceTree) {
       ...node,
       kind: "page_group",
       title: "Endorsements",
-      description: cleanText(node.description, "Endorsement forms grouped by source order"),
+      description: descriptionWithPages(cleanText(node.description, "Endorsement forms grouped by source order"), byParent.get(node.id) ?? [node]),
       metadata: {
         ...node.metadata,
         sourceTreeVersion: "v3",
@@ -9469,9 +9592,10 @@ function applyEndorsementGrouping(sourceTree) {
     };
   });
   for (const [parentId, children] of byParent) {
+    if (parentId !== rootId) continue;
     if (endorsementGroupIds.has(parentId ?? "")) continue;
     const endorsementChildren = children.filter((child) => child.kind === "endorsement" && !isEndorsementGroup(child));
-    if (endorsementChildren.length < 2) continue;
+    if (endorsementChildren.length < 1) continue;
     const endorsementGroupChildren = [];
     let hasSeenEndorsementStart = false;
     for (const child of children) {
@@ -9484,6 +9608,7 @@ function applyEndorsementGrouping(sourceTree) {
         endorsementGroupChildren.push(child);
       }
     }
+    if (endorsementGroupChildren.length < 1) continue;
     const documentId = endorsementChildren[0].documentId;
     const pageStarts = endorsementGroupChildren.map((child) => child.pageStart).filter((page) => typeof page === "number");
     const pageEnds = endorsementGroupChildren.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
@@ -9496,7 +9621,7 @@ function applyEndorsementGrouping(sourceTree) {
       parentId,
       kind: "page_group",
       title: "Endorsements",
-      description: "Endorsement forms grouped by source order",
+      description: descriptionWithPages("Endorsement forms grouped by source order", endorsementGroupChildren),
       textExcerpt: void 0,
       sourceSpanIds: [],
       pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
@@ -9507,11 +9632,13 @@ function applyEndorsementGrouping(sourceTree) {
       metadata: { sourceTreeVersion: "v3", organizer: "endorsement_grouping" }
     };
     const childSpanIds = [...new Set(endorsementGroupChildren.flatMap((child) => child.sourceSpanIds))];
+    const childPageStart = pageStarts.length ? Math.min(...pageStarts) : void 0;
+    const childPageEnd = pageEnds.length ? Math.max(...pageEnds) : void 0;
     const normalizedGroup = {
       ...groupNode,
       sourceSpanIds: groupNode.sourceSpanIds.length ? groupNode.sourceSpanIds : childSpanIds,
-      pageStart: groupNode.pageStart ?? (pageStarts.length ? Math.min(...pageStarts) : void 0),
-      pageEnd: groupNode.pageEnd ?? (pageEnds.length ? Math.max(...pageEnds) : void 0),
+      pageStart: childPageStart === void 0 ? groupNode.pageStart : groupNode.pageStart === void 0 ? childPageStart : Math.min(groupNode.pageStart, childPageStart),
+      pageEnd: childPageEnd === void 0 ? groupNode.pageEnd : groupNode.pageEnd === void 0 ? childPageEnd : Math.max(groupNode.pageEnd, childPageEnd),
       order
     };
     groupsByParent.set(parentId, normalizedGroup);
@@ -9587,10 +9714,6 @@ function organizationBatches(sourceTree) {
     const candidates = /* @__PURE__ */ new Map();
     for (const node of topLevelBatch) {
       candidates.set(node.id, node);
-      const childContext = (byParent.get(node.id) ?? []).filter((child) => child.kind !== "text" && child.kind !== "table_cell").slice(0, ORGANIZATION_CHILD_CONTEXT_LIMIT);
-      for (const child of childContext) {
-        candidates.set(child.id, child);
-      }
     }
     batches.push({
       label: `top-level nodes ${index + 1}-${index + topLevelBatch.length} of ${topLevelNodes.length}`,
@@ -9641,6 +9764,42 @@ Rules:
 Source nodes:
 ${JSON.stringify(nodes, null, 2)}
 
+Return JSON with labels and groups only.`;
+}
+function shouldRunOutlineCleanup(sourceTree) {
+  const topLevel = rootChildren(sourceTree);
+  if (topLevel.length === 0 || topLevel.length > OUTLINE_CLEANUP_MAX_TOP_LEVEL_NODES) return false;
+  const genericPages = topLevel.filter((node) => node.kind === "page" && /^Page\s+\d+$/i.test(node.title));
+  const hasDeclarations = topLevel.some((node) => node.title === "Declarations");
+  const hasPolicyForm = topLevel.some((node) => node.title === "Policy Form");
+  const hasEndorsements = topLevel.some((node) => node.title === "Endorsements");
+  if (hasDeclarations && hasPolicyForm && hasEndorsements) return false;
+  return genericPages.length > 0 || topLevel.length > 6 || !hasDeclarations || !hasPolicyForm || !hasEndorsements;
+}
+function buildOutlineCleanupPrompt(sourceTree) {
+  const topLevel = rootChildren(sourceTree).slice(0, OUTLINE_CLEANUP_MAX_TOP_LEVEL_NODES);
+  const nodes = topLevel.map((node) => compactNode(node, 900));
+  return `You clean a top-level source outline for an insurance policy.
+
+Expected product-facing order:
+1. Optional front matter: policy jacket, important notices, privacy notices, OFAC notices, TRIA/terrorism notices, marketing/admin pages, signatures, countersignatures, or other pages that are not the declarations, policy wording, or endorsements.
+2. Declarations: declarations page(s), schedules, named insured/policy period/premium rows, coverage limit schedules, forms-and-endorsements schedules.
+3. Policy Form: the main policy wording, insuring agreements, definitions, exclusions, conditions, claim provisions, and general policy terms.
+4. Endorsements: one generic "Endorsements" page_group containing each separately numbered endorsement as its own child.
+
+Rules:
+- Use only node IDs from this top-level list: ${JSON.stringify(topLevel.map((node) => node.id))}
+- Group only adjacent top-level nodes.
+- Do not invent text, pages, source spans, limits, or policy facts.
+- Do not merge individually numbered endorsements into one endorsement node; use the generic "Endorsements" parent for the series.
+- Use canonical terse titles: "Notices and Jacket", "Declarations", "Policy Form", "Endorsements", or the printed endorsement number.
+- Keep page_group descriptions short and include the page range when pages are known, for example "Declarations pages 6-8".
+- If a page is an OFAC, privacy, terrorism/TRIA, claim-reporting notice, signature page, or jacket, do not label it as declarations or policy form.
+- If the existing deterministic outline is already correct, return empty labels and groups.
+
+Top-level source nodes:
+${JSON.stringify(nodes, null, 2)}
+
 Return JSON with labels and groups only.`;
 }
 function buildOperationalProfilePrompt(sourceTree, fallback) {
@@ -9695,7 +9854,7 @@ function applyOrganization(sourceTree, organization) {
     if (!children.every((child) => child.parentId === parentId)) continue;
     const documentId = children[0].documentId;
     const title = simplifyOrganizerTitle(group.title, group.title, group.kind);
-    const description = cleanText(group.description, title);
+    const description = descriptionWithPages(cleanText(group.description, title), children);
     const id = groupNodeId(documentId, { ...group, title });
     if (byId.has(id)) continue;
     const sourceSpanIds = [...new Set(children.flatMap((child) => child.sourceSpanIds))];
@@ -9886,8 +10045,7 @@ async function runSourceTreeExtraction(params) {
             schema: SourceTreeOrganizationSchema,
             maxTokens: budget.maxTokens,
             taskKind: "extraction_source_tree",
-            budgetDiagnostics: budget,
-            providerOptions: params.providerOptions
+            budgetDiagnostics: budget
           },
           {
             fallback: { labels: [], groups: [] },
@@ -9909,6 +10067,36 @@ async function runSourceTreeExtraction(params) {
   } else {
     await params.log?.("Deterministic source tree ready; skipped model organizer");
   }
+  if (shouldRunOutlineCleanup(sourceTree)) {
+    try {
+      const budget = params.resolveBudget("extraction_source_tree", 1600);
+      const maxTokens = Math.min(budget.maxTokens, 1600);
+      const startedAt = Date.now();
+      const response = await safeGenerateObject(
+        params.generateObject,
+        {
+          prompt: buildOutlineCleanupPrompt(sourceTree),
+          schema: SourceTreeOrganizationSchema,
+          maxTokens,
+          taskKind: "extraction_source_tree",
+          budgetDiagnostics: { ...budget, maxTokens }
+        },
+        {
+          fallback: { labels: [], groups: [] },
+          log: params.log
+        }
+      );
+      localTrack(response.usage, {
+        taskKind: "extraction_source_tree",
+        label: "source_tree_outline_cleanup",
+        maxTokens,
+        durationMs: Date.now() - startedAt
+      });
+      sourceTree = applyOrganization(sourceTree, response.object);
+    } catch (error) {
+      warnings.push(`Source-tree outline cleanup failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
+    }
+  }
   const deterministicProfile = buildDeterministicOperationalProfile({
     sourceTree,
     sourceSpans