@claritylabs/cl-sdk 3.0.9 → 3.0.11

package/dist/index.js

@@ -3017,14 +3017,24 @@ function normalizeDocumentSourceTreePaths(nodes) {
     group.sort((left, right) => left.order - right.order || left.id.localeCompare(right.id));
   }
   const result = [];
-  const visit = (node, path) => {
-    const next = { ...node, path };
+  const visited = /* @__PURE__ */ new Set();
+  const visit = (node, path, ancestors, parentId) => {
+    if (visited.has(node.id) || ancestors.has(node.id)) return;
+    visited.add(node.id);
+    const next = { ...node, parentId, path };
     result.push(next);
     const children = byParent.get(node.id) ?? [];
-    children.forEach((child, index) => visit(child, `${path}.${index + 1}`));
+    const nextAncestors = new Set(ancestors);
+    nextAncestors.add(node.id);
+    children.forEach((child, index) => visit(child, `${path}.${index + 1}`, nextAncestors, node.id));
   };
   const roots = byParent.get(void 0) ?? [];
-  roots.forEach((root, index) => visit(root, String(index + 1)));
+  roots.forEach((root, index) => visit(root, String(index + 1), /* @__PURE__ */ new Set(), void 0));
+  for (const node of nodes) {
+    if (!visited.has(node.id)) {
+      visit(node, String(result.length + 1), /* @__PURE__ */ new Set(), void 0);
+    }
+  }
   return result;
 }
 function buildDocumentSourceTree(sourceSpans, documentId) {
@@ -9347,6 +9357,8 @@ var ORGANIZABLE_KINDS = [
 ];
 var ORGANIZATION_TOP_LEVEL_BATCH_SIZE = 80;
 var ORGANIZATION_CHILD_CONTEXT_LIMIT = 4;
+var ORGANIZER_MAX_SOURCE_SPANS = 400;
+var ORGANIZER_MAX_TOP_LEVEL_NODES = 18;
 var SourceTreeOrganizationSchema = import_zod41.z.object({
   labels: import_zod41.z.array(import_zod41.z.object({
     nodeId: import_zod41.z.string(),
@@ -9457,6 +9469,12 @@ function looksLikeEndorsementContinuation(node) {
 function endorsementStartTitle(node) {
   return looksLikeEndorsementStart(node) ? endorsementTitle(sourceNodeText(node)) : void 0;
 }
+function endorsementDescription(title, node) {
+  return cleanText(
+    [title, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0].filter(Boolean).join(" | "),
+    title
+  );
+}
 function semanticGroupNodeId(documentId, kind, title, childNodeIds) {
   return [
     documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
@@ -9467,7 +9485,7 @@ function semanticGroupNodeId(documentId, kind, title, childNodeIds) {
   ].join(":");
 }
 function looksLikeDeclarationsStart(node) {
-  return /\bdeclarations?\s+(page|schedule)\b/i.test(sourceNodeText(node));
+  return /(^|\b)declarations?\b/i.test(sourceNodeText(node)) && !/\bforms?\s+and\s+endorsements?\b/i.test(sourceNodeText(node));
 }
 function looksLikeDeclarationsContinuation(node) {
   const text = sourceNodeText(node);
@@ -9475,7 +9493,8 @@ function looksLikeDeclarationsContinuation(node) {
 }
 function looksLikePolicyFormStart(node) {
   const text = sourceNodeText(node);
-  return /\bpolicy form\b/i.test(node.title) || /\btechnology errors?\s*&?\s*omissions\b/i.test(text) && /\bplease read this entire policy carefully\b/i.test(text) || /\bform\s+[A-Z]{2,}-[A-Z0-9-]+\s+\d{2}\s+\d{2}\b/i.test(text);
+  const excerpt = cleanText(node.textExcerpt, "");
+  return /\bpolicy form\b/i.test(node.title) || /^policy\s+form\b/i.test(excerpt) || /\btechnology errors?\s*&?\s*omissions\b/i.test(text) && /\bplease read this entire policy carefully\b/i.test(text) || /\bform\s+[A-Z]{2,}-[A-Z0-9-]+\s+\d{2}\s+\d{2}\b/i.test(text);
 }
 function looksLikePolicyFormContinuation(node) {
   const text = sourceNodeText(node);
@@ -9528,7 +9547,7 @@ function applySemanticPageGrouping(sourceTree) {
         ...node,
         kind: "endorsement",
         title: endorsement,
-        description: cleanText([endorsement, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "), endorsement),
+        description: endorsementDescription(endorsement, node),
         metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
       };
     }
@@ -9752,10 +9771,7 @@ function applyEndorsementGrouping(sourceTree) {
       ...node,
       kind: "endorsement",
       title,
-      description: cleanText(
-        [title, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "),
-        title
-      ),
+      description: endorsementDescription(title, node),
       metadata: {
         ...node.metadata,
         organizerRepair: "normalize_endorsement_grouping"
@@ -9792,10 +9808,7 @@ function applyEndorsementGrouping(sourceTree) {
       ...node,
       kind: "endorsement",
       title,
-      description: cleanText(
-        [title, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "),
-        title
-      ),
+      description: endorsementDescription(title, node),
       metadata: {
         ...node.metadata,
         organizerRepair: "normalize_endorsement_grouping"
@@ -9885,6 +9898,24 @@ function nodesByParent(sourceTree) {
 function sourceTreeRootId(sourceTree) {
   return sourceTree.find((node) => node.kind === "document")?.id;
 }
+function rootChildren(sourceTree) {
+  const byParent = nodesByParent(sourceTree);
+  const rootId = sourceTreeRootId(sourceTree);
+  return (byParent.get(rootId) ?? []).filter((node) => node.kind !== "document");
+}
+function hasDeterministicSemanticOutline(sourceTree) {
+  const children = rootChildren(sourceTree);
+  const semanticCount = children.filter(
+    (node) => node.kind === "page_group" || node.kind === "form" || node.kind === "endorsement" || node.kind === "section" || node.kind === "schedule"
+  ).length;
+  return semanticCount >= 2 || children.some((node) => node.title === "Declarations") || children.some((node) => node.title === "Policy Form") || children.some((node) => node.title === "Endorsements");
+}
+function shouldRunSourceTreeOrganizer(sourceTree, sourceSpans) {
+  const topLevelCount = rootChildren(sourceTree).length;
+  if (sourceSpans.length > ORGANIZER_MAX_SOURCE_SPANS) return false;
+  if (topLevelCount > ORGANIZER_MAX_TOP_LEVEL_NODES) return false;
+  return !hasDeterministicSemanticOutline(sourceTree);
+}
 function organizationBatches(sourceTree) {
   const byParent = nodesByParent(sourceTree);
   const rootId = sourceTreeRootId(sourceTree);
@@ -10166,7 +10197,7 @@ function materializeDocument(params) {
 }
 async function runSourceTreeExtraction(params) {
   const sourceSpans = normalizeSourceSpans(params.sourceSpans);
-  let sourceTree = buildDocumentSourceTree(sourceSpans, params.id);
+  let sourceTree = applySemanticPageGrouping(buildDocumentSourceTree(sourceSpans, params.id));
   const warnings = [];
   let modelCalls = 0;
   let callsWithUsage = 0;
@@ -10188,40 +10219,43 @@ async function runSourceTreeExtraction(params) {
     }
     params.trackUsage(usage, report);
   };
-  try {
-    const organizations = [];
-    const batches = organizationBatches(sourceTree);
-    for (const [batchIndex, batch] of batches.entries()) {
-      const budget = params.resolveBudget("extraction_source_tree", 4096);
-      const startedAt = Date.now();
-      const response = await safeGenerateObject(
-        params.generateObject,
-        {
-          prompt: buildOrganizationPrompt(batch),
-          schema: SourceTreeOrganizationSchema,
-          maxTokens: budget.maxTokens,
+  if (shouldRunSourceTreeOrganizer(sourceTree, sourceSpans)) {
+    try {
+      const organizations = [];
+      const batches = organizationBatches(sourceTree);
+      for (const [batchIndex, batch] of batches.entries()) {
+        const budget = params.resolveBudget("extraction_source_tree", 4096);
+        const startedAt = Date.now();
+        const response = await safeGenerateObject(
+          params.generateObject,
+          {
+            prompt: buildOrganizationPrompt(batch),
+            schema: SourceTreeOrganizationSchema,
+            maxTokens: budget.maxTokens,
+            taskKind: "extraction_source_tree",
+            budgetDiagnostics: budget,
+            providerOptions: params.providerOptions
+          },
+          {
+            fallback: { labels: [], groups: [] },
+            log: params.log
+          }
+        );
+        localTrack(response.usage, {
           taskKind: "extraction_source_tree",
-          budgetDiagnostics: budget,
-          providerOptions: { ...params.providerOptions, sourceSpans }
-        },
-        {
-          fallback: { labels: [], groups: [] },
-          log: params.log
-        }
-      );
-      localTrack(response.usage, {
-        taskKind: "extraction_source_tree",
-        label: batches.length > 1 ? `source_tree_organizer_${batchIndex + 1}` : "source_tree_organizer",
-        maxTokens: budget.maxTokens,
-        durationMs: Date.now() - startedAt
-      });
-      organizations.push(response.object);
+          label: batches.length > 1 ? `source_tree_organizer_${batchIndex + 1}` : "source_tree_organizer",
+          maxTokens: budget.maxTokens,
+          durationMs: Date.now() - startedAt
+        });
+        organizations.push(response.object);
+      }
+      sourceTree = applyOrganization(sourceTree, mergeOrganizationResults(organizations));
+    } catch (error) {
+      warnings.push(`Source-tree organizer failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
     }
-    sourceTree = applyOrganization(sourceTree, mergeOrganizationResults(organizations));
-  } catch (error) {
-    warnings.push(`Source-tree organizer failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
+  } else {
+    await params.log?.("Deterministic source tree ready; skipped model organizer");
   }
-  sourceTree = applySemanticPageGrouping(sourceTree);
   const deterministicProfile = buildDeterministicOperationalProfile({
     sourceTree,
     sourceSpans
@@ -10240,7 +10274,7 @@ async function runSourceTreeExtraction(params) {
         maxTokens: budget.maxTokens,
         taskKind: "extraction_operational_profile",
         budgetDiagnostics: budget,
-        providerOptions: { ...params.providerOptions, sourceSpans, sourceTree }
+        providerOptions: params.providerOptions
       },
       {
         fallback: deterministicProfile,