@claritylabs/cl-sdk 3.0.1 → 3.0.3

package/dist/index.js

@@ -9236,6 +9236,8 @@ var ORGANIZABLE_KINDS = [
   "schedule",
   "clause"
 ];
+var ORGANIZATION_TOP_LEVEL_BATCH_SIZE = 80;
+var ORGANIZATION_CHILD_CONTEXT_LIMIT = 4;
 var SourceTreeOrganizationSchema = import_zod41.z.object({
   labels: import_zod41.z.array(import_zod41.z.object({
     nodeId: import_zod41.z.string(),
@@ -9255,20 +9257,20 @@ var SourceTreeOrganizationSchema = import_zod41.z.object({
     ]).optional(),
     title: import_zod41.z.string().optional(),
     description: import_zod41.z.string().optional()
-  })).default([]),
+  })),
   groups: import_zod41.z.array(import_zod41.z.object({
     kind: import_zod41.z.enum(ORGANIZABLE_KINDS),
     title: import_zod41.z.string(),
     description: import_zod41.z.string().optional(),
     childNodeIds: import_zod41.z.array(import_zod41.z.string()).min(1)
-  })).default([])
+  }))
 });
 var SourceBackedValueForPromptSchema = import_zod41.z.object({
   value: import_zod41.z.string(),
   normalizedValue: import_zod41.z.string().optional(),
   confidence: import_zod41.z.enum(["low", "medium", "high"]).optional(),
-  sourceNodeIds: import_zod41.z.array(import_zod41.z.string()).default([]),
-  sourceSpanIds: import_zod41.z.array(import_zod41.z.string()).default([])
+  sourceNodeIds: import_zod41.z.array(import_zod41.z.string()),
+  sourceSpanIds: import_zod41.z.array(import_zod41.z.string())
 });
 var OperationalProfilePromptSchema = import_zod41.z.object({
   documentType: import_zod41.z.enum(["policy", "quote"]).optional(),
@@ -9290,8 +9292,8 @@ var OperationalProfilePromptSchema = import_zod41.z.object({
     premium: import_zod41.z.string().optional(),
     formNumber: import_zod41.z.string().optional(),
     sectionRef: import_zod41.z.string().optional(),
-    sourceNodeIds: import_zod41.z.array(import_zod41.z.string()).default([]),
-    sourceSpanIds: import_zod41.z.array(import_zod41.z.string()).default([])
+    sourceNodeIds: import_zod41.z.array(import_zod41.z.string()),
+    sourceSpanIds: import_zod41.z.array(import_zod41.z.string())
   })).optional(),
   sourceNodeIds: import_zod41.z.array(import_zod41.z.string()).optional(),
   sourceSpanIds: import_zod41.z.array(import_zod41.z.string()).optional()
@@ -9300,7 +9302,7 @@ function cleanText(value, fallback) {
   const text = value?.replace(/\s+/g, " ").trim();
   return text || fallback;
 }
-function compactNode(node) {
+function compactNode(node, maxText = 700) {
   return {
     id: node.id,
     kind: node.kind,
@@ -9309,19 +9311,89 @@ function compactNode(node) {
     pageStart: node.pageStart,
     pageEnd: node.pageEnd,
     sourceSpanIds: node.sourceSpanIds.slice(0, 8),
-    text: (node.textExcerpt ?? node.description).slice(0, 700)
+    text: (node.textExcerpt ?? node.description).slice(0, maxText)
+  };
+}
+function nodesByParent(sourceTree) {
+  const byParent = /* @__PURE__ */ new Map();
+  for (const node of sourceTree) {
+    const children = byParent.get(node.parentId) ?? [];
+    children.push(node);
+    byParent.set(node.parentId, children);
+  }
+  for (const children of byParent.values()) {
+    children.sort((left, right) => left.order - right.order || left.id.localeCompare(right.id));
+  }
+  return byParent;
+}
+function sourceTreeRootId(sourceTree) {
+  return sourceTree.find((node) => node.kind === "document")?.id;
+}
+function organizationBatches(sourceTree) {
+  const byParent = nodesByParent(sourceTree);
+  const rootId = sourceTreeRootId(sourceTree);
+  const topLevelNodes = (byParent.get(rootId) ?? []).filter((node) => node.kind !== "document");
+  if (topLevelNodes.length === 0) {
+    const nodes = sourceTree.filter((node) => node.kind !== "document").slice(0, 240);
+    return [{
+      label: "fallback node prefix because no document root children were found",
+      topLevelNodeIds: nodes.map((node) => node.id),
+      nodes
+    }];
+  }
+  const batches = [];
+  for (let index = 0; index < topLevelNodes.length; index += ORGANIZATION_TOP_LEVEL_BATCH_SIZE) {
+    const topLevelBatch = topLevelNodes.slice(index, index + ORGANIZATION_TOP_LEVEL_BATCH_SIZE);
+    const candidates = /* @__PURE__ */ new Map();
+    for (const node of topLevelBatch) {
+      candidates.set(node.id, node);
+      const childContext = (byParent.get(node.id) ?? []).filter((child) => child.kind !== "text" && child.kind !== "table_cell").slice(0, ORGANIZATION_CHILD_CONTEXT_LIMIT);
+      for (const child of childContext) {
+        candidates.set(child.id, child);
+      }
+    }
+    batches.push({
+      label: `top-level nodes ${index + 1}-${index + topLevelBatch.length} of ${topLevelNodes.length}`,
+      topLevelNodeIds: topLevelBatch.map((node) => node.id),
+      nodes: [...candidates.values()]
+    });
+  }
+  return batches;
+}
+function mergeOrganizationResults(results) {
+  const labels = /* @__PURE__ */ new Map();
+  const groups = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    for (const label of result.labels) {
+      labels.set(label.nodeId, { ...labels.get(label.nodeId), ...label });
+    }
+    for (const group of result.groups) {
+      const key = `${group.kind}:${group.childNodeIds.join("|")}`;
+      groups.set(key, group);
+    }
+  }
+  return {
+    labels: [...labels.values()],
+    groups: [...groups.values()]
   };
 }
-function buildOrganizationPrompt(sourceTree) {
-  const nodes = sourceTree.filter((node) => node.kind !== "table_cell").slice(0, 180).map(compactNode);
+function buildOrganizationPrompt(batch) {
+  const nodes = batch.nodes.map((node) => compactNode(node, node.kind === "page" ? 900 : 320));
   return `You organize an insurance document source tree.
 
+Scope:
+- ${batch.label}
+- The provided list is a bounded extraction-time batch. It is not necessarily the whole document.
+- Top-level page/form candidates in this batch: ${JSON.stringify(batch.topLevelNodeIds)}
+
 Rules:
 - Use only node IDs from the provided list.
 - Do not invent text, page numbers, source spans, limits, or policy facts.
-- You may relabel existing nodes and group adjacent top-level/page nodes when they are clearly one form, endorsement, declarations set, schedule, or clause family.
+- You may relabel existing nodes and group adjacent top-level/page nodes from this batch when they are clearly one form, endorsement, declarations set, schedule, or clause family.
+- Add concise, human-readable titles to generic text, table, row, and cell nodes when the text makes their role clear.
 - Groups must list existing childNodeIds only.
 - Keep descriptions short and useful for search.
+- Prefer the document's own form titles, endorsement titles, schedules, declarations headings, and page order over keyword-only guessing.
 
 Source nodes:
 ${JSON.stringify(nodes, null, 2)}
@@ -9361,8 +9433,9 @@ function groupNodeId(documentId, group) {
 }
 function applyOrganization(sourceTree, organization) {
   const byId = new Map(sourceTree.map((node) => [node.id, node]));
+  const labels = new Map(organization.labels.map((label) => [label.nodeId, label]));
   let nextTree = sourceTree.map((node) => {
-    const label = organization.labels.find((item) => item.nodeId === node.id);
+    const label = labels.get(node.id);
     if (!label) return node;
     return {
       ...node,
@@ -9371,7 +9444,7 @@ function applyOrganization(sourceTree, organization) {
       description: cleanText(label.description, node.description)
     };
   });
-  for (const group of organization.groups.slice(0, 40)) {
+  for (const group of organization.groups) {
     const children = group.childNodeIds.map((id2) => byId.get(id2)).filter((node2) => Boolean(node2));
     if (children.length === 0) continue;
     const parentId = children[0].parentId;
@@ -9429,6 +9502,7 @@ function sourceTreeToOutline(sourceTree) {
     sourceSpanIds: node.sourceSpanIds,
     sourceTextHash: node.sourceSpanIds.join(":") || void 0,
     interpretationLabels: [node.kind],
+    metadata: node.metadata,
     children: (byParent.get(node.id) ?? []).map(visit)
   });
   return (byParent.get(root?.id) ?? []).map(visit);
@@ -9552,30 +9626,35 @@ async function runSourceTreeExtraction(params) {
     params.trackUsage(usage, report);
   };
   try {
-    const budget = params.resolveBudget("extraction_source_tree", 4096);
-    const startedAt = Date.now();
-    const response = await safeGenerateObject(
-      params.generateObject,
-      {
-        prompt: buildOrganizationPrompt(sourceTree),
-        schema: SourceTreeOrganizationSchema,
-        maxTokens: budget.maxTokens,
+    const organizations = [];
+    const batches = organizationBatches(sourceTree);
+    for (const [batchIndex, batch] of batches.entries()) {
+      const budget = params.resolveBudget("extraction_source_tree", 4096);
+      const startedAt = Date.now();
+      const response = await safeGenerateObject(
+        params.generateObject,
+        {
+          prompt: buildOrganizationPrompt(batch),
+          schema: SourceTreeOrganizationSchema,
+          maxTokens: budget.maxTokens,
+          taskKind: "extraction_source_tree",
+          budgetDiagnostics: budget,
+          providerOptions: { ...params.providerOptions, sourceSpans: params.sourceSpans }
+        },
+        {
+          fallback: { labels: [], groups: [] },
+          log: params.log
+        }
+      );
+      localTrack(response.usage, {
         taskKind: "extraction_source_tree",
-        budgetDiagnostics: budget,
-        providerOptions: { ...params.providerOptions, sourceSpans: params.sourceSpans }
-      },
-      {
-        fallback: { labels: [], groups: [] },
-        log: params.log
-      }
-    );
-    localTrack(response.usage, {
-      taskKind: "extraction_source_tree",
-      label: "source_tree_organizer",
-      maxTokens: budget.maxTokens,
-      durationMs: Date.now() - startedAt
-    });
-    sourceTree = applyOrganization(sourceTree, response.object);
+        label: batches.length > 1 ? `source_tree_organizer_${batchIndex + 1}` : "source_tree_organizer",
+        maxTokens: budget.maxTokens,
+        durationMs: Date.now() - startedAt
+      });
+      organizations.push(response.object);
+    }
+    sourceTree = applyOrganization(sourceTree, mergeOrganizationResults(organizations));
   } catch (error) {
     warnings.push(`Source-tree organizer failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
   }