@claritylabs/cl-sdk 3.1.18 → 3.1.20

package/dist/index.mjs

@@ -20,17 +20,19 @@ function isRetryableError(error) {
   }
   return false;
 }
-async function withRetry(fn, log) {
+async function withRetry(fn, log, options) {
+  const maxRetries = options?.maxRetries ?? MAX_RETRIES;
+  const baseDelayMs = options?.baseDelayMs ?? BASE_DELAY_MS;
   for (let attempt = 0; ; attempt++) {
     try {
       return await fn();
     } catch (error) {
-      if (!isRetryableError(error) || attempt >= MAX_RETRIES) {
+      if (!isRetryableError(error) || attempt >= maxRetries) {
         throw error;
       }
       const jitter = Math.random() * 1e3;
-      const delay = BASE_DELAY_MS * Math.pow(2, attempt) + jitter;
-      await log?.(`Retryable error, retrying in ${(delay / 1e3).toFixed(1)}s (attempt ${attempt + 1}/${MAX_RETRIES})...`);
+      const delay = baseDelayMs * Math.pow(2, attempt) + jitter;
+      await log?.(`Retryable error, retrying in ${(delay / 1e3).toFixed(1)}s (attempt ${attempt + 1}/${maxRetries})...`);
       await new Promise((resolve) => setTimeout(resolve, delay));
     }
   }
@@ -157,10 +159,8 @@ async function safeGenerateObject(generateObject, params, options) {
   const strictParams = { ...params, schema: toStrictSchema(params.schema) };
   for (let attempt = 0; attempt <= maxRetries; attempt++) {
     try {
-      const result = await withRetry(
-        () => generateObject(strictParams),
-        options?.log
-      );
+      const generate = () => generateObject(strictParams);
+      const result = options?.retry === false ? await generate() : await withRetry(generate, options?.log, options?.retry);
       return {
         ...result,
         object: params.schema.parse(sanitizeNulls(result.object))
@@ -9761,6 +9761,12 @@ function endorsementDescription(title, node) {
     title
   );
 }
+function endorsementTitleKey(node) {
+  const title = endorsementTitle(sourceNodeText(node));
+  if (title) return title.toLowerCase();
+  const fallback = cleanText(node.title, "");
+  return fallback ? fallback.toLowerCase() : void 0;
+}
 function nodePageEnd2(node) {
   return node.pageEnd ?? node.pageStart;
 }
@@ -10682,7 +10688,76 @@ function applyEndorsementGrouping(sourceTree) {
       (node) => endorsementGroupChildIds.has(node.id) ? { ...node, parentId: groupId, order: node.order + 1e-3 } : node
     );
   }
-  return normalizeSemanticHierarchy(nextTree);
+  return collapseNestedDuplicateEndorsements(normalizeSemanticHierarchy(nextTree));
+}
+function nearestEndorsementAncestor(node, byId) {
+  let parentId = node.parentId;
+  const seen = /* @__PURE__ */ new Set();
+  while (parentId) {
+    if (seen.has(parentId)) return void 0;
+    seen.add(parentId);
+    const parent = byId.get(parentId);
+    if (!parent) return void 0;
+    if (parent.kind === "endorsement") return parent;
+    parentId = parent.parentId;
+  }
+  return void 0;
+}
+function collapseNestedDuplicateEndorsements(sourceTree) {
+  const byId = new Map(sourceTree.map((node) => [node.id, node]));
+  const replacementById = /* @__PURE__ */ new Map();
+  const duplicateEvidenceByTarget = /* @__PURE__ */ new Map();
+  for (const node of sourceTree) {
+    if (node.kind !== "endorsement") continue;
+    const ancestor = nearestEndorsementAncestor(node, byId);
+    if (!ancestor) continue;
+    if (endorsementTitleKey(node) !== endorsementTitleKey(ancestor)) continue;
+    replacementById.set(node.id, ancestor.id);
+    duplicateEvidenceByTarget.set(ancestor.id, [
+      ...duplicateEvidenceByTarget.get(ancestor.id) ?? [],
+      node
+    ]);
+  }
+  if (replacementById.size === 0) return sourceTree;
+  const replacementParent = (parentId) => {
+    let next = parentId;
+    const seen = /* @__PURE__ */ new Set();
+    while (next && replacementById.has(next) && !seen.has(next)) {
+      seen.add(next);
+      next = replacementById.get(next);
+    }
+    return next;
+  };
+  const updated = sourceTree.filter((node) => !replacementById.has(node.id)).map((node) => {
+    const evidence = duplicateEvidenceByTarget.get(node.id);
+    const parentId = replacementParent(node.parentId);
+    if (!evidence?.length) {
+      return parentId === node.parentId ? node : {
+        ...node,
+        parentId,
+        metadata: {
+          ...node.metadata,
+          organizerRepair: "collapse_duplicate_endorsement_wrapper"
+        }
+      };
+    }
+    const evidenceNodes = [node, ...evidence];
+    const pageStarts = evidenceNodes.map((item) => item.pageStart).filter((page) => typeof page === "number");
+    const pageEnds = evidenceNodes.map((item) => item.pageEnd ?? item.pageStart).filter((page) => typeof page === "number");
+    return {
+      ...node,
+      parentId,
+      sourceSpanIds: [...new Set(evidenceNodes.flatMap((item) => item.sourceSpanIds))],
+      pageStart: pageStarts.length ? Math.min(...pageStarts) : node.pageStart,
+      pageEnd: pageEnds.length ? Math.max(...pageEnds) : node.pageEnd,
+      bbox: evidenceNodes.flatMap((item) => item.bbox ?? []).slice(0, 12),
+      metadata: {
+        ...node.metadata,
+        organizerRepair: "collapse_duplicate_endorsement_wrapper"
+      }
+    };
+  });
+  return normalizeDocumentSourceTreePaths(normalizeContainerEvidenceFromChildren(updated));
 }
 function compactNode2(node, maxText = 700) {
   return {
@@ -10881,9 +10956,13 @@ Rules:
 - Every returned value must include sourceNodeIds or sourceSpanIds from the provided nodes.
 - If a value is not directly supported, omit it.
 - Prefer declarations, schedules, premium tables, and endorsement schedules over generic policy wording.
+- Keep each coverage unit tied to one evidence scope: a declaration/core schedule row, a core policy form section, or one specific endorsement schedule. Do not merge declaration facts and endorsement schedule facts into the same coverage unit, even when they use the same coverage name.
+- If the declarations schedule and an endorsement schedule both list Network Security, Social Engineering Fraud, Regulatory Proceedings, or another same-named coverage, return separate coverage units for each supported source scope.
+- Use the declaration coverage name for declaration/core schedule rows. Use the endorsement title or endorsement schedule coverage name for endorsement rows, and include formNumber and endorsementNumber when source-backed.
 - For life, critical illness, disability, and long-term care policies, keep named benefit units and benefit subconditions as operational facts even when they do not have dollar limits. Examples include death benefit, disability benefit, total disability, catastrophic disability, return of premium, waiver, and conversion options. Put subcondition details in coverages[].limits with kind "other" when they belong under a broader benefit.
 - Treat an endorsement as one coverage unit when it contains a schedule. Do not split an endorsement schedule into generic rows like "Aggregate Limit".
 - For coverage schedules, put each claim, aggregate, sublimit, retention, deductible, and retroactive date values in coverages[].limits with labels and source IDs. Keep the legacy coverages[].limit as the primary display value only.
+- Extract coinsurance, participation percentage, or insurer/named-insured split terms as coverages[].limits entries with kind "other" when they are part of a coverage schedule.
 - Use coverageOrigin: "endorsement" for endorsement units and "core" for declarations/core policy coverage units.
 - Do not copy entire policy wording into fields.
 - Extract facts directly from source nodes. There is no deterministic fact baseline.
@@ -11141,6 +11220,26 @@ function findCellByVisualPosition(sourceCell, targetCells) {
   );
   return positioned[0]?.cell;
 }
+function isDateLikeColumn(label) {
+  return /\b(?:date|effective|expiration|retroactive)\b/i.test(cleanText(label, ""));
+}
+function containsDateValue(text) {
+  return /\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b/.test(text) || /\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{1,2},?\s+\d{2,4}\b/i.test(text);
+}
+function looksLikePolicyProse(text) {
+  const words = text.split(/\s+/).filter(Boolean);
+  if (words.length < 8) return false;
+  return /\b(?:are|is|will|shall|must|may|includes?|included|reduce|subject|payment|terms?|conditions?|provided|pursuant)\b/i.test(text);
+}
+function shouldMergeVisualContinuation(params) {
+  if (startsNewVisualTableItem(params.sourceCells)) return false;
+  const targetLabel = params.targetCell?.title;
+  if (isDateLikeColumn(targetLabel) && !containsDateValue(params.sourceText)) return false;
+  if (looksLikePolicyProse(params.sourceText) && !/\b(?:description|coverage|remarks?|notes?)\b/i.test(cleanText(targetLabel, ""))) {
+    return false;
+  }
+  return true;
+}
 function columnLabelStartIndex(rows) {
   const headerIndex = rows.findIndex(
     ({ row, cells }) => isSourceTreeHeaderRow(row) && cells.length > 1
@@ -11290,6 +11389,7 @@ function applyVisualTableRepair(sourceTree, repair) {
         targetColumnIndex: continuation.targetColumnIndex,
         targetColumnLabel: continuation.targetColumnLabel
       });
+      if (!shouldMergeVisualContinuation({ sourceText, sourceCells, targetCell })) continue;
       const sourceNodes = [sourceRow, ...sourceCells];
       const sourceSpanIds = sourceSpanIdsForNodes(sourceNodes);
       let mergedIntoCells = false;
@@ -11392,7 +11492,7 @@ async function runVisualTableRepair(params) {
   const repairResults = await Promise.all(candidates.map(async (candidate, index) => {
     const label = `source_tree_visual_table_repair_p${candidate.page}`;
     try {
-      const budget = params.resolveBudget("extraction_source_tree", 1800);
+      const budget = params.resolveBudget("extraction_visual_table_repair", 1800);
       const maxTokens = Math.min(budget.maxTokens, 2400);
       const startedAt = Date.now();
       const response = await safeGenerateObject(
@@ -11401,7 +11501,7 @@ async function runVisualTableRepair(params) {
           prompt: buildVisualTableRepairPrompt(candidate),
           schema: SourceTreeVisualTableRepairSchema,
           maxTokens,
-          taskKind: "extraction_source_tree",
+          taskKind: "extraction_visual_table_repair",
           budgetDiagnostics: { ...budget, maxTokens },
           trace: {
             label,
@@ -11414,7 +11514,9 @@ async function runVisualTableRepair(params) {
         },
         {
           fallback: { tables: [], warnings: [] },
-          log: params.log
+          maxRetries: 0,
+          log: params.log,
+          retry: false
         }
       );
       return {
@@ -11440,7 +11542,7 @@ async function runVisualTableRepair(params) {
       continue;
     }
     params.trackUsage(result.usage, {
-      taskKind: "extraction_source_tree",
+      taskKind: "extraction_visual_table_repair",
       label: result.label,
       maxTokens: result.maxTokens,
       durationMs: result.durationMs
@@ -11545,32 +11647,6 @@ var NORMALIZED_COMPATIBILITY_FIELDS = /* @__PURE__ */ new Set([
   "insurer",
   "broker"
 ]);
-function coverageBelongsToEndorsementCleanupChunk(coverage) {
-  return coverage.coverageOrigin === "endorsement" || Boolean(coverage.endorsementNumber);
-}
-function coverageCleanupChunks(profile) {
-  const basePolicy = [];
-  const endorsements = [];
-  profile.coverages.forEach((coverage, coverageIndex) => {
-    if (coverageBelongsToEndorsementCleanupChunk(coverage)) {
-      endorsements.push(coverageIndex);
-    } else {
-      basePolicy.push(coverageIndex);
-    }
-  });
-  return [
-    basePolicy.length ? {
-      group: "base_policy",
-      label: "Coverage cleanup: base policy",
-      coverageIndexes: basePolicy
-    } : void 0,
-    endorsements.length ? {
-      group: "endorsements",
-      label: "Coverage cleanup: endorsements",
-      coverageIndexes: endorsements
-    } : void 0
-  ].filter((chunk) => Boolean(chunk));
-}
 function valueOf(profile, key) {
   const value = profile[key];
   if (!value || typeof value !== "object" || Array.isArray(value) || !("value" in value)) return void 0;
@@ -11700,6 +11776,105 @@ function materializeDocument(params) {
     retroactiveDate: valueOf(profile, "retroactiveDate")
   };
 }
+function coverageCleanupGroups(profile) {
+  const groups = [];
+  const coreIndexes = [];
+  const endorsementIndexes = [];
+  const unclassifiedIndexes = [];
+  profile.coverages.forEach((coverage, coverageIndex) => {
+    if (coverage.coverageOrigin === "core") {
+      coreIndexes.push(coverageIndex);
+    } else if (coverage.coverageOrigin === "endorsement") {
+      endorsementIndexes.push(coverageIndex);
+    } else {
+      unclassifiedIndexes.push(coverageIndex);
+    }
+  });
+  if (coreIndexes.length) {
+    groups.push({
+      id: "policy",
+      label: "Coverage schedule cleanup: policy schedules",
+      coverageIndexes: coreIndexes
+    });
+  }
+  if (endorsementIndexes.length) {
+    groups.push({
+      id: "endorsements",
+      label: "Coverage schedule cleanup: endorsement schedules",
+      coverageIndexes: endorsementIndexes
+    });
+  }
+  if (unclassifiedIndexes.length) {
+    groups.push({
+      id: "source_backed",
+      label: "Coverage schedule cleanup: source-backed schedules",
+      coverageIndexes: unclassifiedIndexes
+    });
+  }
+  return groups.length > 1 ? groups : [{
+    id: "all",
+    label: "Coverage schedule cleanup"
+  }];
+}
+async function cleanupOperationalCoverageSchedules(params) {
+  const groups = coverageCleanupGroups(params.operationalProfile);
+  const validNodeIds = new Set(params.sourceTree.map((node) => node.id));
+  const validSpanIds = new Set(params.sourceSpans.map((span) => span.id));
+  const results = await Promise.all(groups.map(async (group, groupIndex) => {
+    const budget = params.resolveBudget("extraction_coverage_cleanup", 4096);
+    const startedAt = Date.now();
+    const response = await safeGenerateObject(
+      params.generateObject,
+      {
+        prompt: buildOperationalProfileCleanupPrompt(
+          params.sourceTree,
+          params.operationalProfile,
+          { coverageIndexes: group.coverageIndexes, label: group.label }
+        ),
+        schema: OperationalProfileCleanupSchema,
+        maxTokens: budget.maxTokens,
+        taskKind: "extraction_coverage_cleanup",
+        budgetDiagnostics: budget,
+        providerOptions: params.providerOptions,
+        trace: {
+          phase: "coverage_cleanup",
+          label: group.label,
+          itemCount: group.coverageIndexes?.length ?? params.operationalProfile.coverages.length,
+          coverageGroup: group.id,
+          batchIndex: groups.length > 1 ? groupIndex + 1 : void 0,
+          batchCount: groups.length > 1 ? groups.length : void 0,
+          sourceBacked: true
+        }
+      },
+      {
+        fallback: { coverageDecisions: [], warnings: [] },
+        maxRetries: 0,
+        log: params.log,
+        retry: false
+      }
+    );
+    params.trackUsage(response.usage, {
+      taskKind: "extraction_coverage_cleanup",
+      label: group.id === "all" ? "coverage_cleanup" : `coverage_cleanup_${group.id}`,
+      maxTokens: budget.maxTokens,
+      durationMs: Date.now() - startedAt
+    });
+    return response.object;
+  }));
+  const cleanup = {
+    coverageDecisions: results.flatMap((result) => result.coverageDecisions ?? []),
+    warnings: results.flatMap((result) => result.warnings ?? [])
+  };
+  return {
+    operationalProfile: applyOperationalProfileCleanup(
+      params.operationalProfile,
+      cleanup,
+      validNodeIds,
+      validSpanIds
+    ),
+    warnings: cleanup.warnings
+  };
+}
 async function runSourceTreeExtraction(params) {
   const sourceSpans = normalizeSourceSpans(params.sourceSpans);
   const formHints = normalizeFormHints(params.formInventory?.forms, sourceSpans);
@@ -11749,7 +11924,9 @@ async function runSourceTreeExtraction(params) {
           },
           {
             fallback: { labels: [], groups: [] },
-            log: params.log
+            maxRetries: 0,
+            log: params.log,
+            retry: false
           }
         );
         return {
@@ -11794,7 +11971,9 @@ async function runSourceTreeExtraction(params) {
         },
         {
           fallback: { labels: [], groups: [] },
-          log: params.log
+          maxRetries: 0,
+          log: params.log,
+          retry: false
         }
       );
       localTrack(response.usage, {
@@ -11836,7 +12015,9 @@ async function runSourceTreeExtraction(params) {
       },
       {
         fallback: emptyProfile,
-        log: params.log
+        maxRetries: 0,
+        log: params.log,
+        retry: false
       }
     );
     localTrack(response.usage, {
@@ -11856,69 +12037,18 @@ async function runSourceTreeExtraction(params) {
   }
   if (operationalProfile.coverages.length > 0) {
     try {
-      const validNodeIds = new Set(sourceTree.map((node) => node.id));
-      const validSpanIds = new Set(sourceSpans.map((span) => span.id));
-      const chunks = coverageCleanupChunks(operationalProfile);
-      if (chunks.length > 1) {
-        await params.log?.(`Operational profile coverage cleanup reviewing ${chunks.length} coverage groups in parallel`);
-      }
-      const cleanupResults = await Promise.all(chunks.map(async (chunk, batchIndex) => {
-        const budget = params.resolveBudget("extraction_coverage_cleanup", 4096);
-        const startedAt = Date.now();
-        const response = await safeGenerateObject(
-          params.generateObject,
-          {
-            prompt: buildOperationalProfileCleanupPrompt(sourceTree, operationalProfile, {
-              coverageIndexes: chunk.coverageIndexes,
-              label: chunk.label
-            }),
-            schema: OperationalProfileCleanupSchema,
-            maxTokens: budget.maxTokens,
-            taskKind: "extraction_coverage_cleanup",
-            budgetDiagnostics: budget,
-            providerOptions: params.providerOptions,
-            trace: {
-              phase: "coverage_cleanup",
-              label: chunk.label,
-              batchIndex: batchIndex + 1,
-              batchCount: chunks.length,
-              coverageGroup: chunk.group,
-              itemCount: chunk.coverageIndexes.length,
-              sourceBacked: true
-            }
-          },
-          {
-            fallback: { coverageDecisions: [], warnings: [] },
-            maxRetries: 0,
-            log: params.log
-          }
-        );
-        return {
-          batchIndex,
-          chunk,
-          budget,
-          durationMs: Date.now() - startedAt,
-          usage: response.usage,
-          cleanup: response.object
-        };
-      }));
-      const cleanup = { coverageDecisions: [], warnings: [] };
-      for (const result of cleanupResults.sort((left, right) => left.batchIndex - right.batchIndex)) {
-        localTrack(result.usage, {
-          taskKind: "extraction_coverage_cleanup",
-          label: result.chunk.label,
-          maxTokens: result.budget.maxTokens,
-          durationMs: result.durationMs
-        });
-        cleanup.coverageDecisions.push(...result.cleanup.coverageDecisions);
-        cleanup.warnings.push(...result.cleanup.warnings);
-      }
-      operationalProfile = applyOperationalProfileCleanup(
+      const cleanup = await cleanupOperationalCoverageSchedules({
+        sourceTree,
+        sourceSpans,
         operationalProfile,
-        cleanup,
-        validNodeIds,
-        validSpanIds
-      );
+        generateObject: params.generateObject,
+        providerOptions: params.providerOptions,
+        resolveBudget: params.resolveBudget,
+        trackUsage: localTrack,
+        log: params.log
+      });
+      operationalProfile = cleanup.operationalProfile;
+      warnings.push(...cleanup.warnings);
     } catch (error) {
       warnings.push(`Operational profile cleanup pass failed; uncleaned profile used (${error instanceof Error ? error.message : String(error)})`);
     }
@@ -12313,7 +12443,9 @@ ${sourceText}`;
           },
           {
             fallback: { forms: [] },
+            maxRetries: 0,
             log,
+            retry: false,
             onError: (err, attempt) => log?.(`Form inventory attempt ${attempt + 1} failed: ${err instanceof Error ? err.message : String(err)}`)
           }
         );