@claritylabs/cl-sdk 3.1.14 → 3.1.16

package/dist/index.mjs

@@ -9142,6 +9142,10 @@ var OperationalProfileCleanupSchema = z41.object({
   })).default([]),
   warnings: z41.array(z41.string()).default([])
 });
+var CLEANUP_CANDIDATE_ID_LIMIT = 12;
+var CLEANUP_SOURCE_NODE_LIMIT = 90;
+var CLEANUP_SIBLING_WINDOW = 4;
+var CLEANUP_KEYWORD = /\b(coverage|limit|liability|deductible|retention|retroactive|premium|aggregate|sublimit|sub-limit|claim|occurrence|loss|proceeding|endorsement|declarations?)\b|\$[0-9]/i;
 function compactNode(node, maxText = 700) {
   return {
     id: node.id,
@@ -9154,6 +9158,9 @@ function compactNode(node, maxText = 700) {
     text: (node.textExcerpt ?? node.description).slice(0, maxText)
   };
 }
+function compactIds(ids) {
+  return uniqueStrings([...ids ?? []]).slice(0, CLEANUP_CANDIDATE_ID_LIMIT);
+}
 function compactCoverageForCleanup(coverage, coverageIndex) {
   return {
     coverageIndex,
@@ -9163,8 +9170,8 @@ function compactCoverageForCleanup(coverage, coverageIndex) {
     premium: coverage.premium,
     retroactiveDate: coverage.retroactiveDate,
     coverageOrigin: coverage.coverageOrigin,
-    sourceNodeIds: coverage.sourceNodeIds,
-    sourceSpanIds: coverage.sourceSpanIds,
+    sourceNodeIds: compactIds(coverage.sourceNodeIds),
+    sourceSpanIds: compactIds(coverage.sourceSpanIds),
     terms: coverage.limits.map((term, termIndex) => ({
       termIndex,
       kind: term.kind,
@@ -9172,13 +9179,112 @@ function compactCoverageForCleanup(coverage, coverageIndex) {
       value: term.value,
       amount: term.amount,
       appliesTo: term.appliesTo,
-      sourceNodeIds: term.sourceNodeIds,
-      sourceSpanIds: term.sourceSpanIds
+      sourceNodeIds: compactIds(term.sourceNodeIds),
+      sourceSpanIds: compactIds(term.sourceSpanIds)
     }))
   };
 }
+function nodeTextForSelection(node) {
+  return [
+    node.kind,
+    node.title,
+    node.description,
+    node.textExcerpt
+  ].filter(Boolean).join(" ");
+}
+function coverageTextForSelection(coverage) {
+  return [
+    coverage.name,
+    coverage.coverageCode,
+    coverage.limit,
+    coverage.deductible,
+    coverage.premium,
+    coverage.retroactiveDate,
+    coverage.sectionRef,
+    coverage.endorsementNumber,
+    ...coverage.limits.flatMap((term) => [
+      term.kind,
+      term.label,
+      term.value,
+      term.appliesTo
+    ])
+  ].filter(Boolean).join(" ");
+}
+function nodeTextMatchesCoverage(node, coverageTerms) {
+  const text = nodeTextForSelection(node).toLowerCase();
+  return coverageTerms.some((term) => term.length >= 5 && text.includes(term));
+}
+function selectCoverageCleanupNodes(sourceTree, profile) {
+  const nodeById = new Map(sourceTree.map((node) => [node.id, node]));
+  const childrenByParent = /* @__PURE__ */ new Map();
+  for (const node of sourceTree) {
+    if (!node.parentId) continue;
+    const children = childrenByParent.get(node.parentId) ?? [];
+    children.push(node);
+    childrenByParent.set(node.parentId, children);
+  }
+  for (const children of childrenByParent.values()) {
+    children.sort((left, right) => left.order - right.order);
+  }
+  const selected = /* @__PURE__ */ new Map();
+  const addNode = (node, score) => {
+    if (!node || node.kind === "document") return;
+    const current = selected.get(node.id);
+    if (!current || score > current.score) selected.set(node.id, { node, score });
+  };
+  const sourceNodeIds = uniqueStrings(profile.coverages.flatMap((coverage) => [
+    ...coverage.sourceNodeIds,
+    ...coverage.limits.flatMap((term) => term.sourceNodeIds)
+  ]));
+  const coveragePages2 = /* @__PURE__ */ new Set();
+  const coverageTerms = uniqueStrings(profile.coverages.flatMap(
+    (coverage) => coverageTextForSelection(coverage).toLowerCase().split(/[^a-z0-9$,.]+/i).filter((part) => part.length >= 5)
+  ));
+  for (const id of sourceNodeIds) {
+    const node = nodeById.get(id);
+    if (!node) continue;
+    addNode(node, 1e3);
+    if (node.pageStart) coveragePages2.add(node.pageStart);
+    if (node.pageEnd) coveragePages2.add(node.pageEnd);
+    let parentId = node.parentId;
+    let parentScore = 940;
+    while (parentId) {
+      const parent = nodeById.get(parentId);
+      if (!parent) break;
+      addNode(parent, parentScore);
+      parentId = parent.parentId;
+      parentScore -= 30;
+    }
+    const siblings = node.parentId ? childrenByParent.get(node.parentId) ?? [] : [];
+    for (const sibling of siblings) {
+      if (Math.abs(sibling.order - node.order) <= CLEANUP_SIBLING_WINDOW) {
+        addNode(sibling, 850 - Math.abs(sibling.order - node.order));
+      }
+    }
+  }
+  for (const selectedNode of [...selected.values()].map((entry) => entry.node)) {
+    const children = childrenByParent.get(selectedNode.id) ?? [];
+    for (const child of children.slice(0, 24)) addNode(child, 760);
+  }
+  for (const node of sourceTree) {
+    if (node.kind === "document") continue;
+    if (!node.pageStart || !coveragePages2.has(node.pageStart)) continue;
+    const text = nodeTextForSelection(node);
+    if (CLEANUP_KEYWORD.test(text) || nodeTextMatchesCoverage(node, coverageTerms)) {
+      addNode(node, 600);
+    }
+  }
+  return [...selected.values()].sort(
+    (left, right) => right.score - left.score || (left.node.pageStart ?? Number.MAX_SAFE_INTEGER) - (right.node.pageStart ?? Number.MAX_SAFE_INTEGER) || left.node.order - right.node.order
+  ).slice(0, CLEANUP_SOURCE_NODE_LIMIT).sort(
+    (left, right) => (left.node.pageStart ?? Number.MAX_SAFE_INTEGER) - (right.node.pageStart ?? Number.MAX_SAFE_INTEGER) || left.node.order - right.node.order
+  ).map((entry) => entry.node);
+}
 function buildOperationalProfileCleanupPrompt(sourceTree, profile) {
-  const nodes = sourceTree.filter((node) => node.kind !== "document").slice(0, 320).map((node) => compactNode(node, node.kind === "page" ? 900 : 700));
+  const nodes = selectCoverageCleanupNodes(sourceTree, profile).map((node) => compactNode(
+    node,
+    node.kind === "page" || node.kind === "page_group" ? 260 : node.kind === "table_row" || node.kind === "table_cell" ? 520 : 360
+  ));
   const candidate = {
     documentType: profile.documentType,
     policyTypes: profile.policyTypes,
@@ -10749,6 +10855,46 @@ function tableRowTextFromCells(cells) {
   }).filter(Boolean).join(" | ");
   return text ? cleanText(text, text) : void 0;
 }
+function normalizeTableBoundaryDelimiters(value) {
+  const text = cleanText(value, "");
+  if (!text) return void 0;
+  const normalized = text.replace(/\s+\|\s+\|/g, " | ").replace(/\s+\/\s+\|/g, " |").replace(/\s+[|/]\s*$/g, "").replace(/\s{2,}/g, " ").trim();
+  return normalized || void 0;
+}
+function normalizeSourceTreeTableDisplayText(sourceTree) {
+  const byParent = nodesByParent(sourceTree);
+  const updates = /* @__PURE__ */ new Map();
+  const currentNode = (node) => updates.get(node.id) ?? node;
+  for (const node of sourceTree) {
+    if (node.kind !== "table_cell") continue;
+    const textExcerpt = normalizeTableBoundaryDelimiters(node.textExcerpt);
+    const textChanged = textExcerpt !== void 0 && textExcerpt !== node.textExcerpt;
+    const description = textChanged && textExcerpt ? normalizeTableBoundaryDelimiters([node.title, textExcerpt].filter(Boolean).join(" | ")) : normalizeTableBoundaryDelimiters(node.description);
+    if (textExcerpt === node.textExcerpt && description === node.description) continue;
+    updates.set(node.id, {
+      ...node,
+      ...textExcerpt !== void 0 ? { textExcerpt } : {},
+      ...description !== void 0 ? { description } : {}
+    });
+  }
+  for (const node of sourceTree) {
+    if (node.kind !== "table_row") continue;
+    const cells = (byParent.get(node.id) ?? []).filter((child) => child.kind === "table_cell").map(currentNode).sort(
+      (left, right) => tableCellColumnIndex(left, 0) - tableCellColumnIndex(right, 0) || left.order - right.order || left.id.localeCompare(right.id)
+    );
+    const textExcerpt = cells.length ? tableRowTextFromCells(cells) : normalizeTableBoundaryDelimiters(node.textExcerpt);
+    const textChanged = textExcerpt !== void 0 && textExcerpt !== node.textExcerpt;
+    const description = textChanged && textExcerpt ? normalizeTableBoundaryDelimiters([node.title, textExcerpt].filter(Boolean).join(" | ")) : normalizeTableBoundaryDelimiters(node.description);
+    if (textExcerpt === node.textExcerpt && description === node.description) continue;
+    updates.set(node.id, {
+      ...node,
+      ...textExcerpt !== void 0 ? { textExcerpt } : {},
+      ...description !== void 0 ? { description } : {}
+    });
+  }
+  if (updates.size === 0) return sourceTree;
+  return sourceTree.map((node) => updates.get(node.id) ?? node);
+}
 function tableRowTextForPrompt(row, cells) {
   return cleanText(
     cells.length ? cells.map(tableCellText).filter(Boolean).join(" | ") : row.textExcerpt ?? row.description ?? row.title,
@@ -11547,7 +11693,7 @@ async function runSourceTreeExtraction(params) {
     trackUsage: localTrack,
     log: params.log
   });
-  sourceTree = visualTableRepair.sourceTree;
+  sourceTree = normalizeSourceTreeTableDisplayText(visualTableRepair.sourceTree);
   warnings.push(...visualTableRepair.warnings);
   const emptyProfile = emptyOperationalProfile();
   let operationalProfile = emptyProfile;