@claritylabs/cl-sdk 3.0.6 → 3.0.8

package/dist/index.js

@@ -9425,13 +9425,168 @@ function simplifyOrganizerTitle(value, fallback, kind) {
   return title;
 }
 function endorsementReference(value) {
-  return value?.match(/\bendorsement\s+(?:no\.?|number|#)?\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1]?.toUpperCase();
+  const text = cleanText(value, "");
+  const explicit = text.match(/\bendorsement\s+(?:no\.?|number|#)?\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1]?.toUpperCase();
+  if (explicit) return explicit;
+  return text.match(/\b(?:[A-Z]{2,}-)?END\s+0*([0-9]{1,4})\b/i)?.[1]?.toUpperCase();
 }
 function endorsementTitle(value) {
   const text = cleanText(value, "");
-  const number = text.match(/\bendorsement\s+(?:no\.?|number|#)\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1]?.toUpperCase();
+  const explicit = text.match(/\bendorsement\s+(?:no\.?|number|#)\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1]?.toUpperCase();
+  const number = explicit ?? text.match(/\b(?:[A-Z]{2,}-)?END\s+0*([0-9]{1,4})\b/i)?.[1]?.toUpperCase();
   return number ? `Endorsement No. ${number}` : void 0;
 }
+function sourceNodeText(node) {
+  return cleanText([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "), "");
+}
+function looksLikeEndorsementStart(node) {
+  const title = cleanText(node.title, "");
+  const body = cleanText([node.textExcerpt, node.description].filter(Boolean).join(" "), "");
+  const start = body.slice(0, 260);
+  if (/\bthis endorsement changes the policy\b/i.test(start) && endorsementReference(start)) return true;
+  if (/^(?:[A-Z]{2,}-)?END\s+0*[0-9]{1,4}\b/i.test(start)) return true;
+  if (/^endorsement\s+(?:no\.?|number|#)\s*[A-Z0-9][A-Z0-9.-]*\b/i.test(start)) return true;
+  return /^endorsement\s+(?:no\.?|number|#)\s*[A-Z0-9][A-Z0-9.-]*\b/i.test(title) && /\bthis endorsement changes the policy\b/i.test(body);
+}
+function endorsementStartTitle(node) {
+  return looksLikeEndorsementStart(node) ? endorsementTitle(sourceNodeText(node)) : void 0;
+}
+function semanticGroupNodeId(documentId, kind, title, childNodeIds) {
+  return [
+    documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
+    "source_node",
+    kind,
+    title.replace(/[^a-zA-Z0-9_.:-]/g, "_").toLowerCase().slice(0, 48),
+    childNodeIds.join("_").replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 80)
+  ].join(":");
+}
+function looksLikeDeclarationsStart(node) {
+  return /\bdeclarations?\s+(page|schedule)\b/i.test(sourceNodeText(node));
+}
+function looksLikeDeclarationsContinuation(node) {
+  const text = sourceNodeText(node);
+  return looksLikeDeclarationsStart(node) || /\b(item\s+\d+\.|coverage part|each claim limit|aggregate limit|retroactive date|self-insured retention|premium|payment plan|producer|broker|forms? and endorsements?|extended reporting period|discovery period)\b/i.test(text);
+}
+function looksLikePolicyFormStart(node) {
+  const text = sourceNodeText(node);
+  return /\bpolicy form\b/i.test(node.title) || /\btechnology errors?\s*&?\s*omissions\b/i.test(text) && /\bplease read this entire policy carefully\b/i.test(text) || /\bform\s+[A-Z]{2,}-[A-Z0-9-]+\s+\d{2}\s+\d{2}\b/i.test(text);
+}
+function looksLikePolicyFormContinuation(node) {
+  const text = sourceNodeText(node);
+  if (looksLikePolicyFormStart(node)) return true;
+  return /\b(insuring agreement|definitions?|exclusions?|conditions?|claim means|insured means|wrongful act means|limits of liability|notice of claim|cancellation by|action against the company)\b/i.test(text);
+}
+function groupAdjacentChildren(params) {
+  if (params.childIds.length < 2) return params.sourceTree;
+  const children = params.childIds.map((id2) => params.children.find((child) => child.id === id2)).filter((child) => Boolean(child));
+  if (children.length < 2) return params.sourceTree;
+  const parentId = children[0].parentId;
+  if (!children.every((child) => child.parentId === parentId)) return params.sourceTree;
+  const documentId = children[0].documentId;
+  const id = semanticGroupNodeId(documentId, params.kind, params.title, children.map((child) => child.id));
+  if (params.sourceTree.some((node) => node.id === id)) return params.sourceTree;
+  const pageStarts = children.map((child) => child.pageStart).filter((page) => typeof page === "number");
+  const pageEnds = children.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
+  const sourceSpanIds = [...new Set(children.flatMap((child) => child.sourceSpanIds))];
+  const order = Math.min(...children.map((child) => child.order));
+  const groupNode = {
+    id,
+    documentId,
+    parentId,
+    kind: params.kind,
+    title: params.title,
+    description: params.description,
+    textExcerpt: children.map((child) => child.textExcerpt ?? child.description).filter(Boolean).join("\n\n").slice(0, 1600),
+    sourceSpanIds,
+    pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
+    pageEnd: pageEnds.length ? Math.max(...pageEnds) : void 0,
+    bbox: children.flatMap((child) => child.bbox ?? []).slice(0, 12),
+    order,
+    path: "",
+    metadata: { sourceTreeVersion: "v3", organizer: params.organizer }
+  };
+  const wanted = new Set(children.map((child) => child.id));
+  return [
+    ...params.sourceTree.map(
+      (node) => wanted.has(node.id) ? { ...node, parentId: id, order: node.order + 1e-3 } : node
+    ),
+    groupNode
+  ];
+}
+function applySemanticPageGrouping(sourceTree) {
+  const relabeled = sourceTree.map((node) => {
+    if (node.kind === "document" || node.kind === "page_group") return node;
+    const endorsement = endorsementStartTitle(node);
+    if (endorsement && node.kind === "page") {
+      return {
+        ...node,
+        kind: "endorsement",
+        title: endorsement,
+        description: cleanText([endorsement, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "), endorsement),
+        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+      };
+    }
+    if (node.kind === "page" && looksLikeDeclarationsStart(node)) {
+      return {
+        ...node,
+        title: "Declarations",
+        description: cleanText([node.description, "Declarations"].join(" "), "Declarations"),
+        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+      };
+    }
+    if (node.kind === "page" && looksLikePolicyFormStart(node)) {
+      return {
+        ...node,
+        title: "Policy Form",
+        description: cleanText([node.description, "Policy Form"].join(" "), "Policy Form"),
+        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+      };
+    }
+    return node;
+  });
+  const rootId = sourceTreeRootId(relabeled);
+  const children = (nodesByParent(relabeled).get(rootId) ?? []).filter((node) => node.kind !== "document").sort((left, right) => left.order - right.order);
+  let nextTree = relabeled;
+  const declarationsStartIndex = children.findIndex(looksLikeDeclarationsStart);
+  if (declarationsStartIndex >= 0) {
+    const declarationIds = [];
+    for (let index = declarationsStartIndex; index < children.length; index += 1) {
+      const child = children[index];
+      if (index > declarationsStartIndex && (looksLikePolicyFormStart(child) || looksLikeEndorsementStart(child))) break;
+      if (!looksLikeDeclarationsContinuation(child)) break;
+      declarationIds.push(child.id);
+    }
+    nextTree = groupAdjacentChildren({
+      sourceTree: nextTree,
+      children,
+      childIds: declarationIds,
+      kind: "page_group",
+      title: "Declarations",
+      description: "Declarations pages and schedules grouped by source order",
+      organizer: "semantic_declarations_grouping"
+    });
+  }
+  const policyStartIndex = children.findIndex(looksLikePolicyFormStart);
+  if (policyStartIndex >= 0) {
+    const policyIds = [];
+    for (let index = policyStartIndex; index < children.length; index += 1) {
+      const child = children[index];
+      if (index > policyStartIndex && looksLikeEndorsementStart(child)) break;
+      if (!looksLikePolicyFormContinuation(child)) break;
+      policyIds.push(child.id);
+    }
+    nextTree = groupAdjacentChildren({
+      sourceTree: nextTree,
+      children,
+      childIds: policyIds,
+      kind: "form",
+      title: "Policy Form",
+      description: "Policy form pages grouped by source order",
+      organizer: "semantic_policy_form_grouping"
+    });
+  }
+  return applyEndorsementGrouping(normalizeDocumentSourceTreePaths(nextTree));
+}
 function rejectsOrganizerGroup(group, children) {
   if (/^endorsements?\b/i.test(group.title) && group.kind !== "page_group") return true;
   if (group.kind !== "endorsement") return false;
@@ -9455,8 +9610,19 @@ function endorsementGroupNodeId(documentId, parentId) {
 }
 function applyEndorsementGrouping(sourceTree) {
   const relabeledTree = sourceTree.map((node) => {
-    if (node.kind === "document" || isEndorsementGroup(node) || node.kind === "endorsement") return node;
-    const title = endorsementTitle([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "));
+    if (node.kind === "document" || isEndorsementGroup(node)) return node;
+    const title = endorsementStartTitle(node);
+    if (!title && node.kind === "endorsement") {
+      return {
+        ...node,
+        kind: "page",
+        title: node.pageStart ? `Page ${node.pageStart}` : cleanText(node.title, "Page"),
+        metadata: {
+          ...node.metadata,
+          organizerRepair: "demote_incidental_endorsement_reference"
+        }
+      };
+    }
     if (!title) return node;
     return {
       ...node,
@@ -9496,7 +9662,7 @@ function applyEndorsementGrouping(sourceTree) {
   });
   nextTree = nextTree.map((node) => {
     if (!endorsementGroupIds.has(node.parentId ?? "")) return node;
-    const title = endorsementTitle([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "));
+    const title = endorsementStartTitle(node);
     if (!title) return node;
     return {
       ...node,
@@ -9918,6 +10084,7 @@ async function runSourceTreeExtraction(params) {
   } catch (error) {
     warnings.push(`Source-tree organizer failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
   }
+  sourceTree = applySemanticPageGrouping(sourceTree);
   const deterministicProfile = buildDeterministicOperationalProfile({
     sourceTree,
     sourceSpans
@@ -9958,7 +10125,6 @@ async function runSourceTreeExtraction(params) {
   } catch (error) {
     warnings.push(`Operational profile model pass failed; deterministic profile used (${error instanceof Error ? error.message : String(error)})`);
   }
-  sourceTree = applyEndorsementGrouping(sourceTree);
   const document = materializeDocument({
     id: params.id,
     sourceTree,