@claritylabs/cl-sdk 3.0.5 → 3.0.7

package/dist/index.mjs

@@ -9069,6 +9069,7 @@ function simplifyOrganizerTitle(value, fallback, kind) {
   if (/^declarations\b/i.test(title)) return "Declarations";
   if (/^policy\s+form\b/i.test(title)) return "Policy Form";
   if (/^definitions\b/i.test(title)) return "Definitions";
+  if (kind === "page_group" && /^endorsements?\b/i.test(title)) return "Endorsements";
   const endorsementNumber = title.match(/^endorsement\s+(?:no\.?|number|#)?\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1];
   if (endorsementNumber) return `Endorsement No. ${endorsementNumber}`;
   if (kind === "endorsement" && /^endorsements?\s+\d+\s*[–-]\s*\d+\b/i.test(title)) {
@@ -9077,9 +9078,159 @@ function simplifyOrganizerTitle(value, fallback, kind) {
   return title;
 }
 function endorsementReference(value) {
-  return value?.match(/\bendorsement\s+(?:no\.?|number|#)?\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1]?.toUpperCase();
+  const text = cleanText(value, "");
+  const explicit = text.match(/\bendorsement\s+(?:no\.?|number|#)?\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1]?.toUpperCase();
+  if (explicit) return explicit;
+  return text.match(/\b(?:[A-Z]{2,}-)?END\s+0*([0-9]{1,4})\b/i)?.[1]?.toUpperCase();
+}
+function endorsementTitle(value) {
+  const text = cleanText(value, "");
+  const explicit = text.match(/\bendorsement\s+(?:no\.?|number|#)\s*([A-Z0-9][A-Z0-9.-]*)\b/i)?.[1]?.toUpperCase();
+  const number = explicit ?? text.match(/\b(?:[A-Z]{2,}-)?END\s+0*([0-9]{1,4})\b/i)?.[1]?.toUpperCase();
+  return number ? `Endorsement No. ${number}` : void 0;
+}
+function sourceNodeText(node) {
+  return cleanText([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "), "");
+}
+function semanticGroupNodeId(documentId, kind, title, childNodeIds) {
+  return [
+    documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
+    "source_node",
+    kind,
+    title.replace(/[^a-zA-Z0-9_.:-]/g, "_").toLowerCase().slice(0, 48),
+    childNodeIds.join("_").replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 80)
+  ].join(":");
+}
+function looksLikeDeclarationsStart(node) {
+  return /\bdeclarations?\s+(page|schedule)\b/i.test(sourceNodeText(node));
+}
+function looksLikeDeclarationsContinuation(node) {
+  const text = sourceNodeText(node);
+  return looksLikeDeclarationsStart(node) || /\b(item\s+\d+\.|coverage part|each claim limit|aggregate limit|retroactive date|self-insured retention|premium|payment plan|producer|broker|forms? and endorsements?|extended reporting period|discovery period)\b/i.test(text);
+}
+function looksLikePolicyFormStart(node) {
+  const text = sourceNodeText(node);
+  return /\bpolicy form\b/i.test(node.title) || /\btechnology errors?\s*&?\s*omissions\b/i.test(text) && /\bplease read this entire policy carefully\b/i.test(text) || /\bform\s+[A-Z]{2,}-[A-Z0-9-]+\s+\d{2}\s+\d{2}\b/i.test(text);
+}
+function looksLikePolicyFormContinuation(node) {
+  const text = sourceNodeText(node);
+  if (looksLikePolicyFormStart(node)) return true;
+  return /\b(insuring agreement|definitions?|exclusions?|conditions?|claim means|insured means|wrongful act means|limits of liability|notice of claim|cancellation by|action against the company)\b/i.test(text);
+}
+function groupAdjacentChildren(params) {
+  if (params.childIds.length < 2) return params.sourceTree;
+  const children = params.childIds.map((id2) => params.children.find((child) => child.id === id2)).filter((child) => Boolean(child));
+  if (children.length < 2) return params.sourceTree;
+  const parentId = children[0].parentId;
+  if (!children.every((child) => child.parentId === parentId)) return params.sourceTree;
+  const documentId = children[0].documentId;
+  const id = semanticGroupNodeId(documentId, params.kind, params.title, children.map((child) => child.id));
+  if (params.sourceTree.some((node) => node.id === id)) return params.sourceTree;
+  const pageStarts = children.map((child) => child.pageStart).filter((page) => typeof page === "number");
+  const pageEnds = children.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
+  const sourceSpanIds = [...new Set(children.flatMap((child) => child.sourceSpanIds))];
+  const order = Math.min(...children.map((child) => child.order));
+  const groupNode = {
+    id,
+    documentId,
+    parentId,
+    kind: params.kind,
+    title: params.title,
+    description: params.description,
+    textExcerpt: children.map((child) => child.textExcerpt ?? child.description).filter(Boolean).join("\n\n").slice(0, 1600),
+    sourceSpanIds,
+    pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
+    pageEnd: pageEnds.length ? Math.max(...pageEnds) : void 0,
+    bbox: children.flatMap((child) => child.bbox ?? []).slice(0, 12),
+    order,
+    path: "",
+    metadata: { sourceTreeVersion: "v3", organizer: params.organizer }
+  };
+  const wanted = new Set(children.map((child) => child.id));
+  return [
+    ...params.sourceTree.map(
+      (node) => wanted.has(node.id) ? { ...node, parentId: id, order: node.order + 1e-3 } : node
+    ),
+    groupNode
+  ];
+}
+function applySemanticPageGrouping(sourceTree) {
+  const relabeled = sourceTree.map((node) => {
+    if (node.kind === "document" || node.kind === "page_group") return node;
+    const text = sourceNodeText(node);
+    const endorsement = endorsementTitle(text);
+    if (endorsement && node.kind === "page") {
+      return {
+        ...node,
+        kind: "endorsement",
+        title: endorsement,
+        description: cleanText([endorsement, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "), endorsement),
+        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+      };
+    }
+    if (node.kind === "page" && looksLikeDeclarationsStart(node)) {
+      return {
+        ...node,
+        title: "Declarations",
+        description: cleanText([node.description, "Declarations"].join(" "), "Declarations"),
+        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+      };
+    }
+    if (node.kind === "page" && looksLikePolicyFormStart(node)) {
+      return {
+        ...node,
+        title: "Policy Form",
+        description: cleanText([node.description, "Policy Form"].join(" "), "Policy Form"),
+        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+      };
+    }
+    return node;
+  });
+  const rootId = sourceTreeRootId(relabeled);
+  const children = (nodesByParent(relabeled).get(rootId) ?? []).filter((node) => node.kind !== "document").sort((left, right) => left.order - right.order);
+  let nextTree = relabeled;
+  const declarationsStartIndex = children.findIndex(looksLikeDeclarationsStart);
+  if (declarationsStartIndex >= 0) {
+    const declarationIds = [];
+    for (let index = declarationsStartIndex; index < children.length; index += 1) {
+      const child = children[index];
+      if (index > declarationsStartIndex && (looksLikePolicyFormStart(child) || endorsementTitle(sourceNodeText(child)))) break;
+      if (!looksLikeDeclarationsContinuation(child)) break;
+      declarationIds.push(child.id);
+    }
+    nextTree = groupAdjacentChildren({
+      sourceTree: nextTree,
+      children,
+      childIds: declarationIds,
+      kind: "page_group",
+      title: "Declarations",
+      description: "Declarations pages and schedules grouped by source order",
+      organizer: "semantic_declarations_grouping"
+    });
+  }
+  const policyStartIndex = children.findIndex(looksLikePolicyFormStart);
+  if (policyStartIndex >= 0) {
+    const policyIds = [];
+    for (let index = policyStartIndex; index < children.length; index += 1) {
+      const child = children[index];
+      if (index > policyStartIndex && endorsementTitle(sourceNodeText(child))) break;
+      if (!looksLikePolicyFormContinuation(child)) break;
+      policyIds.push(child.id);
+    }
+    nextTree = groupAdjacentChildren({
+      sourceTree: nextTree,
+      children,
+      childIds: policyIds,
+      kind: "form",
+      title: "Policy Form",
+      description: "Policy form pages grouped by source order",
+      organizer: "semantic_policy_form_grouping"
+    });
+  }
+  return applyEndorsementGrouping(normalizeDocumentSourceTreePaths(nextTree));
 }
 function rejectsOrganizerGroup(group, children) {
+  if (/^endorsements?\b/i.test(group.title) && group.kind !== "page_group") return true;
   if (group.kind !== "endorsement") return false;
   if (/^endorsements?\s+\d+\s*[–-]\s*\d+\b/i.test(group.title)) return true;
   const childNumbers = new Set(
@@ -9087,6 +9238,120 @@ function rejectsOrganizerGroup(group, children) {
   );
   return childNumbers.size > 1;
 }
+function isEndorsementGroup(node) {
+  return node.kind === "page_group" && /^endorsements?\b/i.test(node.title);
+}
+function endorsementGroupNodeId(documentId, parentId) {
+  return [
+    documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
+    "source_node",
+    "page_group",
+    "endorsements",
+    parentId?.replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 48) ?? "root"
+  ].join(":");
+}
+function applyEndorsementGrouping(sourceTree) {
+  const relabeledTree = sourceTree.map((node) => {
+    if (node.kind === "document" || isEndorsementGroup(node) || node.kind === "endorsement") return node;
+    const title = endorsementTitle([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "));
+    if (!title) return node;
+    return {
+      ...node,
+      kind: "endorsement",
+      title,
+      description: cleanText(
+        [title, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "),
+        title
+      ),
+      metadata: {
+        ...node.metadata,
+        organizerRepair: "normalize_endorsement_grouping"
+      }
+    };
+  });
+  const byParent = nodesByParent(relabeledTree);
+  const groupsByParent = /* @__PURE__ */ new Map();
+  const endorsementGroupIds = new Set(
+    relabeledTree.filter(isEndorsementGroup).map((node) => node.id)
+  );
+  let nextTree = relabeledTree.map((node) => {
+    if (!isEndorsementGroup(node)) return node;
+    const normalized = {
+      ...node,
+      kind: "page_group",
+      title: "Endorsements",
+      description: cleanText(node.description, "Endorsement forms grouped by source order"),
+      metadata: {
+        ...node.metadata,
+        sourceTreeVersion: "v3",
+        organizer: node.metadata?.organizer ?? "endorsement_grouping"
+      }
+    };
+    groupsByParent.set(node.parentId, normalized);
+    endorsementGroupIds.add(node.id);
+    return normalized;
+  });
+  nextTree = nextTree.map((node) => {
+    if (!endorsementGroupIds.has(node.parentId ?? "")) return node;
+    const title = endorsementTitle([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "));
+    if (!title) return node;
+    return {
+      ...node,
+      kind: "endorsement",
+      title,
+      description: cleanText(
+        [title, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "),
+        title
+      ),
+      metadata: {
+        ...node.metadata,
+        organizerRepair: "normalize_endorsement_grouping"
+      }
+    };
+  });
+  for (const [parentId, children] of byParent) {
+    if (endorsementGroupIds.has(parentId ?? "")) continue;
+    const endorsementChildren = children.filter((child) => child.kind === "endorsement" && !isEndorsementGroup(child));
+    if (endorsementChildren.length < 2) continue;
+    const documentId = endorsementChildren[0].documentId;
+    const pageStarts = endorsementChildren.map((child) => child.pageStart).filter((page) => typeof page === "number");
+    const pageEnds = endorsementChildren.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
+    const order = Math.min(...endorsementChildren.map((child) => child.order));
+    const existingGroup = groupsByParent.get(parentId);
+    const groupId = existingGroup?.id ?? endorsementGroupNodeId(documentId, parentId);
+    const groupNode = existingGroup ?? {
+      id: groupId,
+      documentId,
+      parentId,
+      kind: "page_group",
+      title: "Endorsements",
+      description: "Endorsement forms grouped by source order",
+      textExcerpt: void 0,
+      sourceSpanIds: [],
+      pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
+      pageEnd: pageEnds.length ? Math.max(...pageEnds) : void 0,
+      bbox: endorsementChildren.flatMap((child) => child.bbox ?? []).slice(0, 12),
+      order,
+      path: "",
+      metadata: { sourceTreeVersion: "v3", organizer: "endorsement_grouping" }
+    };
+    const childSpanIds = [...new Set(endorsementChildren.flatMap((child) => child.sourceSpanIds))];
+    const normalizedGroup = {
+      ...groupNode,
+      sourceSpanIds: groupNode.sourceSpanIds.length ? groupNode.sourceSpanIds : childSpanIds,
+      pageStart: groupNode.pageStart ?? (pageStarts.length ? Math.min(...pageStarts) : void 0),
+      pageEnd: groupNode.pageEnd ?? (pageEnds.length ? Math.max(...pageEnds) : void 0),
+      order
+    };
+    groupsByParent.set(parentId, normalizedGroup);
+    if (!existingGroup) nextTree.push(normalizedGroup);
+    else nextTree = nextTree.map((node) => node.id === normalizedGroup.id ? normalizedGroup : node);
+    nextTree = nextTree.map(
+      (node) => endorsementChildren.some((child) => child.id === node.id) ? { ...node, parentId: groupId, order: node.order + 1e-3 } : node
+    );
+  }
+  return normalizeDocumentSourceTreePaths(nextTree);
+}
 function compactNode(node, maxText = 700) {
   return {
     id: node.id,
@@ -9175,7 +9440,8 @@ Rules:
 - Use only node IDs from the provided list.
 - Do not invent text, page numbers, source spans, limits, or policy facts.
 - You may relabel existing nodes and group adjacent top-level/page nodes from this batch only when they are clearly one continuous form, one declarations set, one schedule, or one clause family.
-- Do not group separately numbered endorsements into one parent. Never create rollup titles such as "Endorsements 1-3 (...)".
+- Group adjacent separately numbered endorsements under a single generic "Endorsements" page_group parent, with each individual endorsement preserved as its own child node.
+- Never create rollup titles such as "Endorsements 1-3 (...)" or merge multiple endorsements into one endorsement node.
 - Add concise, human-readable titles to generic text, table, row, and cell nodes when the text makes their role clear.
 - Keep organizer titles terse. Use the printed heading or a compact canonical title such as "Declarations", "Policy Form", "Definitions", or "Endorsement No. 3"; do not add parenthetical summaries.
 - Groups must list existing childNodeIds only.
@@ -9270,7 +9536,7 @@ function applyOrganization(sourceTree, organization) {
     ];
     byId.set(id, node);
   }
-  return normalizeDocumentSourceTreePaths(nextTree);
+  return applyEndorsementGrouping(normalizeDocumentSourceTreePaths(nextTree));
 }
 function sourceTreeToOutline(sourceTree) {
   const byParent = /* @__PURE__ */ new Map();
@@ -9449,6 +9715,7 @@ async function runSourceTreeExtraction(params) {
   } catch (error) {
     warnings.push(`Source-tree organizer failed; deterministic tree used (${error instanceof Error ? error.message : String(error)})`);
   }
+  sourceTree = applySemanticPageGrouping(sourceTree);
   const deterministicProfile = buildDeterministicOperationalProfile({
     sourceTree,
     sourceSpans