@claritylabs/cl-sdk 3.0.7 → 3.0.9

package/dist/index.js

@@ -9439,6 +9439,24 @@ function endorsementTitle(value) {
 function sourceNodeText(node) {
   return cleanText([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "), "");
 }
+function looksLikeEndorsementStart(node) {
+  const title = cleanText(node.title, "");
+  const body = cleanText([node.textExcerpt, node.description].filter(Boolean).join(" "), "");
+  const start = body.slice(0, 260);
+  if (/\bthis endorsement changes the policy\b/i.test(start) && endorsementReference(start)) return true;
+  if (/^(?:[A-Z]{2,}-)?END\s+0*[0-9]{1,4}\b/i.test(start)) return true;
+  if (/^endorsement\s+(?:no\.?|number|#)\s*[A-Z0-9][A-Z0-9.-]*\b/i.test(start)) return true;
+  return /^endorsement\s+(?:no\.?|number|#)\s*[A-Z0-9][A-Z0-9.-]*\b/i.test(title) && /\bthis endorsement changes the policy\b/i.test(body);
+}
+function looksLikeEndorsementContinuation(node) {
+  if (looksLikeEndorsementStart(node)) return false;
+  const title = cleanText(node.title, "");
+  const text = sourceNodeText(node);
+  return /\bendorsement\b/i.test(text) || /\bcontinuation\b/i.test(title) || /\ball\s+other\s+terms\s+and\s+conditions\b/i.test(text);
+}
+function endorsementStartTitle(node) {
+  return looksLikeEndorsementStart(node) ? endorsementTitle(sourceNodeText(node)) : void 0;
+}
 function semanticGroupNodeId(documentId, kind, title, childNodeIds) {
   return [
     documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
@@ -9504,8 +9522,7 @@ function groupAdjacentChildren(params) {
 function applySemanticPageGrouping(sourceTree) {
   const relabeled = sourceTree.map((node) => {
     if (node.kind === "document" || node.kind === "page_group") return node;
-    const text = sourceNodeText(node);
-    const endorsement = endorsementTitle(text);
+    const endorsement = endorsementStartTitle(node);
     if (endorsement && node.kind === "page") {
       return {
         ...node,
@@ -9541,7 +9558,7 @@ function applySemanticPageGrouping(sourceTree) {
     const declarationIds = [];
     for (let index = declarationsStartIndex; index < children.length; index += 1) {
       const child = children[index];
-      if (index > declarationsStartIndex && (looksLikePolicyFormStart(child) || endorsementTitle(sourceNodeText(child)))) break;
+      if (index > declarationsStartIndex && (looksLikePolicyFormStart(child) || looksLikeEndorsementStart(child))) break;
       if (!looksLikeDeclarationsContinuation(child)) break;
       declarationIds.push(child.id);
     }
@@ -9560,7 +9577,7 @@ function applySemanticPageGrouping(sourceTree) {
     const policyIds = [];
     for (let index = policyStartIndex; index < children.length; index += 1) {
       const child = children[index];
-      if (index > policyStartIndex && endorsementTitle(sourceNodeText(child))) break;
+      if (index > policyStartIndex && looksLikeEndorsementStart(child)) break;
       if (!looksLikePolicyFormContinuation(child)) break;
       policyIds.push(child.id);
     }
@@ -9597,10 +9614,139 @@ function endorsementGroupNodeId(documentId, parentId) {
     parentId?.replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 48) ?? "root"
   ].join(":");
 }
+function isPolicyFormNode(node) {
+  return node.title === "Policy Form" && (node.kind === "form" || node.kind === "page_group");
+}
+function isDeclarationsNode(node) {
+  return node.kind === "page_group" && node.title === "Declarations";
+}
+function normalizePolicyFormStructure(sourceTree) {
+  let nextTree = sourceTree;
+  const byParent = nodesByParent(nextTree);
+  const nodesToRemove = /* @__PURE__ */ new Set();
+  for (const form of nextTree.filter((node) => node.kind === "form" && node.title === "Policy Form")) {
+    const children = byParent.get(form.id) ?? [];
+    const declarationsChildren = children.filter(isDeclarationsNode);
+    const nestedPolicyForm = children.find((child) => child.id !== form.id && isPolicyFormNode(child));
+    if (declarationsChildren.length === 0 && !nestedPolicyForm) continue;
+    const declarationIds = new Set(declarationsChildren.map((child) => child.id));
+    nextTree = nextTree.map((node) => {
+      if (declarationIds.has(node.id)) {
+        return {
+          ...node,
+          parentId: form.parentId,
+          metadata: {
+            ...node.metadata,
+            organizerRepair: "promote_declarations_from_policy_form"
+          }
+        };
+      }
+      if (nestedPolicyForm && node.parentId === nestedPolicyForm.id) {
+        return {
+          ...node,
+          parentId: form.id,
+          metadata: {
+            ...node.metadata,
+            organizerRepair: "collapse_nested_policy_form"
+          }
+        };
+      }
+      return node;
+    });
+    if (nestedPolicyForm) nodesToRemove.add(nestedPolicyForm.id);
+  }
+  if (nodesToRemove.size > 0) {
+    nextTree = nextTree.filter((node) => !nodesToRemove.has(node.id));
+  }
+  return nextTree;
+}
+function nestEndorsementContinuationPages(sourceTree) {
+  const byParent = nodesByParent(sourceTree);
+  const continuationParentById = /* @__PURE__ */ new Map();
+  for (const group of sourceTree.filter(isEndorsementGroup)) {
+    const children = byParent.get(group.id) ?? [];
+    let currentEndorsement;
+    for (const child of children) {
+      if (child.kind === "endorsement" && endorsementStartTitle(child)) {
+        currentEndorsement = child;
+        continue;
+      }
+      if (!currentEndorsement || child.kind !== "page") continue;
+      continuationParentById.set(child.id, currentEndorsement.id);
+    }
+  }
+  if (continuationParentById.size === 0) return sourceTree;
+  return sourceTree.map((node) => {
+    const parentId = continuationParentById.get(node.id);
+    if (!parentId) return node;
+    return {
+      ...node,
+      parentId,
+      metadata: {
+        ...node.metadata,
+        organizerRepair: "nest_endorsement_continuation"
+      }
+    };
+  });
+}
+function nodeDepth(node) {
+  return node.path ? node.path.split("/").filter(Boolean).length : 0;
+}
+function shouldUseOwnEvidenceForContainer(node) {
+  return node.kind === "endorsement" || node.kind === "page" || node.kind === "table" || node.kind === "table_row" || node.kind === "table_cell" || node.kind === "text";
+}
+function normalizeContainerEvidenceFromChildren(sourceTree) {
+  const byParent = nodesByParent(sourceTree);
+  const byId = new Map(sourceTree.map((node) => [node.id, node]));
+  const sorted = [...sourceTree].sort((left, right) => nodeDepth(right) - nodeDepth(left));
+  for (const originalNode of sorted) {
+    const children = (byParent.get(originalNode.id) ?? []).map((child) => byId.get(child.id)).filter((child) => Boolean(child));
+    if (children.length === 0) continue;
+    const currentNode = byId.get(originalNode.id) ?? originalNode;
+    const evidenceNodes = shouldUseOwnEvidenceForContainer(currentNode) ? [currentNode, ...children] : children;
+    const pageStarts = evidenceNodes.map((node) => node.pageStart).filter((page) => typeof page === "number");
+    const pageEnds = evidenceNodes.map((node) => node.pageEnd ?? node.pageStart).filter((page) => typeof page === "number");
+    const sourceSpanIds = [...new Set(evidenceNodes.flatMap((node) => node.sourceSpanIds))];
+    const bbox = evidenceNodes.flatMap((node) => node.bbox ?? []).slice(0, 12);
+    const childText = children.map((child) => child.textExcerpt ?? child.description).filter(Boolean).join("\n\n").slice(0, 1600);
+    byId.set(currentNode.id, {
+      ...currentNode,
+      sourceSpanIds,
+      pageStart: pageStarts.length ? Math.min(...pageStarts) : currentNode.pageStart,
+      pageEnd: pageEnds.length ? Math.max(...pageEnds) : currentNode.pageEnd,
+      bbox,
+      order: Math.min(currentNode.order, ...children.map((child) => child.order)),
+      textExcerpt: shouldUseOwnEvidenceForContainer(currentNode) ? currentNode.textExcerpt : childText || currentNode.textExcerpt
+    });
+  }
+  return sourceTree.map((node) => byId.get(node.id) ?? node);
+}
+function normalizeSemanticHierarchy(sourceTree) {
+  return normalizeDocumentSourceTreePaths(
+    normalizeContainerEvidenceFromChildren(
+      nestEndorsementContinuationPages(
+        normalizePolicyFormStructure(
+          normalizeDocumentSourceTreePaths(sourceTree)
+        )
+      )
+    )
+  );
+}
 function applyEndorsementGrouping(sourceTree) {
   const relabeledTree = sourceTree.map((node) => {
-    if (node.kind === "document" || isEndorsementGroup(node) || node.kind === "endorsement") return node;
-    const title = endorsementTitle([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "));
+    if (node.kind === "document" || isEndorsementGroup(node)) return node;
+    const title = endorsementStartTitle(node);
+    if (!title && node.kind === "endorsement") {
+      return {
+        ...node,
+        kind: "page",
+        title: node.pageStart ? `Page ${node.pageStart}` : cleanText(node.title, "Page"),
+        metadata: {
+          ...node.metadata,
+          organizerRepair: "demote_incidental_endorsement_reference"
+        }
+      };
+    }
     if (!title) return node;
     return {
       ...node,
@@ -9640,7 +9786,7 @@ function applyEndorsementGrouping(sourceTree) {
   });
   nextTree = nextTree.map((node) => {
     if (!endorsementGroupIds.has(node.parentId ?? "")) return node;
-    const title = endorsementTitle([node.title, node.description, node.textExcerpt].filter(Boolean).join(" "));
+    const title = endorsementStartTitle(node);
     if (!title) return node;
     return {
       ...node,
@@ -9660,9 +9806,21 @@ function applyEndorsementGrouping(sourceTree) {
     if (endorsementGroupIds.has(parentId ?? "")) continue;
     const endorsementChildren = children.filter((child) => child.kind === "endorsement" && !isEndorsementGroup(child));
     if (endorsementChildren.length < 2) continue;
+    const endorsementGroupChildren = [];
+    let hasSeenEndorsementStart = false;
+    for (const child of children) {
+      if (child.kind === "endorsement" && !isEndorsementGroup(child)) {
+        hasSeenEndorsementStart = true;
+        endorsementGroupChildren.push(child);
+        continue;
+      }
+      if (hasSeenEndorsementStart && child.kind === "page" && looksLikeEndorsementContinuation(child)) {
+        endorsementGroupChildren.push(child);
+      }
+    }
     const documentId = endorsementChildren[0].documentId;
-    const pageStarts = endorsementChildren.map((child) => child.pageStart).filter((page) => typeof page === "number");
-    const pageEnds = endorsementChildren.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
+    const pageStarts = endorsementGroupChildren.map((child) => child.pageStart).filter((page) => typeof page === "number");
+    const pageEnds = endorsementGroupChildren.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
     const order = Math.min(...endorsementChildren.map((child) => child.order));
     const existingGroup = groupsByParent.get(parentId);
     const groupId = existingGroup?.id ?? endorsementGroupNodeId(documentId, parentId);
@@ -9677,12 +9835,12 @@ function applyEndorsementGrouping(sourceTree) {
       sourceSpanIds: [],
       pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
       pageEnd: pageEnds.length ? Math.max(...pageEnds) : void 0,
-      bbox: endorsementChildren.flatMap((child) => child.bbox ?? []).slice(0, 12),
+      bbox: endorsementGroupChildren.flatMap((child) => child.bbox ?? []).slice(0, 12),
       order,
       path: "",
       metadata: { sourceTreeVersion: "v3", organizer: "endorsement_grouping" }
     };
-    const childSpanIds = [...new Set(endorsementChildren.flatMap((child) => child.sourceSpanIds))];
+    const childSpanIds = [...new Set(endorsementGroupChildren.flatMap((child) => child.sourceSpanIds))];
     const normalizedGroup = {
       ...groupNode,
       sourceSpanIds: groupNode.sourceSpanIds.length ? groupNode.sourceSpanIds : childSpanIds,
@@ -9693,11 +9851,12 @@ function applyEndorsementGrouping(sourceTree) {
     groupsByParent.set(parentId, normalizedGroup);
     if (!existingGroup) nextTree.push(normalizedGroup);
     else nextTree = nextTree.map((node) => node.id === normalizedGroup.id ? normalizedGroup : node);
+    const endorsementGroupChildIds = new Set(endorsementGroupChildren.map((child) => child.id));
     nextTree = nextTree.map(
-      (node) => endorsementChildren.some((child) => child.id === node.id) ? { ...node, parentId: groupId, order: node.order + 1e-3 } : node
+      (node) => endorsementGroupChildIds.has(node.id) ? { ...node, parentId: groupId, order: node.order + 1e-3 } : node
     );
   }
-  return normalizeDocumentSourceTreePaths(nextTree);
+  return normalizeSemanticHierarchy(nextTree);
 }
 function compactNode(node, maxText = 700) {
   return {