@claritylabs/cl-sdk 3.0.8 → 3.0.10

package/dist/index.mjs

@@ -9101,9 +9101,21 @@ function looksLikeEndorsementStart(node) {
   if (/^endorsement\s+(?:no\.?|number|#)\s*[A-Z0-9][A-Z0-9.-]*\b/i.test(start)) return true;
   return /^endorsement\s+(?:no\.?|number|#)\s*[A-Z0-9][A-Z0-9.-]*\b/i.test(title) && /\bthis endorsement changes the policy\b/i.test(body);
 }
+function looksLikeEndorsementContinuation(node) {
+  if (looksLikeEndorsementStart(node)) return false;
+  const title = cleanText(node.title, "");
+  const text = sourceNodeText(node);
+  return /\bendorsement\b/i.test(text) || /\bcontinuation\b/i.test(title) || /\ball\s+other\s+terms\s+and\s+conditions\b/i.test(text);
+}
 function endorsementStartTitle(node) {
   return looksLikeEndorsementStart(node) ? endorsementTitle(sourceNodeText(node)) : void 0;
 }
+function endorsementDescription(title, node) {
+  return cleanText(
+    [title, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0].filter(Boolean).join(" | "),
+    title
+  );
+}
 function semanticGroupNodeId(documentId, kind, title, childNodeIds) {
   return [
     documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
@@ -9175,7 +9187,7 @@ function applySemanticPageGrouping(sourceTree) {
         ...node,
         kind: "endorsement",
         title: endorsement,
-        description: cleanText([endorsement, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "), endorsement),
+        description: endorsementDescription(endorsement, node),
         metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
       };
     }
@@ -9261,6 +9273,124 @@ function endorsementGroupNodeId(documentId, parentId) {
     parentId?.replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 48) ?? "root"
   ].join(":");
 }
+function isPolicyFormNode(node) {
+  return node.title === "Policy Form" && (node.kind === "form" || node.kind === "page_group");
+}
+function isDeclarationsNode(node) {
+  return node.kind === "page_group" && node.title === "Declarations";
+}
+function normalizePolicyFormStructure(sourceTree) {
+  let nextTree = sourceTree;
+  const byParent = nodesByParent(nextTree);
+  const nodesToRemove = /* @__PURE__ */ new Set();
+  for (const form of nextTree.filter((node) => node.kind === "form" && node.title === "Policy Form")) {
+    const children = byParent.get(form.id) ?? [];
+    const declarationsChildren = children.filter(isDeclarationsNode);
+    const nestedPolicyForm = children.find((child) => child.id !== form.id && isPolicyFormNode(child));
+    if (declarationsChildren.length === 0 && !nestedPolicyForm) continue;
+    const declarationIds = new Set(declarationsChildren.map((child) => child.id));
+    nextTree = nextTree.map((node) => {
+      if (declarationIds.has(node.id)) {
+        return {
+          ...node,
+          parentId: form.parentId,
+          metadata: {
+            ...node.metadata,
+            organizerRepair: "promote_declarations_from_policy_form"
+          }
+        };
+      }
+      if (nestedPolicyForm && node.parentId === nestedPolicyForm.id) {
+        return {
+          ...node,
+          parentId: form.id,
+          metadata: {
+            ...node.metadata,
+            organizerRepair: "collapse_nested_policy_form"
+          }
+        };
+      }
+      return node;
+    });
+    if (nestedPolicyForm) nodesToRemove.add(nestedPolicyForm.id);
+  }
+  if (nodesToRemove.size > 0) {
+    nextTree = nextTree.filter((node) => !nodesToRemove.has(node.id));
+  }
+  return nextTree;
+}
+function nestEndorsementContinuationPages(sourceTree) {
+  const byParent = nodesByParent(sourceTree);
+  const continuationParentById = /* @__PURE__ */ new Map();
+  for (const group of sourceTree.filter(isEndorsementGroup)) {
+    const children = byParent.get(group.id) ?? [];
+    let currentEndorsement;
+    for (const child of children) {
+      if (child.kind === "endorsement" && endorsementStartTitle(child)) {
+        currentEndorsement = child;
+        continue;
+      }
+      if (!currentEndorsement || child.kind !== "page") continue;
+      continuationParentById.set(child.id, currentEndorsement.id);
+    }
+  }
+  if (continuationParentById.size === 0) return sourceTree;
+  return sourceTree.map((node) => {
+    const parentId = continuationParentById.get(node.id);
+    if (!parentId) return node;
+    return {
+      ...node,
+      parentId,
+      metadata: {
+        ...node.metadata,
+        organizerRepair: "nest_endorsement_continuation"
+      }
+    };
+  });
+}
+function nodeDepth(node) {
+  return node.path ? node.path.split("/").filter(Boolean).length : 0;
+}
+function shouldUseOwnEvidenceForContainer(node) {
+  return node.kind === "endorsement" || node.kind === "page" || node.kind === "table" || node.kind === "table_row" || node.kind === "table_cell" || node.kind === "text";
+}
+function normalizeContainerEvidenceFromChildren(sourceTree) {
+  const byParent = nodesByParent(sourceTree);
+  const byId = new Map(sourceTree.map((node) => [node.id, node]));
+  const sorted = [...sourceTree].sort((left, right) => nodeDepth(right) - nodeDepth(left));
+  for (const originalNode of sorted) {
+    const children = (byParent.get(originalNode.id) ?? []).map((child) => byId.get(child.id)).filter((child) => Boolean(child));
+    if (children.length === 0) continue;
+    const currentNode = byId.get(originalNode.id) ?? originalNode;
+    const evidenceNodes = shouldUseOwnEvidenceForContainer(currentNode) ? [currentNode, ...children] : children;
+    const pageStarts = evidenceNodes.map((node) => node.pageStart).filter((page) => typeof page === "number");
+    const pageEnds = evidenceNodes.map((node) => node.pageEnd ?? node.pageStart).filter((page) => typeof page === "number");
+    const sourceSpanIds = [...new Set(evidenceNodes.flatMap((node) => node.sourceSpanIds))];
+    const bbox = evidenceNodes.flatMap((node) => node.bbox ?? []).slice(0, 12);
+    const childText = children.map((child) => child.textExcerpt ?? child.description).filter(Boolean).join("\n\n").slice(0, 1600);
+    byId.set(currentNode.id, {
+      ...currentNode,
+      sourceSpanIds,
+      pageStart: pageStarts.length ? Math.min(...pageStarts) : currentNode.pageStart,
+      pageEnd: pageEnds.length ? Math.max(...pageEnds) : currentNode.pageEnd,
+      bbox,
+      order: Math.min(currentNode.order, ...children.map((child) => child.order)),
+      textExcerpt: shouldUseOwnEvidenceForContainer(currentNode) ? currentNode.textExcerpt : childText || currentNode.textExcerpt
+    });
+  }
+  return sourceTree.map((node) => byId.get(node.id) ?? node);
+}
+function normalizeSemanticHierarchy(sourceTree) {
+  return normalizeDocumentSourceTreePaths(
+    normalizeContainerEvidenceFromChildren(
+      nestEndorsementContinuationPages(
+        normalizePolicyFormStructure(
+          normalizeDocumentSourceTreePaths(sourceTree)
+        )
+      )
+    )
+  );
+}
 function applyEndorsementGrouping(sourceTree) {
   const relabeledTree = sourceTree.map((node) => {
     if (node.kind === "document" || isEndorsementGroup(node)) return node;
@@ -9281,10 +9411,7 @@ function applyEndorsementGrouping(sourceTree) {
       ...node,
       kind: "endorsement",
       title,
-      description: cleanText(
-        [title, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "),
-        title
-      ),
+      description: endorsementDescription(title, node),
       metadata: {
         ...node.metadata,
         organizerRepair: "normalize_endorsement_grouping"
@@ -9321,10 +9448,7 @@ function applyEndorsementGrouping(sourceTree) {
       ...node,
       kind: "endorsement",
       title,
-      description: cleanText(
-        [title, "endorsement", node.pageStart ? `page ${node.pageStart}` : void 0, node.textExcerpt].filter(Boolean).join(" | "),
-        title
-      ),
+      description: endorsementDescription(title, node),
       metadata: {
         ...node.metadata,
         organizerRepair: "normalize_endorsement_grouping"
@@ -9335,9 +9459,21 @@ function applyEndorsementGrouping(sourceTree) {
     if (endorsementGroupIds.has(parentId ?? "")) continue;
     const endorsementChildren = children.filter((child) => child.kind === "endorsement" && !isEndorsementGroup(child));
     if (endorsementChildren.length < 2) continue;
+    const endorsementGroupChildren = [];
+    let hasSeenEndorsementStart = false;
+    for (const child of children) {
+      if (child.kind === "endorsement" && !isEndorsementGroup(child)) {
+        hasSeenEndorsementStart = true;
+        endorsementGroupChildren.push(child);
+        continue;
+      }
+      if (hasSeenEndorsementStart && child.kind === "page" && looksLikeEndorsementContinuation(child)) {
+        endorsementGroupChildren.push(child);
+      }
+    }
     const documentId = endorsementChildren[0].documentId;
-    const pageStarts = endorsementChildren.map((child) => child.pageStart).filter((page) => typeof page === "number");
-    const pageEnds = endorsementChildren.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
+    const pageStarts = endorsementGroupChildren.map((child) => child.pageStart).filter((page) => typeof page === "number");
+    const pageEnds = endorsementGroupChildren.map((child) => child.pageEnd ?? child.pageStart).filter((page) => typeof page === "number");
     const order = Math.min(...endorsementChildren.map((child) => child.order));
     const existingGroup = groupsByParent.get(parentId);
     const groupId = existingGroup?.id ?? endorsementGroupNodeId(documentId, parentId);
@@ -9352,12 +9488,12 @@ function applyEndorsementGrouping(sourceTree) {
       sourceSpanIds: [],
       pageStart: pageStarts.length ? Math.min(...pageStarts) : void 0,
       pageEnd: pageEnds.length ? Math.max(...pageEnds) : void 0,
-      bbox: endorsementChildren.flatMap((child) => child.bbox ?? []).slice(0, 12),
+      bbox: endorsementGroupChildren.flatMap((child) => child.bbox ?? []).slice(0, 12),
       order,
       path: "",
       metadata: { sourceTreeVersion: "v3", organizer: "endorsement_grouping" }
     };
-    const childSpanIds = [...new Set(endorsementChildren.flatMap((child) => child.sourceSpanIds))];
+    const childSpanIds = [...new Set(endorsementGroupChildren.flatMap((child) => child.sourceSpanIds))];
     const normalizedGroup = {
       ...groupNode,
       sourceSpanIds: groupNode.sourceSpanIds.length ? groupNode.sourceSpanIds : childSpanIds,
@@ -9368,11 +9504,12 @@ function applyEndorsementGrouping(sourceTree) {
     groupsByParent.set(parentId, normalizedGroup);
     if (!existingGroup) nextTree.push(normalizedGroup);
     else nextTree = nextTree.map((node) => node.id === normalizedGroup.id ? normalizedGroup : node);
+    const endorsementGroupChildIds = new Set(endorsementGroupChildren.map((child) => child.id));
     nextTree = nextTree.map(
-      (node) => endorsementChildren.some((child) => child.id === node.id) ? { ...node, parentId: groupId, order: node.order + 1e-3 } : node
+      (node) => endorsementGroupChildIds.has(node.id) ? { ...node, parentId: groupId, order: node.order + 1e-3 } : node
     );
   }
-  return normalizeDocumentSourceTreePaths(nextTree);
+  return normalizeSemanticHierarchy(nextTree);
 }
 function compactNode(node, maxText = 700) {
   return {
@@ -9756,7 +9893,7 @@ async function runSourceTreeExtraction(params) {
         maxTokens: budget.maxTokens,
         taskKind: "extraction_operational_profile",
         budgetDiagnostics: budget,
-        providerOptions: { ...params.providerOptions, sourceSpans, sourceTree }
+        providerOptions: params.providerOptions
       },
       {
         fallback: deterministicProfile,