@claritylabs/cl-sdk 3.0.13 → 3.0.15

package/dist/index.js

@@ -2700,7 +2700,7 @@ function isDiscardableBoilerplate(text, unit) {
 }
 function isBoilerplateLine(line) {
   const cleaned = normalizeWhitespace(line.replace(/\bColumn\s+\d+:\s*/gi, ""));
-  return isDiscardableBoilerplate(cleaned) || /^IMPORTANT NOTICE\s*[-—]\s*/i.test(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
+  return isDiscardableBoilerplate(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
 }
 function removedBoilerplateLines(text) {
   return text.split(/\s{2,}|\r?\n/).map(normalizeWhitespace).filter((line) => line && isBoilerplateLine(line));
@@ -2999,7 +2999,18 @@ function metadataString(metadata, key) {
 }
 function isTitleContentNode(node) {
   if (node.kind !== "text") return false;
-  return metadataString(node.metadata, "elementType") === "title" || metadataString(node.metadata, "sourceUnit") === "title";
+  const isMarkedTitle = metadataString(node.metadata, "elementType") === "title" || metadataString(node.metadata, "sourceUnit") === "title";
+  if (!isMarkedTitle) return false;
+  const text = normalizeWhitespace2(node.textExcerpt ?? node.title);
+  if (!text || text.length > 140) return false;
+  const words = text.split(/\s+/);
+  if (words.length > 14) return false;
+  const startsWithStructuredHeading = /^(section|item|part|coverage part|endorsement|schedule|article)\b|^[A-Z]\.\s|\b[IVX]+\.\s/i.test(text);
+  const uppercaseLetters = [...text].filter((char) => /[A-Z]/.test(char)).length;
+  const lowercaseLetters = [...text].filter((char) => /[a-z]/.test(char)).length;
+  const mostlyUppercase = uppercaseLetters > 0 && uppercaseLetters >= lowercaseLetters * 1.6;
+  const sentenceLike = /\b(is|are|was|were|will|shall|may|must|means|includes|provided|subject|available|attached|remain|constitutes)\b/i.test(text) && /[a-z]/.test(text);
+  return startsWithStructuredHeading || mostlyUppercase && !sentenceLike;
 }
 function nodePageEnd(node) {
   return node.pageEnd ?? node.pageStart;
@@ -3039,10 +3050,10 @@ function groupPageContentByTitles(nodes) {
       const bbox = evidenceNodes.flatMap((node) => node.bbox ?? []).slice(0, 12);
       byId.set(activeTitle.id, {
         ...activeTitle,
-        kind: "section",
+        kind: "text",
         title,
         description: nodeTextDescription({
-          kind: "section",
+          kind: "text",
           title,
           text: evidenceNodes.map((node) => node.textExcerpt).filter(Boolean).join("\n\n"),
           page: activeTitle.pageStart
@@ -9599,6 +9610,7 @@ function looksLikeDeclarationsContinuation(node) {
 function looksLikePolicyFormStart(node) {
   const text = sourceNodeText(node);
   const excerpt = cleanText(node.textExcerpt, "");
+  if (isAdministrativeNoticeNode(node) || looksLikeDeclarationsStart(node)) return false;
   return /\bpolicy form\b/i.test(node.title) || /^policy\s+form\b/i.test(excerpt) || /\btechnology errors?\s*&?\s*omissions\b/i.test(text) && /\bplease read this entire policy carefully\b/i.test(text) || /\bform\s+[A-Z]{2,}-[A-Z0-9-]+\s+\d{2}\s+\d{2}\b/i.test(text);
 }
 function looksLikePolicyFormContinuation(node) {
@@ -9607,9 +9619,9 @@ function looksLikePolicyFormContinuation(node) {
   return /\b(insuring agreement|definitions?|exclusions?|conditions?|claim means|insured means|wrongful act means|limits of liability|notice of claim|cancellation by|action against the company)\b/i.test(text);
 }
 function groupAdjacentChildren(params) {
-  if (params.childIds.length < 2) return params.sourceTree;
+  if (params.childIds.length < 1) return params.sourceTree;
   const children = params.childIds.map((id2) => params.children.find((child) => child.id === id2)).filter((child) => Boolean(child));
-  if (children.length < 2) return params.sourceTree;
+  if (children.length < 1) return params.sourceTree;
   const parentId = children[0].parentId;
   if (!children.every((child) => child.parentId === parentId)) return params.sourceTree;
   const documentId = children[0].documentId;
@@ -9718,6 +9730,11 @@ function applySemanticPageGrouping(sourceTree) {
     for (let index = policyStartIndex; index < children.length; index += 1) {
       const child = children[index];
       if (index > policyStartIndex && looksLikeEndorsementStart(child)) break;
+      if (isAdministrativeNoticeNode(child) || looksLikeDeclarationsStart(child)) break;
+      if (index > policyStartIndex && child.kind === "page") {
+        policyIds.push(child.id);
+        continue;
+      }
       if (!looksLikePolicyFormContinuation(child)) break;
       policyIds.push(child.id);
     }
@@ -9745,6 +9762,9 @@ function rejectsOrganizerGroup(group, children) {
 function isEndorsementGroup(node) {
   return node.kind === "page_group" && /^endorsements?\b/i.test(node.title);
 }
+function isNoticesGroup(node) {
+  return node.kind === "page_group" && /^notices?\s+and\s+jacket$/i.test(node.title);
+}
 function endorsementGroupNodeId(documentId, parentId) {
   return [
     documentId.replace(/[^a-zA-Z0-9_.:-]/g, "_"),
@@ -9760,6 +9780,56 @@ function isPolicyFormNode(node) {
 function isDeclarationsNode(node) {
   return node.kind === "page_group" && node.title === "Declarations";
 }
+function isAdministrativeNoticeNode(node) {
+  const text = sourceNodeText(node);
+  return /\b(specimen policy|policy jacket|important notice|privacy notice|ofac advisory|terrorism risk insurance act|tria|trade or economic sanctions|economic sanctions limitation|signature|countersignature|how to report a claim)\b/i.test(text);
+}
+function mergeAdministrativeNoticesIntoFrontMatter(sourceTree) {
+  const rootId = sourceTreeRootId(sourceTree);
+  if (!rootId) return sourceTree;
+  const children = (nodesByParent(sourceTree).get(rootId) ?? []).filter((node) => node.kind !== "document");
+  const noticesGroup = children.find(isNoticesGroup);
+  if (!noticesGroup) return sourceTree;
+  const noticesGroupChildren = new Set(
+    (nodesByParent(sourceTree).get(noticesGroup.id) ?? []).map((node) => node.id)
+  );
+  const noticeIds = new Set(
+    children.filter(
+      (node) => node.id !== noticesGroup.id && !noticesGroupChildren.has(node.id) && node.kind === "page" && isAdministrativeNoticeNode(node)
+    ).map((node) => node.id)
+  );
+  if (noticeIds.size === 0) return sourceTree;
+  return sourceTree.map(
+    (node) => noticeIds.has(node.id) ? {
+      ...node,
+      parentId: noticesGroup.id,
+      metadata: {
+        ...node.metadata,
+        organizerRepair: "merge_administrative_notice"
+      }
+    } : node
+  );
+}
+function rootSemanticRank(node) {
+  if (isNoticesGroup(node)) return 0;
+  if (node.title === "Declarations") return 1;
+  if (node.title === "Policy Form") return 2;
+  if (isEndorsementGroup(node)) return 3;
+  if (isAdministrativeNoticeNode(node)) return 0.5;
+  return 2.5;
+}
+function normalizeRootSemanticOrder(sourceTree) {
+  const rootId = sourceTreeRootId(sourceTree);
+  if (!rootId) return sourceTree;
+  const rootChildren2 = (nodesByParent(sourceTree).get(rootId) ?? []).filter((node) => node.kind !== "document").sort(
+    (left, right) => rootSemanticRank(left) - rootSemanticRank(right) || (left.pageStart ?? Number.MAX_SAFE_INTEGER) - (right.pageStart ?? Number.MAX_SAFE_INTEGER) || left.order - right.order || left.id.localeCompare(right.id)
+  );
+  const orderById = new Map(rootChildren2.map((node, index) => [node.id, index + 1]));
+  return sourceTree.map((node) => {
+    const order = orderById.get(node.id);
+    return order === void 0 ? node : { ...node, order };
+  });
+}
 function normalizePolicyFormStructure(sourceTree) {
   let nextTree = sourceTree;
   const byParent = nodesByParent(nextTree);
@@ -9868,9 +9938,10 @@ function normalizeSemanticHierarchy(sourceTree) {
     )
   );
   const nested = normalizeDocumentSourceTreePaths(nestEndorsementContinuationPages(normalized));
-  const withEvidence = normalizeContainerEvidenceFromChildren(nested);
+  const mergedNotices = normalizeDocumentSourceTreePaths(mergeAdministrativeNoticesIntoFrontMatter(nested));
+  const withEvidence = normalizeContainerEvidenceFromChildren(mergedNotices);
   return normalizeDocumentSourceTreePaths(
-    normalizeContainerEvidenceFromChildren(withEvidence)
+    normalizeRootSemanticOrder(normalizeContainerEvidenceFromChildren(withEvidence))
   );
 }
 function applyEndorsementGrouping(sourceTree) {