@claritylabs/cl-sdk 3.0.14 → 3.0.15

package/dist/index.js

@@ -2700,7 +2700,7 @@ function isDiscardableBoilerplate(text, unit) {
 }
 function isBoilerplateLine(line) {
   const cleaned = normalizeWhitespace(line.replace(/\bColumn\s+\d+:\s*/gi, ""));
-  return isDiscardableBoilerplate(cleaned) || /^IMPORTANT NOTICE\s*[-—]\s*/i.test(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
+  return isDiscardableBoilerplate(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
 }
 function removedBoilerplateLines(text) {
   return text.split(/\s{2,}|\r?\n/).map(normalizeWhitespace).filter((line) => line && isBoilerplateLine(line));
@@ -2999,7 +2999,18 @@ function metadataString(metadata, key) {
 }
 function isTitleContentNode(node) {
   if (node.kind !== "text") return false;
-  return metadataString(node.metadata, "elementType") === "title" || metadataString(node.metadata, "sourceUnit") === "title";
+  const isMarkedTitle = metadataString(node.metadata, "elementType") === "title" || metadataString(node.metadata, "sourceUnit") === "title";
+  if (!isMarkedTitle) return false;
+  const text = normalizeWhitespace2(node.textExcerpt ?? node.title);
+  if (!text || text.length > 140) return false;
+  const words = text.split(/\s+/);
+  if (words.length > 14) return false;
+  const startsWithStructuredHeading = /^(section|item|part|coverage part|endorsement|schedule|article)\b|^[A-Z]\.\s|\b[IVX]+\.\s/i.test(text);
+  const uppercaseLetters = [...text].filter((char) => /[A-Z]/.test(char)).length;
+  const lowercaseLetters = [...text].filter((char) => /[a-z]/.test(char)).length;
+  const mostlyUppercase = uppercaseLetters > 0 && uppercaseLetters >= lowercaseLetters * 1.6;
+  const sentenceLike = /\b(is|are|was|were|will|shall|may|must|means|includes|provided|subject|available|attached|remain|constitutes)\b/i.test(text) && /[a-z]/.test(text);
+  return startsWithStructuredHeading || mostlyUppercase && !sentenceLike;
 }
 function nodePageEnd(node) {
   return node.pageEnd ?? node.pageStart;
@@ -3039,10 +3050,10 @@ function groupPageContentByTitles(nodes) {
       const bbox = evidenceNodes.flatMap((node) => node.bbox ?? []).slice(0, 12);
       byId.set(activeTitle.id, {
         ...activeTitle,
-        kind: "section",
+        kind: "text",
         title,
         description: nodeTextDescription({
-          kind: "section",
+          kind: "text",
           title,
           text: evidenceNodes.map((node) => node.textExcerpt).filter(Boolean).join("\n\n"),
           page: activeTitle.pageStart
@@ -9599,6 +9610,7 @@ function looksLikeDeclarationsContinuation(node) {
 function looksLikePolicyFormStart(node) {
   const text = sourceNodeText(node);
   const excerpt = cleanText(node.textExcerpt, "");
+  if (isAdministrativeNoticeNode(node) || looksLikeDeclarationsStart(node)) return false;
   return /\bpolicy form\b/i.test(node.title) || /^policy\s+form\b/i.test(excerpt) || /\btechnology errors?\s*&?\s*omissions\b/i.test(text) && /\bplease read this entire policy carefully\b/i.test(text) || /\bform\s+[A-Z]{2,}-[A-Z0-9-]+\s+\d{2}\s+\d{2}\b/i.test(text);
 }
 function looksLikePolicyFormContinuation(node) {
@@ -9607,9 +9619,9 @@ function looksLikePolicyFormContinuation(node) {
   return /\b(insuring agreement|definitions?|exclusions?|conditions?|claim means|insured means|wrongful act means|limits of liability|notice of claim|cancellation by|action against the company)\b/i.test(text);
 }
 function groupAdjacentChildren(params) {
-  if (params.childIds.length < 2) return params.sourceTree;
+  if (params.childIds.length < 1) return params.sourceTree;
   const children = params.childIds.map((id2) => params.children.find((child) => child.id === id2)).filter((child) => Boolean(child));
-  if (children.length < 2) return params.sourceTree;
+  if (children.length < 1) return params.sourceTree;
   const parentId = children[0].parentId;
   if (!children.every((child) => child.parentId === parentId)) return params.sourceTree;
   const documentId = children[0].documentId;
@@ -9718,6 +9730,11 @@ function applySemanticPageGrouping(sourceTree) {
     for (let index = policyStartIndex; index < children.length; index += 1) {
       const child = children[index];
       if (index > policyStartIndex && looksLikeEndorsementStart(child)) break;
+      if (isAdministrativeNoticeNode(child) || looksLikeDeclarationsStart(child)) break;
+      if (index > policyStartIndex && child.kind === "page") {
+        policyIds.push(child.id);
+        continue;
+      }
       if (!looksLikePolicyFormContinuation(child)) break;
       policyIds.push(child.id);
     }