@claritylabs/cl-sdk 3.0.14 → 3.0.16

package/dist/index.js

@@ -2700,7 +2700,7 @@ function isDiscardableBoilerplate(text, unit) {
 }
 function isBoilerplateLine(line) {
   const cleaned = normalizeWhitespace(line.replace(/\bColumn\s+\d+:\s*/gi, ""));
-  return isDiscardableBoilerplate(cleaned) || /^IMPORTANT NOTICE\s*[-—]\s*/i.test(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
+  return isDiscardableBoilerplate(cleaned) || /^THIS IS A CLAIMS-MADE AND REPORTED POLICY\.? PLEASE READ IT CAREFULLY\.?$/i.test(cleaned);
 }
 function removedBoilerplateLines(text) {
   return text.split(/\s{2,}|\r?\n/).map(normalizeWhitespace).filter((line) => line && isBoilerplateLine(line));
@@ -2999,7 +2999,19 @@ function metadataString(metadata, key) {
 }
 function isTitleContentNode(node) {
   if (node.kind !== "text") return false;
-  return metadataString(node.metadata, "elementType") === "title" || metadataString(node.metadata, "sourceUnit") === "title";
+  const isMarkedTitle = metadataString(node.metadata, "elementType") === "title" || metadataString(node.metadata, "sourceUnit") === "title";
+  if (!isMarkedTitle) return false;
+  const text = normalizeWhitespace2(node.textExcerpt ?? node.title);
+  if (!text || text.length > 140) return false;
+  const words = text.split(/\s+/);
+  if (words.length > 14) return false;
+  const startsWithStructuredHeading = /^(section|item|part|endorsement|schedule|article)\b|^[A-Z]\.\s|\b[IVX]+\.\s/i.test(text);
+  const uppercaseLetters = [...text].filter((char) => /[A-Z]/.test(char)).length;
+  const lowercaseLetters = [...text].filter((char) => /[a-z]/.test(char)).length;
+  const mostlyUppercase = uppercaseLetters > 0 && uppercaseLetters >= lowercaseLetters * 1.6;
+  const containsSentencePunctuation = /[.;:]\s+\S/.test(text) || /[.;:]$/.test(text);
+  const sentenceLike = /\b(is|are|was|were|will|shall|may|must|means|includes|provided|subject|available|attached|remain|constitutes)\b/i.test(text) && /[a-z]/.test(text);
+  return startsWithStructuredHeading || mostlyUppercase && !sentenceLike && !containsSentencePunctuation;
 }
 function nodePageEnd(node) {
   return node.pageEnd ?? node.pageStart;
@@ -3039,10 +3051,10 @@ function groupPageContentByTitles(nodes) {
       const bbox = evidenceNodes.flatMap((node) => node.bbox ?? []).slice(0, 12);
       byId.set(activeTitle.id, {
         ...activeTitle,
-        kind: "section",
+        kind: "text",
         title,
         description: nodeTextDescription({
-          kind: "section",
+          kind: "text",
           title,
           text: evidenceNodes.map((node) => node.textExcerpt).filter(Boolean).join("\n\n"),
           page: activeTitle.pageStart
@@ -9563,12 +9575,28 @@ function nodePageEnd2(node) {
   return node.pageEnd ?? node.pageStart;
 }
 function pageRangeForNodes(nodes) {
-  const pageStarts = nodes.map((node) => node.pageStart).filter((page) => typeof page === "number");
-  const pageEnds = nodes.map(nodePageEnd2).filter((page) => typeof page === "number");
-  if (pageStarts.length === 0 || pageEnds.length === 0) return void 0;
-  const start = Math.min(...pageStarts);
-  const end = Math.max(...pageEnds);
-  return start === end ? `page ${start}` : `pages ${start}-${end}`;
+  const pages = [...new Set(nodes.flatMap((node) => {
+    if (typeof node.pageStart !== "number") return [];
+    const end = nodePageEnd2(node) ?? node.pageStart;
+    const values = [];
+    for (let page = node.pageStart; page <= end; page += 1) values.push(page);
+    return values;
+  }))].sort((left, right) => left - right);
+  if (pages.length === 0) return void 0;
+  const ranges = [];
+  let start = pages[0];
+  let previous = pages[0];
+  for (const page of pages.slice(1)) {
+    if (page === previous + 1) {
+      previous = page;
+      continue;
+    }
+    ranges.push(start === previous ? String(start) : `${start}-${previous}`);
+    start = page;
+    previous = page;
+  }
+  ranges.push(start === previous ? String(start) : `${start}-${previous}`);
+  return ranges.length === 1 && !ranges[0].includes("-") ? `page ${ranges[0]}` : `pages ${ranges.join(", ")}`;
 }
 function descriptionWithPages(description, nodes) {
   const range = pageRangeForNodes(nodes);
@@ -9599,6 +9627,7 @@ function looksLikeDeclarationsContinuation(node) {
 function looksLikePolicyFormStart(node) {
   const text = sourceNodeText(node);
   const excerpt = cleanText(node.textExcerpt, "");
+  if (isAdministrativeNoticeNode(node) || looksLikeDeclarationsStart(node)) return false;
   return /\bpolicy form\b/i.test(node.title) || /^policy\s+form\b/i.test(excerpt) || /\btechnology errors?\s*&?\s*omissions\b/i.test(text) && /\bplease read this entire policy carefully\b/i.test(text) || /\bform\s+[A-Z]{2,}-[A-Z0-9-]+\s+\d{2}\s+\d{2}\b/i.test(text);
 }
 function looksLikePolicyFormContinuation(node) {
@@ -9607,9 +9636,9 @@ function looksLikePolicyFormContinuation(node) {
   return /\b(insuring agreement|definitions?|exclusions?|conditions?|claim means|insured means|wrongful act means|limits of liability|notice of claim|cancellation by|action against the company)\b/i.test(text);
 }
 function groupAdjacentChildren(params) {
-  if (params.childIds.length < 2) return params.sourceTree;
+  if (params.childIds.length < 1) return params.sourceTree;
   const children = params.childIds.map((id2) => params.children.find((child) => child.id === id2)).filter((child) => Boolean(child));
-  if (children.length < 2) return params.sourceTree;
+  if (children.length < 1) return params.sourceTree;
   const parentId = children[0].parentId;
   if (!children.every((child) => child.parentId === parentId)) return params.sourceTree;
   const documentId = children[0].documentId;
@@ -9718,6 +9747,11 @@ function applySemanticPageGrouping(sourceTree) {
     for (let index = policyStartIndex; index < children.length; index += 1) {
       const child = children[index];
       if (index > policyStartIndex && looksLikeEndorsementStart(child)) break;
+      if (isAdministrativeNoticeNode(child) || looksLikeDeclarationsStart(child)) break;
+      if (index > policyStartIndex && child.kind === "page") {
+        policyIds.push(child.id);
+        continue;
+      }
       if (!looksLikePolicyFormContinuation(child)) break;
       policyIds.push(child.id);
     }
@@ -9909,6 +9943,10 @@ function normalizeContainerEvidenceFromChildren(sourceTree) {
       pageEnd: pageEnds.length ? Math.max(...pageEnds) : currentNode.pageEnd,
       bbox,
       order: Math.min(currentNode.order, ...children.map((child) => child.order)),
+      description: descriptionWithPages(
+        currentNode.description.replace(/;\s*pages?\s+[0-9,\s-]+$/i, ""),
+        evidenceNodes
+      ),
       textExcerpt: shouldUseOwnEvidenceForContainer(currentNode) ? currentNode.textExcerpt : childText || currentNode.textExcerpt
     });
   }