@claritylabs/cl-sdk 3.0.15 → 3.0.16

package/dist/index.js

@@ -3005,12 +3005,13 @@ function isTitleContentNode(node) {
   if (!text || text.length > 140) return false;
   const words = text.split(/\s+/);
   if (words.length > 14) return false;
-  const startsWithStructuredHeading = /^(section|item|part|coverage part|endorsement|schedule|article)\b|^[A-Z]\.\s|\b[IVX]+\.\s/i.test(text);
+  const startsWithStructuredHeading = /^(section|item|part|endorsement|schedule|article)\b|^[A-Z]\.\s|\b[IVX]+\.\s/i.test(text);
   const uppercaseLetters = [...text].filter((char) => /[A-Z]/.test(char)).length;
   const lowercaseLetters = [...text].filter((char) => /[a-z]/.test(char)).length;
   const mostlyUppercase = uppercaseLetters > 0 && uppercaseLetters >= lowercaseLetters * 1.6;
+  const containsSentencePunctuation = /[.;:]\s+\S/.test(text) || /[.;:]$/.test(text);
   const sentenceLike = /\b(is|are|was|were|will|shall|may|must|means|includes|provided|subject|available|attached|remain|constitutes)\b/i.test(text) && /[a-z]/.test(text);
-  return startsWithStructuredHeading || mostlyUppercase && !sentenceLike;
+  return startsWithStructuredHeading || mostlyUppercase && !sentenceLike && !containsSentencePunctuation;
 }
 function nodePageEnd(node) {
   return node.pageEnd ?? node.pageStart;
@@ -9574,12 +9575,28 @@ function nodePageEnd2(node) {
   return node.pageEnd ?? node.pageStart;
 }
 function pageRangeForNodes(nodes) {
-  const pageStarts = nodes.map((node) => node.pageStart).filter((page) => typeof page === "number");
-  const pageEnds = nodes.map(nodePageEnd2).filter((page) => typeof page === "number");
-  if (pageStarts.length === 0 || pageEnds.length === 0) return void 0;
-  const start = Math.min(...pageStarts);
-  const end = Math.max(...pageEnds);
-  return start === end ? `page ${start}` : `pages ${start}-${end}`;
+  const pages = [...new Set(nodes.flatMap((node) => {
+    if (typeof node.pageStart !== "number") return [];
+    const end = nodePageEnd2(node) ?? node.pageStart;
+    const values = [];
+    for (let page = node.pageStart; page <= end; page += 1) values.push(page);
+    return values;
+  }))].sort((left, right) => left - right);
+  if (pages.length === 0) return void 0;
+  const ranges = [];
+  let start = pages[0];
+  let previous = pages[0];
+  for (const page of pages.slice(1)) {
+    if (page === previous + 1) {
+      previous = page;
+      continue;
+    }
+    ranges.push(start === previous ? String(start) : `${start}-${previous}`);
+    start = page;
+    previous = page;
+  }
+  ranges.push(start === previous ? String(start) : `${start}-${previous}`);
+  return ranges.length === 1 && !ranges[0].includes("-") ? `page ${ranges[0]}` : `pages ${ranges.join(", ")}`;
 }
 function descriptionWithPages(description, nodes) {
   const range = pageRangeForNodes(nodes);
@@ -9926,6 +9943,10 @@ function normalizeContainerEvidenceFromChildren(sourceTree) {
       pageEnd: pageEnds.length ? Math.max(...pageEnds) : currentNode.pageEnd,
       bbox,
       order: Math.min(currentNode.order, ...children.map((child) => child.order)),
+      description: descriptionWithPages(
+        currentNode.description.replace(/;\s*pages?\s+[0-9,\s-]+$/i, ""),
+        evidenceNodes
+      ),
       textExcerpt: shouldUseOwnEvidenceForContainer(currentNode) ? currentNode.textExcerpt : childText || currentNode.textExcerpt
     });
   }