@claritylabs/cl-sdk 3.0.19 → 3.0.21

package/dist/index.mjs

@@ -6757,6 +6757,8 @@ For EACH form, extract:
 
 Critical rules:
 - Include declarations page sets even if they do not show a standard form number.
+- Do not classify policy jackets, claim-reporting notices, privacy notices, OFAC notices, terrorism/TRIA notices, sanctions notices, signatures, countersignatures, or marketing/admin pages as declarations.
+- Declarations pages contain policy-specific schedules such as named insured, policy number, policy period, premium, limits, retentions, coverage parts, or forms-and-endorsements schedules.
 - Use original document page numbers, not local chunk page numbers.
 - Do not emit duplicate entries for repeated headers/footers.
 - Multi-page forms should be represented once with pageStart/pageEnd covering the full span when visible.
@@ -8890,6 +8892,17 @@ function buildTemplateHints(primaryType, documentType, pageCount, template) {
     `Total pages: ${pageCount}`
   ].join("\n");
 }
+function buildFormInventoryHints(primaryType, documentType, pageCount, template) {
+  return [
+    `Document type: ${primaryType} ${documentType}`,
+    `Expected sections: ${template.expectedSections.join(", ")}`,
+    "Expected source order: jacket/front matter and notices, declarations, policy or coverage form, endorsements/amendatory forms.",
+    "Front matter/notices include policy jackets, claim-reporting notices, privacy notices, OFAC notices, terrorism/TRIA notices, sanctions notices, signatures, countersignatures, and marketing/admin pages.",
+    "Declarations include named insured, policy number, policy period, premium, coverage limits, retentions, forms-and-endorsements schedules, and similar schedule rows.",
+    "Do not classify a page as declarations merely because it appears early in the document.",
+    `Total pages: ${pageCount}`
+  ].join("\n");
+}
 function groupContiguousPages(pages) {
   if (pages.length === 0) return [];
   const sorted = [...new Set(pages)].sort((a, b) => a - b);
@@ -9311,6 +9324,7 @@ function pageTitleFromText(text, fallback) {
   const endorsement = normalized.match(/\bENDORSEMENT\s+(?:NO\.?|NUMBER|#)\s*[A-Z0-9][A-Z0-9.-]*\b/i)?.[0];
   if (endorsement) return cleanText(endorsement, fallback);
   const firstSentence = normalized.split(/(?<=\.)\s+/)[0];
+  if (/^page\s+\d+\b/i.test(firstSentence)) return fallback;
   if (firstSentence && firstSentence.length <= 120) return firstSentence.replace(/[.]$/, "");
   return fallback;
 }
@@ -9321,7 +9335,16 @@ function pageFormTypeFromText(text) {
   if (/\b(important notice|privacy notice|ofac advisory|terrorism risk insurance act|tria|trade or economic sanctions)\b/i.test(text)) return "notice";
   return "other";
 }
-function inferFormHintsFromSourceSpans(sourceSpans) {
+function administrativeFormTypeFromText(text) {
+  if (/\b(important notice|privacy notice|ofac advisory|terrorism risk insurance act|tria|trade or economic sanctions|economic sanctions limitation|how to report a claim)\b/i.test(text)) {
+    return "notice";
+  }
+  if (/\b(specimen policy|policy jacket|countersigned|countersignature|licensed resident agent|corporate secretary|president and ceo|application of insurance executed)\b/i.test(text)) {
+    return "other";
+  }
+  return void 0;
+}
+function pageTextByNumber(sourceSpans) {
   const pageTexts = /* @__PURE__ */ new Map();
   const pageSpanTexts = /* @__PURE__ */ new Map();
   for (const span of sourceSpans) {
@@ -9337,9 +9360,28 @@ function inferFormHintsFromSourceSpans(sourceSpans) {
       if (existing.length < 4e3) pageTexts.set(page, cleanText([existing, span.text].filter(Boolean).join(" "), ""));
     }
   }
-  if (pageSpanTexts.size === 0) return [];
-  const pageHints = [.../* @__PURE__ */ new Set([...pageTexts.keys(), ...pageSpanTexts.keys()])].sort((left, right) => left - right).map((page) => {
-    const text = pageSpanTexts.get(page) ?? pageTexts.get(page) ?? "";
+  return new Map([.../* @__PURE__ */ new Set([...pageTexts.keys(), ...pageSpanTexts.keys()])].map((page) => [
+    page,
+    pageSpanTexts.get(page) ?? pageTexts.get(page) ?? ""
+  ]));
+}
+function reconcileFormTypeWithSourceText(form, pageTexts) {
+  const pages = [];
+  const start = form.pageStart;
+  const end = form.pageEnd ?? start;
+  if (typeof start === "number" && typeof end === "number") {
+    for (let page = start; page <= end; page += 1) pages.push(pageTexts.get(page) ?? "");
+  }
+  const text = cleanText([form.title, form.formNumber, ...pages].filter(Boolean).join(" "), "");
+  const administrativeType = administrativeFormTypeFromText(text);
+  if (administrativeType && form.formType !== administrativeType) return administrativeType;
+  return form.formType;
+}
+function inferFormHintsFromSourceSpans(sourceSpans) {
+  const pageTexts = pageTextByNumber(sourceSpans);
+  if (pageTexts.size === 0) return [];
+  const pageHints = [...pageTexts.keys()].sort((left, right) => left - right).map((page) => {
+    const text = pageTexts.get(page) ?? "";
     const formNumber = formNumberFromText(text);
     return {
       formNumber,
@@ -9367,10 +9409,12 @@ function inferFormHintsFromSourceSpans(sourceSpans) {
   return merged;
 }
 function normalizeFormHints(forms, sourceSpans) {
+  const pageTexts = pageTextByNumber(sourceSpans);
   const provided = (forms ?? []).filter(
     (form) => typeof form.pageStart === "number" && typeof form.pageEnd === "number" && form.pageStart > 0 && form.pageEnd >= form.pageStart
   ).map((form) => ({
     ...form,
+    formType: reconcileFormTypeWithSourceText(form, pageTexts),
     title: form.title ? cleanText(form.title, "") : void 0
   })).sort(
     (left, right) => (left.pageStart ?? Number.MAX_SAFE_INTEGER) - (right.pageStart ?? Number.MAX_SAFE_INTEGER) || (left.pageEnd ?? Number.MAX_SAFE_INTEGER) - (right.pageEnd ?? Number.MAX_SAFE_INTEGER)
@@ -9545,33 +9589,44 @@ function reparentNodes(sourceTree, childIds, parentId, organizerRepair) {
 function applySemanticPageGrouping(sourceTree) {
   const relabeled = sourceTree.map((node) => {
     if (node.kind === "document" || node.kind === "page_group") return node;
-    const endorsement = endorsementStartTitle(node);
-    if (endorsement && node.kind === "page") {
+    let nextNode = node;
+    if (node.kind === "page" && /^page\s+\d+$/i.test(node.title)) {
+      const title = pageTitleFromText(sourceNodeText(node), node.title);
+      if (title !== node.title) {
+        nextNode = {
+          ...node,
+          title: simplifyOrganizerTitle(title, title, node.kind),
+          metadata: { ...node.metadata, organizerRepair: "semantic_page_title" }
+        };
+      }
+    }
+    const endorsement = endorsementStartTitle(nextNode);
+    if (endorsement && nextNode.kind === "page") {
       return {
-        ...node,
+        ...nextNode,
         kind: "endorsement",
         title: endorsement,
-        description: endorsementDescription(endorsement, node),
-        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+        description: endorsementDescription(endorsement, nextNode),
+        metadata: { ...nextNode.metadata, organizerRepair: "semantic_page_grouping" }
       };
     }
-    if (node.kind === "page" && looksLikeDeclarationsStart(node)) {
+    if (nextNode.kind === "page" && looksLikeDeclarationsStart(nextNode)) {
       return {
-        ...node,
+        ...nextNode,
         title: "Declarations",
-        description: cleanText([node.description, "Declarations"].join(" "), "Declarations"),
-        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+        description: cleanText([nextNode.description, "Declarations"].join(" "), "Declarations"),
+        metadata: { ...nextNode.metadata, organizerRepair: "semantic_page_grouping" }
       };
     }
-    if (node.kind === "page" && looksLikePolicyFormStart(node)) {
+    if (nextNode.kind === "page" && looksLikePolicyFormStart(nextNode)) {
       return {
-        ...node,
+        ...nextNode,
         title: "Policy Form",
-        description: cleanText([node.description, "Policy Form"].join(" "), "Policy Form"),
-        metadata: { ...node.metadata, organizerRepair: "semantic_page_grouping" }
+        description: cleanText([nextNode.description, "Policy Form"].join(" "), "Policy Form"),
+        metadata: { ...nextNode.metadata, organizerRepair: "semantic_page_grouping" }
       };
     }
-    return node;
+    return nextNode;
   });
   const rootId = sourceTreeRootId(relabeled);
   const children = (nodesByParent(relabeled).get(rootId) ?? []).filter((node) => node.kind !== "document").sort((left, right) => left.order - right.order);
@@ -10970,7 +11025,7 @@ ${span.text}` : span.text;
         onProgress?.("Building form inventory from source spans...");
         const budget = resolveBudget("extraction_form_inventory", 2048);
         const startedAt = Date.now();
-        const templateHints2 = buildTemplateHints("other", "policy", pageCount2, getTemplate("other"));
+        const templateHints2 = buildFormInventoryHints("other", "policy", pageCount2, getTemplate("other"));
         const sourceText = formatSourceSpanText(sourceSpans);
         const prompt = `${buildFormInventoryPrompt(templateHints2)}
 
@@ -11233,7 +11288,9 @@ ${pageText}`;
       const formInventoryResponse = await safeGenerateObject(
         generateObject,
         {
-          prompt: withFullDocumentTextContext(buildFormInventoryPrompt(templateHints)),
+          prompt: withFullDocumentTextContext(buildFormInventoryPrompt(
+            buildFormInventoryHints(primaryType, documentType, pageCount, template)
+          )),
           schema: FormInventorySchema,
           maxTokens: budget.maxTokens,
           taskKind: "extraction_form_inventory",