@claritylabs/cl-sdk 1.3.2 → 1.3.3

package/dist/index.js

@@ -8185,7 +8185,7 @@ function normalizePageAssignments(pageAssignments, formInventory) {
   }
   return pageAssignments.map((assignment) => {
     let extractorNames = [...new Set(
-      (assignment.extractorNames.length > 0 ? assignment.extractorNames : ["sections"]).filter(Boolean)
+      assignment.extractorNames.filter(Boolean)
     )];
     const hasDeclarations = extractorNames.includes("declarations");
     const hasConditions = extractorNames.includes("conditions");
@@ -8204,9 +8204,6 @@ function normalizePageAssignments(pageAssignments, formInventory) {
     if (inventoryTypes?.has("endorsement") && !extractorNames.includes("endorsements")) {
       extractorNames = [...extractorNames, "endorsements"];
     }
-    if (extractorNames.length === 0) {
-      extractorNames = ["sections"];
-    }
     return {
       ...assignment,
       extractorNames
@@ -8243,20 +8240,12 @@ function groupContiguousPages(pages) {
 function buildPlanFromPageAssignments(pageAssignments, pageCount, formInventory) {
   const extractorPages = /* @__PURE__ */ new Map();
   for (const assignment of pageAssignments) {
-    const extractors = assignment.extractorNames.length > 0 ? assignment.extractorNames : ["sections"];
+    const focusedExtractors = assignment.extractorNames.filter((name) => name !== "sections");
+    const extractors = focusedExtractors.length > 0 ? focusedExtractors : assignment.extractorNames;
     for (const extractorName of extractors) {
       extractorPages.set(extractorName, [...extractorPages.get(extractorName) ?? [], assignment.localPageNumber]);
     }
   }
-  const coveredPages = /* @__PURE__ */ new Set();
-  for (const pages of extractorPages.values()) {
-    for (const page of pages) coveredPages.add(page);
-  }
-  for (let page = 1; page <= pageCount; page += 1) {
-    if (!coveredPages.has(page)) {
-      extractorPages.set("sections", [...extractorPages.get("sections") ?? [], page]);
-    }
-  }
   const contextualExtractors = /* @__PURE__ */ new Set(["conditions", "covered_reasons", "definitions", "exclusions", "endorsements"]);
   const contextualForms = (formInventory?.forms ?? []).filter(
     (form) => form.pageStart != null && (form.pageEnd ?? form.pageStart) != null
@@ -8543,6 +8532,139 @@ function createExtractor(config) {
     ranges.push({ startPage, endPage: previousPage });
     return ranges;
   }
+  function pageNumberForSpan(span) {
+    return span.pageStart ?? span.location?.startPage ?? span.location?.page;
+  }
+  function spansForPageRange(spans, startPage, endPage) {
+    return spans.filter((span) => {
+      const start = span.pageStart ?? span.location?.startPage ?? span.location?.page;
+      const end = span.pageEnd ?? span.location?.endPage ?? start;
+      return typeof start === "number" && typeof end === "number" && start <= endPage && end >= startPage;
+    });
+  }
+  function inferSectionType(title, text) {
+    const value = `${title} ${text.slice(0, 500)}`.toLowerCase();
+    if (/\bdefinition|defined terms?\b/.test(value)) return "definition";
+    if (/\bexclusion|not covered|does not apply\b/.test(value)) return "exclusion";
+    if (/\bcondition|duties|loss condition|general condition\b/.test(value)) return "condition";
+    if (/\bendorsement|amend|additional insured\b/.test(value)) return "endorsement";
+    if (/\bcovered cause|covered reason|covered peril|cause of loss|perils insured\b/.test(value)) return "covered_reason";
+    if (/\bdeclaration|schedule\b/.test(value)) return "declarations";
+    return "policy_form";
+  }
+  function buildSourceBackedSectionIndex(spans, startPage, endPage) {
+    const candidateSpans = spansForPageRange(spans, startPage, endPage).filter((span) => span.text.trim().length > 0).filter((span) => span.metadata?.sourceUnit === "section_candidate" || span.sectionId || span.text.length >= 160);
+    return {
+      sections: candidateSpans.map((span, index) => {
+        const pageStart = span.pageStart ?? span.location?.startPage ?? span.location?.page ?? startPage;
+        const pageEnd = span.pageEnd ?? span.location?.endPage ?? pageStart;
+        const title = span.sectionId ?? span.formNumber ?? firstHeadingLine(span.text) ?? `Policy text page ${pageStart}`;
+        return {
+          title,
+          sectionNumber: span.formNumber,
+          pageStart,
+          pageEnd,
+          type: inferSectionType(title, span.text),
+          excerpt: span.text.slice(0, 240),
+          recordId: `section_index_${pageStart}_${index}`,
+          sourceSpanIds: [span.id],
+          sourceTextHash: span.textHash ?? sourceSpanTextHash(span.text)
+        };
+      })
+    };
+  }
+  function firstHeadingLine(text) {
+    const line = text.split(/\r?\n/).map((item) => item.trim()).find((item) => item.length > 0);
+    if (!line) return void 0;
+    return line.slice(0, 100);
+  }
+  function buildDeterministicFormInventory(spans, pageCount) {
+    if (spans.length === 0) return void 0;
+    const formPattern = /\b([A-Z]{1,4}\s?\d{2,5}(?:\s?\d{2,4})?|[A-Z]{2,8}-\d{2,8})\b/g;
+    const seen = /* @__PURE__ */ new Map();
+    for (const span of spans) {
+      const page = pageNumberForSpan(span);
+      if (!page) continue;
+      const title = firstHeadingLine(span.text) ?? span.sectionId;
+      const matches = /* @__PURE__ */ new Set([
+        ...span.formNumber ? [span.formNumber] : [],
+        ...Array.from(span.text.slice(0, 1200).matchAll(formPattern), (match) => match[1].trim())
+      ]);
+      for (const formNumber of matches) {
+        const key = `${formNumber}:${title ?? ""}`;
+        const existing = seen.get(key);
+        const formType = inferFormType(title ?? "", span.text);
+        if (existing) {
+          existing.pageStart = Math.min(existing.pageStart ?? page, page);
+          existing.pageEnd = Math.max(existing.pageEnd ?? page, page);
+          continue;
+        }
+        seen.set(key, {
+          formNumber,
+          title,
+          formType,
+          pageStart: Math.min(page, pageCount),
+          pageEnd: Math.min(page, pageCount)
+        });
+      }
+    }
+    const forms = [...seen.values()].sort((a, b) => (a.pageStart ?? 0) - (b.pageStart ?? 0));
+    return forms.length > 0 ? { forms } : void 0;
+  }
+  function inferFormType(title, text) {
+    const value = `${title} ${text.slice(0, 700)}`.toLowerCase();
+    if (/\bdeclarations?|schedule\b/.test(value)) return "declarations";
+    if (/\bendorsement|amendatory|additional insured\b/.test(value)) return "endorsement";
+    if (/\bnotice|department of insurance|complaint|privacy\b/.test(value)) return "notice";
+    if (/\bapplication|questionnaire\b/.test(value)) return "application";
+    if (/\bcoverage|policy form|conditions|exclusions|definitions|causes of loss\b/.test(value)) return "coverage";
+    return "other";
+  }
+  function buildDeterministicPageAssignments(spans, pageCount) {
+    if (spans.length === 0) return [];
+    const pageTexts = /* @__PURE__ */ new Map();
+    for (const span of spans) {
+      const page = pageNumberForSpan(span);
+      if (!page) continue;
+      pageTexts.set(page, [pageTexts.get(page), span.sectionId, span.text].filter(Boolean).join("\n"));
+    }
+    return Array.from({ length: pageCount }, (_, index) => {
+      const page = index + 1;
+      const text = pageTexts.get(page) ?? "";
+      const lower = text.toLowerCase();
+      const extractorNames = [];
+      if (/\bdeclarations?|policy period|named insured|producer|schedule\b/.test(lower)) {
+        extractorNames.push("carrier_info", "named_insured", "declarations");
+      }
+      if (/\blimit|deductible|premium|coinsurance|scheduled amount|blanket\b/.test(lower) && /\$|\b\d{2,}(?:,\d{3})*\b/.test(lower)) {
+        extractorNames.push("coverage_limits");
+      }
+      if (/\bpremium|tax|fee|surcharge\b/.test(lower) && /\$/.test(lower)) extractorNames.push("premium_breakdown");
+      if (/\bendorsement|additional insured|amendatory\b/.test(lower)) extractorNames.push("endorsements");
+      if (/\bexclusion|not covered|does not apply\b/.test(lower)) extractorNames.push("exclusions");
+      if (/\bcondition|duties in the event|loss condition|general condition\b/.test(lower)) extractorNames.push("conditions");
+      if (/\bdefinition|defined terms?\b/.test(lower)) extractorNames.push("definitions");
+      if (/\bcovered cause|covered peril|cause of loss|perils insured\b/.test(lower)) extractorNames.push("covered_reasons");
+      if (textIncludesSupplementarySignal(text)) extractorNames.push("supplementary");
+      if (extractorNames.length === 0 && text.trim().length > 180) extractorNames.push("sections");
+      return {
+        localPageNumber: page,
+        extractorNames: [...new Set(extractorNames)],
+        pageRole: inferPageRole(lower),
+        hasScheduleValues: /\$|\b\d{2,}(?:,\d{3})*\b/.test(lower) && /\blimit|deductible|premium|schedule|class|location\b/.test(lower),
+        confidence: text.trim().length > 0 ? 0.8 : 0,
+        notes: "Deterministic source-span page assignment"
+      };
+    });
+  }
+  function inferPageRole(lowerText) {
+    if (/\bdeclarations?|schedule\b/.test(lowerText)) return "declarations_schedule";
+    if (/\bendorsement\b/.test(lowerText)) return "endorsement_form";
+    if (/\bexclusion|condition\b/.test(lowerText)) return "condition_exclusion_form";
+    if (/\bnotice|department of insurance|complaint\b/.test(lowerText)) return "supplementary";
+    if (lowerText.trim()) return "policy_form";
+    return "other";
+  }
   function shouldRunLlmReview(mode, report, sourceSpansAvailable) {
     if (mode === "skip" || maxReviewRounds <= 0) return false;
     if (mode === "always") return true;
@@ -8578,7 +8700,14 @@ function createExtractor(config) {
     }
     return lines.length > 0 ? lines.join("\n") : "";
   }
-  async function runFocusedExtractorTask(task, pdfInput, memory, pageRangeCache, getPageRangePdf, getPageImages, getPageRangeText) {
+  async function runFocusedExtractorTask(task, pdfInput, memory, sourceSpansForSections, pageRangeCache, getPageRangePdf, getPageImages, getPageRangeText) {
+    if (task.extractorName === "sections" && sourceSpansForSections.length > 0) {
+      return {
+        name: "sections",
+        data: buildSourceBackedSectionIndex(sourceSpansForSections, task.startPage, task.endPage),
+        usage: void 0
+      };
+    }
     if (task.extractorName === "supplementary") {
       const alreadyExtractedSummary = buildAlreadyExtractedSummary(memory);
       const budget = resolveBudget("extraction_focused", 4096);
@@ -8827,6 +8956,17 @@ ${pageText || "(No Docling text was available for this page range.)"}`;
       formInventory = resumed.formInventory;
       memory.set("form_inventory", formInventory);
       onProgress?.("Resuming from checkpoint (form inventory complete)...");
+    } else if (sourceSpans.length > 0) {
+      formInventory = buildDeterministicFormInventory(sourceSpans, pageCount) ?? { forms: [] };
+      onProgress?.(`Built deterministic form inventory for ${primaryType} ${documentType}.`);
+      memory.set("form_inventory", formInventory);
+      await pipelineCtx.save("form_inventory", {
+        id,
+        pageCount,
+        classifyResult,
+        formInventory,
+        memory: Object.fromEntries(memory)
+      });
     } else {
       onProgress?.(`Building form inventory for ${primaryType} ${documentType}...`);
       const budget = resolveBudget("extraction_form_inventory", 2048);
@@ -8867,6 +9007,20 @@ ${pageText || "(No Docling text was available for this page range.)"}`;
     if (resumed?.pageAssignments && pipelineCtx.isPhaseComplete("page_map")) {
       pageAssignments = resumed.pageAssignments;
       onProgress?.("Resuming from checkpoint (page map complete)...");
+    } else if (sourceSpans.length > 0) {
+      onProgress?.(`Mapping document pages from source spans for ${primaryType} ${documentType}...`);
+      pageAssignments = normalizePageAssignments(
+        buildDeterministicPageAssignments(sourceSpans, pageCount),
+        formInventory
+      );
+      await pipelineCtx.save("page_map", {
+        id,
+        pageCount,
+        classifyResult,
+        formInventory,
+        pageAssignments,
+        memory: Object.fromEntries(memory)
+      });
     } else {
       onProgress?.(`Mapping document pages for ${primaryType} ${documentType}...`);
       const chunkSize = 8;
@@ -8986,6 +9140,7 @@ ${pageText || "(No Docling text was available for this page range.)"}`;
               task,
               extractionPdfInput,
               memory,
+              sourceSpans,
               completedPageRangePdfCache,
               getPageRangePdf,
               convertPdfToImages ? getPageImages : void 0,
@@ -9150,6 +9305,7 @@ ${pageText || "(No Docling text was available for this page range.)"}`;
                   task,
                   extractionPdfInput,
                   memory,
+                  sourceSpans,
                   completedPageRangePdfCache,
                   getPageRangePdf,
                   convertPdfToImages ? getPageImages : void 0,