@claritylabs/cl-sdk 1.2.1 → 1.3.1

package/dist/index.mjs

@@ -673,7 +673,8 @@ var EndorsementSchema = z5.object({
   namedParties: z5.array(EndorsementPartySchema).optional(),
   keyTerms: z5.array(z5.string()).optional(),
   premiumImpact: z5.string().optional(),
-  content: z5.string(),
+  excerpt: z5.string().optional(),
+  content: z5.string().optional(),
   pageStart: z5.number(),
   pageEnd: z5.number().optional(),
   recordId: z5.string().optional(),
@@ -1286,7 +1287,10 @@ var SubsectionSchema = z16.object({
   title: z16.string(),
   sectionNumber: z16.string().optional(),
   pageNumber: z16.number().optional(),
-  content: z16.string()
+  excerpt: z16.string().optional(),
+  content: z16.string().optional(),
+  sourceSpanIds: z16.array(z16.string()).optional(),
+  sourceTextHash: z16.string().optional()
 });
 var SectionSchema = z16.object({
   title: z16.string(),
@@ -1295,7 +1299,8 @@ var SectionSchema = z16.object({
   pageEnd: z16.number().optional(),
   type: z16.string(),
   coverageType: z16.string().optional(),
-  content: z16.string(),
+  excerpt: z16.string().optional(),
+  content: z16.string().optional(),
   subsections: z16.array(SubsectionSchema).optional(),
   recordId: z16.string().optional(),
   sourceSpanIds: z16.array(z16.string()).optional(),
@@ -2813,6 +2818,14 @@ ${pageText || "(No Docling text was available for this page range.)"}`;
       maxTokens,
       taskKind,
       budgetDiagnostics,
+      trace: {
+        label: `${name} pages ${startPage}-${endPage}`,
+        extractorName: name,
+        startPage,
+        endPage,
+        phase: "extractor",
+        sourceBacked: !!sourceContext
+      },
       providerOptions: extractorProviderOptions
     })
   );
@@ -3678,21 +3691,30 @@ function collectContentFields(doc) {
       entries.push({ id: id++, path, text });
     }
   }
+  function hasSourceBacking(record) {
+    return Array.isArray(record.sourceSpanIds) && record.sourceSpanIds.length > 0 || !!record.sourceTextHash;
+  }
   add("summary", doc.summary);
   if (doc.sections) {
     for (let i = 0; i < doc.sections.length; i++) {
       const s = doc.sections[i];
-      add(`sections[${i}].content`, s.content);
+      if (!hasSourceBacking(s)) {
+        add(`sections[${i}].content`, s.content);
+      }
       if (s.subsections) {
         for (let j = 0; j < s.subsections.length; j++) {
-          add(`sections[${i}].subsections[${j}].content`, s.subsections[j].content);
+          if (!hasSourceBacking(s.subsections[j])) {
+            add(`sections[${i}].subsections[${j}].content`, s.subsections[j].content);
+          }
         }
       }
     }
   }
   if (doc.endorsements) {
     for (let i = 0; i < doc.endorsements.length; i++) {
-      add(`endorsements[${i}].content`, doc.endorsements[i].content);
+      if (!hasSourceBacking(doc.endorsements[i])) {
+        add(`endorsements[${i}].content`, doc.endorsements[i].content);
+      }
     }
   }
   if (doc.exclusions) {
@@ -3794,6 +3816,12 @@ async function formatDocumentContent(doc, generateText, options) {
             maxTokens: options?.maxTokens ?? 16384,
             taskKind: options?.taskKind,
             budgetDiagnostics: options?.budgetDiagnostics,
+            trace: {
+              label: `format content batch ${batchIdx + 1}/${batches.length}`,
+              phase: "format",
+              batchIndex: batchIdx + 1,
+              batchCount: batches.length
+            },
             providerOptions: options?.providerOptions
           })
         );
@@ -3868,7 +3896,7 @@ function chunkDocument(doc) {
       documentId: docId,
       type,
       text,
-      metadata: stringMetadata(metadata)
+      metadata: stringMetadata({ evidenceKind: "structured_fact", ...metadata })
     });
   }
   pushChunk(
@@ -4147,16 +4175,29 @@ function chunkDocument(doc) {
     );
   });
   doc.endorsements?.forEach((end, i) => {
+    const text = lines([
+      `Endorsement: ${end.title}`,
+      end.formNumber ? `Form: ${end.formNumber}` : null,
+      end.editionDate ? `Edition: ${end.editionDate}` : null,
+      `Type: ${end.endorsementType}`,
+      end.effectiveDate ? `Effective Date: ${end.effectiveDate}` : null,
+      end.affectedCoverageParts?.length ? `Affected Coverage Parts: ${end.affectedCoverageParts.join(", ")}` : null,
+      end.keyTerms?.length ? `Key Terms: ${end.keyTerms.join("; ")}` : null,
+      end.premiumImpact ? `Premium Impact: ${end.premiumImpact}` : null,
+      end.excerpt ? `Excerpt: ${end.excerpt}` : null
+    ]);
+    if (!text.trim()) return;
     pushChunk(
       `endorsement:${i}`,
       "endorsement",
-      `Endorsement: ${end.title}
-${end.content}`.trim(),
+      text,
       {
         endorsementType: end.endorsementType,
         formNumber: end.formNumber,
         pageStart: end.pageStart,
         pageEnd: end.pageEnd,
+        sourceSpanIds: end.sourceSpanIds?.join(","),
+        sourceTextHash: end.sourceTextHash,
         documentType: doc.type
       }
     );
@@ -4278,93 +4319,51 @@ ${declLines.join("\n")}`, declMeta);
   }
   doc.sections?.forEach((sec, i) => {
     const hasSubsections = sec.subsections && sec.subsections.length > 0;
-    const contentLength = sec.content.length;
-    if (hasSubsections) {
-      pushChunk(
-        `section:${i}`,
-        "section",
-        `Section: ${sec.title}
-${sec.content}`,
-        {
-          sectionType: sec.type,
-          sectionNumber: sec.sectionNumber,
-          pageStart: sec.pageStart,
-          pageEnd: sec.pageEnd,
-          documentType: doc.type,
-          hasSubsections: "true"
-        }
-      );
-      sec.subsections.forEach((sub, j) => {
-        pushChunk(
-          `section:${i}:sub:${j}`,
-          "section",
-          `${sec.title} > ${sub.title}
-${sub.content}`,
-          {
-            sectionType: sec.type,
-            parentSection: sec.title,
-            sectionNumber: sub.sectionNumber,
-            pageNumber: sub.pageNumber,
-            documentType: doc.type
-          }
-        );
-      });
-    } else if (contentLength > 2e3) {
-      const paragraphs = sec.content.split(/\n\n+/);
-      let currentChunk = "";
-      let chunkIndex = 0;
-      for (const para of paragraphs) {
-        if (currentChunk.length + para.length > 1e3 && currentChunk.length > 0) {
-          pushChunk(
-            `section:${i}:part:${chunkIndex}`,
-            "section",
-            `Section: ${sec.title} (part ${chunkIndex + 1})
-${currentChunk.trim()}`,
-            {
-              sectionType: sec.type,
-              sectionNumber: sec.sectionNumber,
-              pageStart: sec.pageStart,
-              pageEnd: sec.pageEnd,
-              documentType: doc.type,
-              partIndex: chunkIndex
-            }
-          );
-          currentChunk = "";
-          chunkIndex++;
-        }
-        currentChunk += (currentChunk ? "\n\n" : "") + para;
-      }
-      if (currentChunk.trim()) {
-        pushChunk(
-          `section:${i}:part:${chunkIndex}`,
-          "section",
-          `Section: ${sec.title} (part ${chunkIndex + 1})
-${currentChunk.trim()}`,
-          {
-            sectionType: sec.type,
-            sectionNumber: sec.sectionNumber,
-            pageStart: sec.pageStart,
-            pageEnd: sec.pageEnd,
-            documentType: doc.type,
-            partIndex: chunkIndex
-          }
-        );
+    pushChunk(
+      `section:${i}`,
+      "section",
+      lines([
+        `Section: ${sec.title}`,
+        `Type: ${sec.type}`,
+        sec.sectionNumber ? `Section Number: ${sec.sectionNumber}` : null,
+        `Pages: ${sec.pageStart}${sec.pageEnd ? `-${sec.pageEnd}` : ""}`,
+        sec.excerpt ? `Excerpt: ${sec.excerpt}` : null,
+        hasSubsections ? `Subsections: ${sec.subsections.map((sub) => sub.title).join(", ")}` : null
+      ]),
+      {
+        evidenceKind: "navigation",
+        sectionType: sec.type,
+        sectionNumber: sec.sectionNumber,
+        pageStart: sec.pageStart,
+        pageEnd: sec.pageEnd,
+        sourceSpanIds: sec.sourceSpanIds?.join(","),
+        sourceTextHash: sec.sourceTextHash,
+        documentType: doc.type,
+        hasSubsections
       }
-    } else {
+    );
+    sec.subsections?.forEach((sub, j) => {
       pushChunk(
-        `section:${i}`,
+        `section:${i}:sub:${j}`,
         "section",
-        `Section: ${sec.title}
-${sec.content}`,
+        lines([
+          `Section: ${sec.title} > ${sub.title}`,
+          sub.sectionNumber ? `Section Number: ${sub.sectionNumber}` : null,
+          sub.pageNumber ? `Page: ${sub.pageNumber}` : null,
+          sub.excerpt ? `Excerpt: ${sub.excerpt}` : null
+        ]),
         {
+          evidenceKind: "navigation",
           sectionType: sec.type,
-          sectionNumber: sec.sectionNumber,
-          pageStart: sec.pageStart,
-          pageEnd: sec.pageEnd,
+          parentSection: sec.title,
+          sectionNumber: sub.sectionNumber,
+          pageNumber: sub.pageNumber,
+          sourceSpanIds: sub.sourceSpanIds?.join(","),
+          sourceTextHash: sub.sourceTextHash,
           documentType: doc.type
         }
       );
-    }
+    });
   });
   doc.locations?.forEach((loc, i) => {
     chunks.push({
@@ -6236,14 +6235,17 @@ var EndorsementsSchema = z29.object({
       ).optional().describe("Named parties (additional insureds, loss payees, etc.)"),
       keyTerms: z29.array(z29.string()).optional().describe("Key terms or notable provisions in the endorsement"),
       premiumImpact: z29.string().optional().describe("Additional premium or credit"),
-      content: z29.string().describe("Full verbatim text of the endorsement"),
+      excerpt: z29.string().optional().describe("Short source excerpt, not full verbatim text"),
+      content: z29.string().optional().describe("Legacy fallback only; do not return full text when sourceSpanIds are available"),
       pageStart: z29.number().describe("Starting page number of this endorsement"),
-      pageEnd: z29.number().optional().describe("Ending page number of this endorsement")
+      pageEnd: z29.number().optional().describe("Ending page number of this endorsement"),
+      sourceSpanIds: z29.array(z29.string()).optional().describe("Source span IDs grounding this endorsement"),
+      sourceTextHash: z29.string().optional().describe("Hash of the source text when available")
     })
   ).describe("All endorsements found in the document")
 });
 function buildEndorsementsPrompt() {
-  return `You are an expert insurance document analyst. Extract ALL endorsements from this document. Preserve original language verbatim.
+  return `You are an expert insurance document analyst. Build a compact source-backed endorsement index for this document. Do not reproduce full endorsement language in the JSON output.
 
 For EACH endorsement, extract:
 - formNumber: the form identifier (e.g. "CG 21 47") \u2014 REQUIRED
@@ -6255,7 +6257,9 @@ For EACH endorsement, extract:
 - namedParties: for each party, extract name, role (additional_insured, loss_payee, mortgage_holder, certificate_holder, waiver_beneficiary, designated_person, other), relationship, and scope
 - keyTerms: notable provisions or key terms
 - premiumImpact: additional premium or credit if shown
-- content: full verbatim text \u2014 REQUIRED
+- excerpt: short identifying source excerpt, capped at 300 characters
+- sourceSpanIds: source span IDs from the provided SOURCE SPANS that ground this endorsement
+- content: legacy fallback only; omit/null when sourceSpanIds are available
 - pageStart: page number where endorsement begins \u2014 REQUIRED
 - pageEnd: page number where endorsement ends
 
@@ -6265,6 +6269,11 @@ PERSONAL LINES ENDORSEMENT RECOGNITION:
 - HO 17 XX series: mobilehome endorsements
 - DP 04 XX series: dwelling fire endorsements
 
+Critical rules:
+- Return compact metadata plus source references. The original endorsement wording lives in source spans.
+- Do not return full endorsement text in content when sourceSpanIds are available.
+- Prefer sourceSpanIds over generated prose for evidence.
+
 Return JSON only.`;
 }
 
@@ -6511,7 +6520,10 @@ var SubsectionSchema2 = z35.object({
   title: z35.string().describe("Subsection title"),
   sectionNumber: z35.string().optional().describe("Subsection number"),
   pageNumber: z35.number().optional().describe("Page number"),
-  content: z35.string().describe("Full verbatim text")
+  excerpt: z35.string().optional().describe("Short source excerpt, not full verbatim text"),
+  content: z35.string().optional().describe("Legacy fallback only; do not return full text when sourceSpanIds are available"),
+  sourceSpanIds: z35.array(z35.string()).optional().describe("Source span IDs grounding this subsection"),
+  sourceTextHash: z35.string().optional().describe("Hash of the source text when available")
 });
 var SectionsSchema = z35.object({
   sections: z35.array(
@@ -6532,15 +6544,18 @@ var SectionsSchema = z35.object({
         "regulatory",
         "other"
       ]).describe("Section type classification"),
-      content: z35.string().describe("Full verbatim text of the section"),
+      excerpt: z35.string().optional().describe("Short source excerpt, not full verbatim text"),
+      content: z35.string().optional().describe("Legacy fallback only; do not return full text when sourceSpanIds are available"),
       pageStart: z35.number().describe("Starting page number"),
       pageEnd: z35.number().optional().describe("Ending page number"),
+      sourceSpanIds: z35.array(z35.string()).optional().describe("Source span IDs grounding this section"),
+      sourceTextHash: z35.string().optional().describe("Hash of the source text when available"),
       subsections: z35.array(SubsectionSchema2).optional().describe("Subsections within this section")
     })
   ).describe("All document sections")
 });
 function buildSectionsPrompt() {
-  return `You are an expert insurance document analyst. Extract ALL sections, clauses, endorsements, and schedules from this document. Preserve the original language verbatim \u2014 do not summarize or paraphrase.
+  return `You are an expert insurance document analyst. Build a compact source-backed section index for this document. Do not reproduce full policy language in the JSON output.
 
 For each section, classify its type:
 - "declarations" \u2014 declarations page(s) listing named insured, policy period, limits, premiums
@@ -6554,10 +6569,13 @@ For each section, classify its type:
 - "notice", "regulatory" \u2014 notice provisions or regulatory disclosures
 - "other" \u2014 anything that doesn't fit the above categories
 
-Include accurate page numbers for every section. Include subsections only if the section has clearly defined subsections with their own titles.
+Include accurate page numbers for every section. Include sourceSpanIds from the provided SOURCE SPANS whenever available. Include subsections only if the section has clearly defined subsections with their own titles.
 If a page begins or ends in the middle of a section, treat it as a continuation of the existing section instead of creating a new orphan section from the fragment.
 
 Critical rules:
+- Return compact metadata plus source references. The original policy wording lives in source spans.
+- Use excerpt only for a short identifying snippet, capped at 300 characters.
+- Do not return full section text in content when sourceSpanIds are available. Leave content omitted/null in source-backed mode.
 - Ignore table-of-contents entries, page-number references, repeating headers/footers, and other navigational artifacts.
 - Do not create a new section from a lone continuation fragment such as a single paragraph tail or list item that clearly belongs to the previous page's section.
 - When a section spans multiple pages, keep it as one section with pageStart/pageEnd covering the full span represented in this extraction.