@claritylabs/cl-sdk 1.1.0 → 1.3.1

package/dist/index.mjs

@@ -673,7 +673,8 @@ var EndorsementSchema = z5.object({
   namedParties: z5.array(EndorsementPartySchema).optional(),
   keyTerms: z5.array(z5.string()).optional(),
   premiumImpact: z5.string().optional(),
-  content: z5.string(),
+  excerpt: z5.string().optional(),
+  content: z5.string().optional(),
   pageStart: z5.number(),
   pageEnd: z5.number().optional(),
   recordId: z5.string().optional(),
@@ -1286,7 +1287,10 @@ var SubsectionSchema = z16.object({
   title: z16.string(),
   sectionNumber: z16.string().optional(),
   pageNumber: z16.number().optional(),
-  content: z16.string()
+  excerpt: z16.string().optional(),
+  content: z16.string().optional(),
+  sourceSpanIds: z16.array(z16.string()).optional(),
+  sourceTextHash: z16.string().optional()
 });
 var SectionSchema = z16.object({
   title: z16.string(),
@@ -1295,7 +1299,8 @@ var SectionSchema = z16.object({
   pageEnd: z16.number().optional(),
   type: z16.string(),
   coverageType: z16.string().optional(),
-  content: z16.string(),
+  excerpt: z16.string().optional(),
+  content: z16.string().optional(),
   subsections: z16.array(SubsectionSchema).optional(),
   recordId: z16.string().optional(),
   sourceSpanIds: z16.array(z16.string()).optional(),
@@ -2471,6 +2476,254 @@ async function overlayTextOnPdf(pdfBytes, overlays) {
   return await pdfDoc.save();
 }
 
+// src/extraction/docling.ts
+function isDoclingExtractionInput(input) {
+  return Boolean(
+    input && typeof input === "object" && input.kind === "docling_document" && input.document && typeof input.document === "object"
+  );
+}
+function normalizeDoclingDocument(document, options) {
+  const itemMap = buildItemMap(document);
+  const orderedRefs = getOrderedBodyRefs(document, itemMap);
+  const orderedItems = orderedRefs.length > 0 ? orderedRefs.map((ref) => itemMap.get(ref)).filter((item) => Boolean(item)) : getFallbackOrderedItems(document, itemMap);
+  const units = orderedItems.map(({ ref, item }) => normalizeItem(ref, item)).filter((unit) => Boolean(unit && unit.text.trim()));
+  const pageCount = inferPageCount(document, units);
+  const pageTexts = /* @__PURE__ */ new Map();
+  for (const unit of units) {
+    const page = clampPage(unit.pageStart ?? 1, pageCount);
+    pageTexts.set(page, appendText(pageTexts.get(page), unit.text));
+  }
+  const fullText = Array.from({ length: pageCount }, (_, index) => {
+    const pageNumber = index + 1;
+    const text = pageTexts.get(pageNumber)?.trim();
+    return text ? `Page ${pageNumber}
+${text}` : "";
+  }).filter(Boolean).join("\n\n");
+  const sourceKind = options.sourceKind ?? "policy_pdf";
+  const sourceSpans = units.map((unit, index) => {
+    const span = buildSourceSpan(
+      {
+        documentId: options.documentId,
+        sourceKind,
+        text: unit.text,
+        pageStart: unit.pageStart,
+        pageEnd: unit.pageEnd,
+        sectionId: unit.label,
+        metadata: {
+          sourceSystem: "docling",
+          sourceUnit: "docling_item",
+          doclingRef: unit.ref,
+          ...unit.label ? { doclingLabel: unit.label } : {}
+        }
+      },
+      index
+    );
+    return {
+      ...span,
+      kind: "plain_text",
+      bbox: unit.bboxes?.length ? unit.bboxes : void 0
+    };
+  });
+  return {
+    pageCount,
+    fullText,
+    pageTexts,
+    units,
+    sourceSpans
+  };
+}
+function getDoclingPageRangeText(normalized, startPage, endPage) {
+  const start = clampPage(startPage, normalized.pageCount);
+  const end = clampPage(endPage, normalized.pageCount);
+  const lines = [];
+  for (let page = start; page <= end; page++) {
+    const text = normalized.pageTexts.get(page)?.trim();
+    if (text) {
+      lines.push(`Page ${page}
+${text}`);
+    }
+  }
+  return lines.join("\n\n");
+}
+function buildDoclingProviderOptions(normalized, existingOptions) {
+  return {
+    ...existingOptions,
+    doclingText: normalized.fullText,
+    doclingPageCount: normalized.pageCount
+  };
+}
+function mergeSourceSpans(spans) {
+  const seen = /* @__PURE__ */ new Set();
+  const merged = [];
+  for (const span of spans) {
+    const key = [
+      span.documentId,
+      span.pageStart ?? span.location?.startPage ?? span.location?.page ?? "na",
+      span.pageEnd ?? span.location?.endPage ?? span.pageStart ?? "na",
+      span.sectionId ?? span.location?.fieldPath ?? "na",
+      span.textHash ?? sourceSpanTextHash(span.text)
+    ].join(":");
+    if (seen.has(key)) continue;
+    seen.add(key);
+    merged.push(span);
+  }
+  return merged;
+}
+function buildItemMap(document) {
+  const map = /* @__PURE__ */ new Map();
+  addItems(map, "#/texts", document.texts ?? []);
+  addItems(map, "#/tables", document.tables ?? []);
+  addItems(map, "#/key_value_items", document.key_value_items ?? document.keyValueItems ?? []);
+  addItems(map, "#/pictures", document.pictures ?? []);
+  return map;
+}
+function addItems(map, baseRef, items) {
+  items.forEach((item, index) => {
+    const ref = getSelfRef(item) ?? `${baseRef}/${index}`;
+    map.set(ref, { ref, item });
+  });
+}
+function getFallbackOrderedItems(document, itemMap) {
+  const refs = [
+    ...(document.texts ?? []).map((item, index) => getSelfRef(item) ?? `#/texts/${index}`),
+    ...(document.tables ?? []).map((item, index) => getSelfRef(item) ?? `#/tables/${index}`),
+    ...(document.key_value_items ?? document.keyValueItems ?? []).map((item, index) => getSelfRef(item) ?? `#/key_value_items/${index}`)
+  ];
+  return refs.map((ref) => itemMap.get(ref)).filter((item) => Boolean(item));
+}
+function getOrderedBodyRefs(document, itemMap) {
+  const groupMap = /* @__PURE__ */ new Map();
+  (document.groups ?? []).forEach((group, index) => {
+    groupMap.set(getSelfRef(group) ?? `#/groups/${index}`, group);
+  });
+  const refs = [];
+  const visited = /* @__PURE__ */ new Set();
+  const visitRef = (ref) => {
+    const itemEntry = itemMap.get(ref);
+    if (itemEntry) {
+      if (!visited.has(ref)) {
+        visited.add(ref);
+        refs.push(ref);
+      }
+      visitNode(itemEntry.item);
+      return;
+    }
+    visitNode(groupMap.get(ref));
+  };
+  const visitNode = (node) => {
+    for (const child of node?.children ?? []) {
+      const ref = getRef(child);
+      if (!ref) continue;
+      visitRef(ref);
+    }
+  };
+  visitNode(document.body);
+  return refs;
+}
+function normalizeItem(ref, item) {
+  const text = getItemText(item).trim();
+  if (!text) return void 0;
+  const pages = (item.prov ?? []).map((prov) => getPageNumber(prov)).filter((page) => typeof page === "number" && page > 0);
+  const pageStart = pages.length ? Math.min(...pages) : void 0;
+  const pageEnd = pages.length ? Math.max(...pages) : pageStart;
+  const bboxes = (item.prov ?? []).map((prov) => toSourceSpanBBox(prov)).filter((bbox) => Boolean(bbox));
+  return {
+    ref,
+    label: typeof item.label === "string" ? item.label : void 0,
+    text,
+    pageStart,
+    pageEnd,
+    bboxes: bboxes.length ? bboxes : void 0
+  };
+}
+function getItemText(item) {
+  if (typeof item.text === "string" && item.text.trim()) return item.text;
+  if (typeof item.orig === "string" && item.orig.trim()) return item.orig;
+  const table = tableToMarkdown(item.data);
+  if (table) return table;
+  return "";
+}
+function tableToMarkdown(data) {
+  const record = asRecord(data);
+  const cells = Array.isArray(record?.table_cells) ? record.table_cells : Array.isArray(record?.tableCells) ? record.tableCells : void 0;
+  if (!cells) return void 0;
+  const parsedCells = cells.map((cell) => asRecord(cell)).filter((cell) => Boolean(cell)).map((cell) => ({
+    row: firstNumber2([cell.start_row_offset, cell.row_header, cell.row, cell.rowIndex]) ?? 0,
+    col: firstNumber2([cell.start_col_offset, cell.col, cell.colIndex]) ?? 0,
+    text: firstString([cell.text, cell.orig, cell.content])
+  })).filter((cell) => cell.text);
+  if (parsedCells.length === 0) return void 0;
+  const maxRow = Math.max(...parsedCells.map((cell) => cell.row));
+  const maxCol = Math.max(...parsedCells.map((cell) => cell.col));
+  const rows = Array.from({ length: maxRow + 1 }, () => Array.from({ length: maxCol + 1 }, () => ""));
+  for (const cell of parsedCells) {
+    rows[cell.row][cell.col] = cell.text;
+  }
+  if (rows.length === 1) return rows[0].filter(Boolean).join(" | ");
+  const header = rows[0];
+  const separator = header.map(() => "---");
+  return [header, separator, ...rows.slice(1)].map((row) => `| ${row.map((value) => value.trim()).join(" | ")} |`).join("\n");
+}
+function inferPageCount(document, units) {
+  const pages = document.pages;
+  if (Array.isArray(pages)) return Math.max(1, pages.length);
+  if (pages && typeof pages === "object") {
+    const keys = Object.keys(pages);
+    const numericMax = Math.max(0, ...keys.map((key) => Number(key)).filter((value) => Number.isFinite(value)));
+    return Math.max(1, numericMax || keys.length);
+  }
+  return Math.max(1, ...units.flatMap((unit) => [unit.pageStart ?? 0, unit.pageEnd ?? 0]));
+}
+function getSelfRef(value) {
+  return value.self_ref ?? value.selfRef;
+}
+function getRef(value) {
+  if (typeof value === "string") return value;
+  return value.$ref ?? value.ref;
+}
+function getPageNumber(prov) {
+  return prov.page_no ?? prov.pageNo ?? prov.page;
+}
+function toSourceSpanBBox(prov) {
+  const page = getPageNumber(prov);
+  const bbox = asRecord(prov.bbox);
+  if (!page || !bbox) return void 0;
+  const x = firstNumber2([bbox.x, bbox.l, bbox.left]);
+  const y = firstNumber2([bbox.y, bbox.t, bbox.top]);
+  const width = firstNumber2([bbox.width]);
+  const height = firstNumber2([bbox.height]);
+  const right = firstNumber2([bbox.r, bbox.right]);
+  const bottom = firstNumber2([bbox.b, bbox.bottom]);
+  if (x == null || y == null) return void 0;
+  const resolvedWidth = width ?? (right != null ? right - x : void 0);
+  const resolvedHeight = height ?? (bottom != null ? bottom - y : void 0);
+  if (resolvedWidth == null || resolvedHeight == null) return void 0;
+  return { page, x, y, width: resolvedWidth, height: resolvedHeight };
+}
+function clampPage(page, pageCount) {
+  return Math.max(1, Math.min(pageCount, page));
+}
+function appendText(existing, next) {
+  return existing ? `${existing}
+
+${next}` : next;
+}
+function asRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function firstString(values) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return "";
+}
+function firstNumber2(values) {
+  for (const value of values) {
+    if (typeof value === "number" && Number.isFinite(value)) return value;
+  }
+  return void 0;
+}
+
 // src/extraction/extractor.ts
 function sourceSpansForPageRange(providerOptions, startPage, endPage) {
   const sourceSpans = providerOptions?.sourceSpans;
@@ -2519,15 +2772,31 @@ async function runExtractor(params) {
   } = params;
   const extractorProviderOptions = { ...providerOptions };
   let fullPrompt;
-  const needsPdfBase64 = convertPdfToImages && !params.getPageImages || !convertPdfToImages && !params.getPageRangePdf;
-  const pdfBase64 = needsPdfBase64 ? await pdfInputToBase64(pdfInput) : void 0;
-  if (convertPdfToImages) {
+  if (params.getPageRangeText) {
+    const pageText = await params.getPageRangeText(startPage, endPage);
+    extractorProviderOptions.doclingText = pageText;
+    extractorProviderOptions.doclingPageRange = { startPage, endPage };
+    fullPrompt = `${prompt}
+
+[Document pages ${startPage}-${endPage} are provided below as Docling-extracted text.]
+
+${pageText || "(No Docling text was available for this page range.)"}`;
+  } else if (convertPdfToImages) {
+    if (!pdfInput) {
+      throw new Error("pdfInput is required when extracting page images.");
+    }
+    const needsPdfBase64 = !params.getPageImages;
+    const pdfBase64 = needsPdfBase64 ? await pdfInputToBase64(pdfInput) : void 0;
     const images = params.getPageImages ? await params.getPageImages(startPage, endPage) : await convertPdfToImages(pdfBase64, startPage, endPage);
     extractorProviderOptions.images = images;
     fullPrompt = `${prompt}
 
 [Document pages ${startPage}-${endPage} are provided as images.]`;
   } else {
+    if (!pdfInput) {
+      throw new Error("pdfInput is required when extracting page PDFs.");
+    }
+    const pdfBase64 = params.getPageRangePdf ? void 0 : await pdfInputToBase64(pdfInput);
     const cacheKey = `${startPage}-${endPage}`;
     const cachedPagesPdf = pageRangeCache?.get(cacheKey);
     const pagesPdf = cachedPagesPdf ?? (params.getPageRangePdf ? await params.getPageRangePdf(startPage, endPage) : await extractPageRange(pdfBase64, startPage, endPage));
@@ -2549,6 +2818,14 @@ async function runExtractor(params) {
       maxTokens,
       taskKind,
       budgetDiagnostics,
+      trace: {
+        label: `${name} pages ${startPage}-${endPage}`,
+        extractorName: name,
+        startPage,
+        endPage,
+        phase: "extractor",
+        sourceBacked: !!sourceContext
+      },
       providerOptions: extractorProviderOptions
     })
   );
@@ -3414,21 +3691,30 @@ function collectContentFields(doc) {
       entries.push({ id: id++, path, text });
     }
   }
+  function hasSourceBacking(record) {
+    return Array.isArray(record.sourceSpanIds) && record.sourceSpanIds.length > 0 || !!record.sourceTextHash;
+  }
   add("summary", doc.summary);
   if (doc.sections) {
     for (let i = 0; i < doc.sections.length; i++) {
       const s = doc.sections[i];
-      add(`sections[${i}].content`, s.content);
+      if (!hasSourceBacking(s)) {
+        add(`sections[${i}].content`, s.content);
+      }
       if (s.subsections) {
         for (let j = 0; j < s.subsections.length; j++) {
-          add(`sections[${i}].subsections[${j}].content`, s.subsections[j].content);
+          if (!hasSourceBacking(s.subsections[j])) {
+            add(`sections[${i}].subsections[${j}].content`, s.subsections[j].content);
+          }
         }
       }
     }
   }
   if (doc.endorsements) {
     for (let i = 0; i < doc.endorsements.length; i++) {
-      add(`endorsements[${i}].content`, doc.endorsements[i].content);
+      if (!hasSourceBacking(doc.endorsements[i])) {
+        add(`endorsements[${i}].content`, doc.endorsements[i].content);
+      }
     }
   }
   if (doc.exclusions) {
@@ -3530,6 +3816,12 @@ async function formatDocumentContent(doc, generateText, options) {
             maxTokens: options?.maxTokens ?? 16384,
             taskKind: options?.taskKind,
             budgetDiagnostics: options?.budgetDiagnostics,
+            trace: {
+              label: `format content batch ${batchIdx + 1}/${batches.length}`,
+              phase: "format",
+              batchIndex: batchIdx + 1,
+              batchCount: batches.length
+            },
             providerOptions: options?.providerOptions
           })
         );
@@ -3567,7 +3859,7 @@ function formatAddress(addr) {
 function asRecordArray(value) {
   return Array.isArray(value) ? value.filter((item) => Boolean(item) && typeof item === "object" && !Array.isArray(item)) : [];
 }
-function firstString(item, keys) {
+function firstString2(item, keys) {
   for (const key of keys) {
     const value = item[key];
     if (typeof value === "string" && value.trim()) return value;
@@ -3604,7 +3896,7 @@ function chunkDocument(doc) {
       documentId: docId,
       type,
       text,
-      metadata: stringMetadata(metadata)
+      metadata: stringMetadata({ evidenceKind: "structured_fact", ...metadata })
     });
   }
   pushChunk(
@@ -3883,16 +4175,29 @@ function chunkDocument(doc) {
     );
   });
   doc.endorsements?.forEach((end, i) => {
+    const text = lines([
+      `Endorsement: ${end.title}`,
+      end.formNumber ? `Form: ${end.formNumber}` : null,
+      end.editionDate ? `Edition: ${end.editionDate}` : null,
+      `Type: ${end.endorsementType}`,
+      end.effectiveDate ? `Effective Date: ${end.effectiveDate}` : null,
+      end.affectedCoverageParts?.length ? `Affected Coverage Parts: ${end.affectedCoverageParts.join(", ")}` : null,
+      end.keyTerms?.length ? `Key Terms: ${end.keyTerms.join("; ")}` : null,
+      end.premiumImpact ? `Premium Impact: ${end.premiumImpact}` : null,
+      end.excerpt ? `Excerpt: ${end.excerpt}` : null
+    ]);
+    if (!text.trim()) return;
     pushChunk(
       `endorsement:${i}`,
       "endorsement",
-      `Endorsement: ${end.title}
-${end.content}`.trim(),
+      text,
       {
         endorsementType: end.endorsementType,
         formNumber: end.formNumber,
         pageStart: end.pageStart,
         pageEnd: end.pageEnd,
+        sourceSpanIds: end.sourceSpanIds?.join(","),
+        sourceTextHash: end.sourceTextHash,
         documentType: doc.type
       }
     );
@@ -3924,32 +4229,32 @@ ${exc.content}`.trim(), {
     );
   });
   asRecordArray(extendedDoc.definitions).forEach((definition, i) => {
-    const term = firstString(definition, ["term", "name", "title"]) ?? `Definition ${i + 1}`;
-    const body = firstString(definition, ["definition", "content", "text", "meaning"]);
+    const term = firstString2(definition, ["term", "name", "title"]) ?? `Definition ${i + 1}`;
+    const body = firstString2(definition, ["definition", "content", "text", "meaning"]);
     pushChunk(
       `definition:${i}`,
       "definition",
       lines([
         `Definition: ${term}`,
         body,
-        firstString(definition, ["originalContent", "source"]) ? `Source: ${firstString(definition, ["originalContent", "source"])}` : null
+        firstString2(definition, ["originalContent", "source"]) ? `Source: ${firstString2(definition, ["originalContent", "source"])}` : null
       ]),
       {
         term,
-        formNumber: firstString(definition, ["formNumber"]),
-        formTitle: firstString(definition, ["formTitle"]),
+        formNumber: firstString2(definition, ["formNumber"]),
+        formTitle: firstString2(definition, ["formTitle"]),
         pageNumber: typeof definition.pageNumber === "number" ? definition.pageNumber : void 0,
-        sectionRef: firstString(definition, ["sectionRef", "sectionTitle"]),
+        sectionRef: firstString2(definition, ["sectionRef", "sectionTitle"]),
         documentType: doc.type
       }
     );
   });
   const coveredReasons = asRecordArray(extendedDoc.coveredReasons ?? extendedDoc.covered_reasons);
   coveredReasons.forEach((coveredReason, i) => {
-    const title = firstString(coveredReason, ["title", "name", "reason", "peril", "cause"]) ?? `Covered Reason ${i + 1}`;
-    const coverageName = firstString(coveredReason, ["coverageName", "coverage", "coveragePart"]);
-    const reasonNumber = firstString(coveredReason, ["reasonNumber", "number"]);
-    const body = firstString(coveredReason, ["content", "description", "text", "coverageGrant"]);
+    const title = firstString2(coveredReason, ["title", "name", "reason", "peril", "cause"]) ?? `Covered Reason ${i + 1}`;
+    const coverageName = firstString2(coveredReason, ["coverageName", "coverage", "coveragePart"]);
+    const reasonNumber = firstString2(coveredReason, ["reasonNumber", "number"]);
+    const body = firstString2(coveredReason, ["content", "description", "text", "coverageGrant"]);
     pushChunk(
       `covered_reason:${i}`,
       "covered_reason",
@@ -3958,16 +4263,16 @@ ${exc.content}`.trim(), {
         reasonNumber ? `Reason Number: ${reasonNumber}` : null,
         `Covered Reason: ${title}`,
         body,
-        firstString(coveredReason, ["originalContent", "source"]) ? `Source: ${firstString(coveredReason, ["originalContent", "source"])}` : null
+        firstString2(coveredReason, ["originalContent", "source"]) ? `Source: ${firstString2(coveredReason, ["originalContent", "source"])}` : null
       ]),
       {
         coverageName,
         reasonNumber,
         title,
-        formNumber: firstString(coveredReason, ["formNumber"]),
-        formTitle: firstString(coveredReason, ["formTitle"]),
+        formNumber: firstString2(coveredReason, ["formNumber"]),
+        formTitle: firstString2(coveredReason, ["formTitle"]),
         pageNumber: typeof coveredReason.pageNumber === "number" ? coveredReason.pageNumber : void 0,
-        sectionRef: firstString(coveredReason, ["sectionRef", "sectionTitle"]),
+        sectionRef: firstString2(coveredReason, ["sectionRef", "sectionTitle"]),
         documentType: doc.type
       }
     );
@@ -3987,10 +4292,10 @@ ${exc.content}`.trim(), {
           reasonNumber,
           title,
           conditionIndex,
-          formNumber: firstString(coveredReason, ["formNumber"]),
-          formTitle: firstString(coveredReason, ["formTitle"]),
+          formNumber: firstString2(coveredReason, ["formNumber"]),
+          formTitle: firstString2(coveredReason, ["formTitle"]),
           pageNumber: typeof coveredReason.pageNumber === "number" ? coveredReason.pageNumber : void 0,
-          sectionRef: firstString(coveredReason, ["sectionRef", "sectionTitle"]),
+          sectionRef: firstString2(coveredReason, ["sectionRef", "sectionTitle"]),
           documentType: doc.type
         }
       );
@@ -4014,93 +4319,51 @@ ${declLines.join("\n")}`, declMeta);
   }
   doc.sections?.forEach((sec, i) => {
     const hasSubsections = sec.subsections && sec.subsections.length > 0;
-    const contentLength = sec.content.length;
-    if (hasSubsections) {
-      pushChunk(
-        `section:${i}`,
-        "section",
-        `Section: ${sec.title}
-${sec.content}`,
-        {
-          sectionType: sec.type,
-          sectionNumber: sec.sectionNumber,
-          pageStart: sec.pageStart,
-          pageEnd: sec.pageEnd,
-          documentType: doc.type,
-          hasSubsections: "true"
-        }
-      );
-      sec.subsections.forEach((sub, j) => {
-        pushChunk(
-          `section:${i}:sub:${j}`,
-          "section",
-          `${sec.title} > ${sub.title}
-${sub.content}`,
-          {
-            sectionType: sec.type,
-            parentSection: sec.title,
-            sectionNumber: sub.sectionNumber,
-            pageNumber: sub.pageNumber,
-            documentType: doc.type
-          }
-        );
-      });
-    } else if (contentLength > 2e3) {
-      const paragraphs = sec.content.split(/\n\n+/);
-      let currentChunk = "";
-      let chunkIndex = 0;
-      for (const para of paragraphs) {
-        if (currentChunk.length + para.length > 1e3 && currentChunk.length > 0) {
-          pushChunk(
-            `section:${i}:part:${chunkIndex}`,
-            "section",
-            `Section: ${sec.title} (part ${chunkIndex + 1})
-${currentChunk.trim()}`,
-            {
-              sectionType: sec.type,
-              sectionNumber: sec.sectionNumber,
-              pageStart: sec.pageStart,
-              pageEnd: sec.pageEnd,
-              documentType: doc.type,
-              partIndex: chunkIndex
-            }
-          );
-          currentChunk = "";
-          chunkIndex++;
-        }
-        currentChunk += (currentChunk ? "\n\n" : "") + para;
-      }
-      if (currentChunk.trim()) {
-        pushChunk(
-          `section:${i}:part:${chunkIndex}`,
-          "section",
-          `Section: ${sec.title} (part ${chunkIndex + 1})
-${currentChunk.trim()}`,
-          {
-            sectionType: sec.type,
-            sectionNumber: sec.sectionNumber,
-            pageStart: sec.pageStart,
-            pageEnd: sec.pageEnd,
-            documentType: doc.type,
-            partIndex: chunkIndex
-          }
-        );
+    pushChunk(
+      `section:${i}`,
+      "section",
+      lines([
+        `Section: ${sec.title}`,
+        `Type: ${sec.type}`,
+        sec.sectionNumber ? `Section Number: ${sec.sectionNumber}` : null,
+        `Pages: ${sec.pageStart}${sec.pageEnd ? `-${sec.pageEnd}` : ""}`,
+        sec.excerpt ? `Excerpt: ${sec.excerpt}` : null,
+        hasSubsections ? `Subsections: ${sec.subsections.map((sub) => sub.title).join(", ")}` : null
+      ]),
+      {
+        evidenceKind: "navigation",
+        sectionType: sec.type,
+        sectionNumber: sec.sectionNumber,
+        pageStart: sec.pageStart,
+        pageEnd: sec.pageEnd,
+        sourceSpanIds: sec.sourceSpanIds?.join(","),
+        sourceTextHash: sec.sourceTextHash,
+        documentType: doc.type,
+        hasSubsections
       }
-    } else {
+    );
+    sec.subsections?.forEach((sub, j) => {
       pushChunk(
-        `section:${i}`,
+        `section:${i}:sub:${j}`,
         "section",
-        `Section: ${sec.title}
-${sec.content}`,
+        lines([
+          `Section: ${sec.title} > ${sub.title}`,
+          sub.sectionNumber ? `Section Number: ${sub.sectionNumber}` : null,
+          sub.pageNumber ? `Page: ${sub.pageNumber}` : null,
+          sub.excerpt ? `Excerpt: ${sub.excerpt}` : null
+        ]),
         {
+          evidenceKind: "navigation",
           sectionType: sec.type,
-          sectionNumber: sec.sectionNumber,
-          pageStart: sec.pageStart,
-          pageEnd: sec.pageEnd,
+          parentSection: sec.title,
+          sectionNumber: sub.sectionNumber,
+          pageNumber: sub.pageNumber,
+          sourceSpanIds: sub.sourceSpanIds?.join(","),
+          sourceTextHash: sub.sourceTextHash,
           documentType: doc.type
         }
       );
-    }
+    });
   });
   doc.locations?.forEach((loc, i) => {
     chunks.push({
@@ -5972,14 +6235,17 @@ var EndorsementsSchema = z29.object({
       ).optional().describe("Named parties (additional insureds, loss payees, etc.)"),
       keyTerms: z29.array(z29.string()).optional().describe("Key terms or notable provisions in the endorsement"),
       premiumImpact: z29.string().optional().describe("Additional premium or credit"),
-      content: z29.string().describe("Full verbatim text of the endorsement"),
+      excerpt: z29.string().optional().describe("Short source excerpt, not full verbatim text"),
+      content: z29.string().optional().describe("Legacy fallback only; do not return full text when sourceSpanIds are available"),
       pageStart: z29.number().describe("Starting page number of this endorsement"),
-      pageEnd: z29.number().optional().describe("Ending page number of this endorsement")
+      pageEnd: z29.number().optional().describe("Ending page number of this endorsement"),
+      sourceSpanIds: z29.array(z29.string()).optional().describe("Source span IDs grounding this endorsement"),
+      sourceTextHash: z29.string().optional().describe("Hash of the source text when available")
     })
   ).describe("All endorsements found in the document")
 });
 function buildEndorsementsPrompt() {
-  return `You are an expert insurance document analyst. Extract ALL endorsements from this document. Preserve original language verbatim.
+  return `You are an expert insurance document analyst. Build a compact source-backed endorsement index for this document. Do not reproduce full endorsement language in the JSON output.
 
 For EACH endorsement, extract:
 - formNumber: the form identifier (e.g. "CG 21 47") \u2014 REQUIRED
@@ -5991,7 +6257,9 @@ For EACH endorsement, extract:
 - namedParties: for each party, extract name, role (additional_insured, loss_payee, mortgage_holder, certificate_holder, waiver_beneficiary, designated_person, other), relationship, and scope
 - keyTerms: notable provisions or key terms
 - premiumImpact: additional premium or credit if shown
-- content: full verbatim text \u2014 REQUIRED
+- excerpt: short identifying source excerpt, capped at 300 characters
+- sourceSpanIds: source span IDs from the provided SOURCE SPANS that ground this endorsement
+- content: legacy fallback only; omit/null when sourceSpanIds are available
 - pageStart: page number where endorsement begins \u2014 REQUIRED
 - pageEnd: page number where endorsement ends
 
@@ -6001,6 +6269,11 @@ PERSONAL LINES ENDORSEMENT RECOGNITION:
 - HO 17 XX series: mobilehome endorsements
 - DP 04 XX series: dwelling fire endorsements
 
+Critical rules:
+- Return compact metadata plus source references. The original endorsement wording lives in source spans.
+- Do not return full endorsement text in content when sourceSpanIds are available.
+- Prefer sourceSpanIds over generated prose for evidence.
+
 Return JSON only.`;
 }
 
@@ -6247,7 +6520,10 @@ var SubsectionSchema2 = z35.object({
   title: z35.string().describe("Subsection title"),
   sectionNumber: z35.string().optional().describe("Subsection number"),
   pageNumber: z35.number().optional().describe("Page number"),
-  content: z35.string().describe("Full verbatim text")
+  excerpt: z35.string().optional().describe("Short source excerpt, not full verbatim text"),
+  content: z35.string().optional().describe("Legacy fallback only; do not return full text when sourceSpanIds are available"),
+  sourceSpanIds: z35.array(z35.string()).optional().describe("Source span IDs grounding this subsection"),
+  sourceTextHash: z35.string().optional().describe("Hash of the source text when available")
 });
 var SectionsSchema = z35.object({
   sections: z35.array(
@@ -6268,15 +6544,18 @@ var SectionsSchema = z35.object({
         "regulatory",
         "other"
       ]).describe("Section type classification"),
-      content: z35.string().describe("Full verbatim text of the section"),
+      excerpt: z35.string().optional().describe("Short source excerpt, not full verbatim text"),
+      content: z35.string().optional().describe("Legacy fallback only; do not return full text when sourceSpanIds are available"),
       pageStart: z35.number().describe("Starting page number"),
       pageEnd: z35.number().optional().describe("Ending page number"),
+      sourceSpanIds: z35.array(z35.string()).optional().describe("Source span IDs grounding this section"),
+      sourceTextHash: z35.string().optional().describe("Hash of the source text when available"),
       subsections: z35.array(SubsectionSchema2).optional().describe("Subsections within this section")
     })
   ).describe("All document sections")
 });
 function buildSectionsPrompt() {
-  return `You are an expert insurance document analyst. Extract ALL sections, clauses, endorsements, and schedules from this document. Preserve the original language verbatim \u2014 do not summarize or paraphrase.
+  return `You are an expert insurance document analyst. Build a compact source-backed section index for this document. Do not reproduce full policy language in the JSON output.
 
 For each section, classify its type:
 - "declarations" \u2014 declarations page(s) listing named insured, policy period, limits, premiums
@@ -6290,10 +6569,13 @@ For each section, classify its type:
 - "notice", "regulatory" \u2014 notice provisions or regulatory disclosures
 - "other" \u2014 anything that doesn't fit the above categories
 
-Include accurate page numbers for every section. Include subsections only if the section has clearly defined subsections with their own titles.
+Include accurate page numbers for every section. Include sourceSpanIds from the provided SOURCE SPANS whenever available. Include subsections only if the section has clearly defined subsections with their own titles.
 If a page begins or ends in the middle of a section, treat it as a continuation of the existing section instead of creating a new orphan section from the fragment.
 
 Critical rules:
+- Return compact metadata plus source references. The original policy wording lives in source spans.
+- Use excerpt only for a short identifying snippet, capped at 300 characters.
+- Do not return full section text in content when sourceSpanIds are available. Leave content omitted/null in source-backed mode.
 - Ignore table-of-contents entries, page-number references, repeating headers/footers, and other navigational artifacts.
 - Do not create a new section from a lone continuation fragment such as a single paragraph tail or list item that clearly belongs to the previous page's section.
 - When a section spans multiple pages, keep it as one section with pageStart/pageEnd covering the full span represented in this extraction.
@@ -6461,21 +6743,21 @@ Return JSON only.`;
 }
 
 // src/prompts/extractors/index.ts
-function asRecord(data) {
+function asRecord2(data) {
   return data && typeof data === "object" ? data : void 0;
 }
 function getSections2(data) {
-  const sections = asRecord(data)?.sections;
+  const sections = asRecord2(data)?.sections;
   return Array.isArray(sections) ? sections : [];
 }
 function isCoveredReasonsEmpty(data) {
-  const record = asRecord(data);
+  const record = asRecord2(data);
   if (!record) return true;
   const coveredReasons = Array.isArray(record.coveredReasons) ? record.coveredReasons : Array.isArray(record.covered_reasons) ? record.covered_reasons : [];
   return coveredReasons.length === 0;
 }
 function isDefinitionsEmpty(data) {
-  const definitions = asRecord(data)?.definitions;
+  const definitions = asRecord2(data)?.definitions;
   return !Array.isArray(definitions) || definitions.length === 0;
 }
 function sectionLooksLikeCoveredReason(section) {
@@ -6709,6 +6991,14 @@ function decideReferentialResolutionAction(params) {
 }
 
 // src/extraction/resolve-referential.ts
+function formatDoclingTextContext(providerOptions) {
+  const doclingText = providerOptions?.doclingText;
+  if (typeof doclingText !== "string" || !doclingText.trim()) return "";
+  return `
+
+DOCLING DOCUMENT TEXT:
+${doclingText}`;
+}
 function parseReferenceTarget(text) {
   if (typeof text !== "string") return void 0;
   const normalized = text.trim();
@@ -6790,12 +7080,12 @@ Return the page range (1-indexed) where this section is located. If the section
 
 If you cannot find the section, return startPage: 0 and endPage: 0.
 
-Return JSON only.`,
+Return JSON only.${formatDoclingTextContext(providerOptions)}`,
         schema: PageLocationSchema,
         maxTokens: budget.maxTokens,
         taskKind: "extraction_referential_lookup",
         budgetDiagnostics: budget,
-        providerOptions: await buildPdfProviderOptions(pdfInput, providerOptions)
+        providerOptions: pdfInput ? await buildPdfProviderOptions(pdfInput, providerOptions) : providerOptions
       },
       {
         fallback: { startPage: 0, endPage: 0 },
@@ -6829,6 +7119,7 @@ async function resolveReferentialCoverages(params) {
     convertPdfToImages,
     getPageRangePdf,
     getPageImages,
+    getPageRangeText,
     concurrency = 2,
     providerOptions,
     modelCapabilities,
@@ -6940,6 +7231,7 @@ async function resolveReferentialCoverages(params) {
             convertPdfToImages,
             getPageRangePdf,
             getPageImages,
+            getPageRangeText,
             maxTokens: budget.maxTokens,
             taskKind: "extraction_referential_lookup",
             budgetDiagnostics: budget,
@@ -7035,6 +7327,7 @@ async function runFocusedExtractorWithFallback(params) {
     pageRangeCache,
     getPageRangePdf,
     getPageImages,
+    getPageRangeText,
     trackUsage,
     resolveBudget,
     log
@@ -7064,7 +7357,8 @@ async function runFocusedExtractorWithFallback(params) {
       providerOptions,
       pageRangeCache,
       getPageRangePdf,
-      getPageImages
+      getPageImages,
+      getPageRangeText
     });
     trackUsage(result.usage, {
       taskKind,
@@ -7109,7 +7403,8 @@ async function runFocusedExtractorWithFallback(params) {
       providerOptions,
       pageRangeCache,
       getPageRangePdf,
-      getPageImages
+      getPageImages,
+      getPageRangeText
     });
     trackUsage(fallbackResult.usage, {
       taskKind,
@@ -7953,7 +8248,7 @@ function createExtractor(config) {
     }
     return lines.length > 0 ? lines.join("\n") : "";
   }
-  async function runFocusedExtractorTask(task, pdfInput, memory, pageRangeCache, getPageRangePdf, getPageImages) {
+  async function runFocusedExtractorTask(task, pdfInput, memory, pageRangeCache, getPageRangePdf, getPageImages, getPageRangeText) {
     if (task.extractorName === "supplementary") {
       const alreadyExtractedSummary = buildAlreadyExtractedSummary(memory);
       const budget = resolveBudget("extraction_focused", 4096);
@@ -7973,7 +8268,8 @@ function createExtractor(config) {
         providerOptions: activeProviderOptions,
         pageRangeCache,
         getPageRangePdf,
-        getPageImages
+        getPageImages,
+        getPageRangeText
       });
       trackUsage(result.usage, {
         taskKind: "extraction_focused",
@@ -7992,6 +8288,7 @@ function createExtractor(config) {
       pageRangeCache,
       getPageRangePdf,
       getPageImages,
+      getPageRangeText,
       trackUsage,
       resolveBudget,
       log
@@ -8007,8 +8304,14 @@ function createExtractor(config) {
     if (extractorPages.size === 0) return "No page assignments available.";
     return [...extractorPages.entries()].map(([extractorName, pages]) => `${extractorName}: ${pages.length} page(s), pages ${pages.join(", ")}`).join("\n");
   }
-  async function extract(pdfInput, documentId, options) {
+  async function extract(input, documentId, options) {
     const id = documentId ?? `doc-${Date.now()}`;
+    const isDoclingInput = isDoclingExtractionInput(input);
+    const pdfInput = isDoclingInput ? void 0 : input;
+    const doclingDocument = isDoclingInput ? normalizeDoclingDocument(input.document, {
+      documentId: id,
+      sourceKind: input.sourceKind
+    }) : void 0;
     const memory = /* @__PURE__ */ new Map();
     totalUsage = { inputTokens: 0, outputTokens: 0 };
     modelCalls = 0;
@@ -8018,7 +8321,10 @@ function createExtractor(config) {
       modelCalls: [],
       totalModelCallDurationMs: 0
     };
-    const sourceSpans = options?.sourceSpans ?? [];
+    const sourceSpans = mergeSourceSpans([
+      ...doclingDocument?.sourceSpans ?? [],
+      ...options?.sourceSpans ?? []
+    ]);
     const sourceChunks = sourceSpans.length ? chunkSourceSpans(sourceSpans) : [];
     activeProviderOptions = sourceSpans.length ? { ...providerOptions, sourceSpans, sourceChunks } : providerOptions;
     if (sourceStore && sourceSpans.length > 0) {
@@ -8047,24 +8353,40 @@ function createExtractor(config) {
     let fullPdfProviderOptionsPromise;
     let pageCountPromise;
     async function getPdfBase64ForExtraction() {
+      if (!pdfInput) {
+        throw new Error("PDF input is not available for Docling extraction.");
+      }
       if (pdfBase64Cache === void 0) {
         pdfBase64Cache = await pdfInputToBase64(pdfInput);
       }
       return pdfBase64Cache;
     }
     async function getCachedPageCount() {
+      if (doclingDocument) return doclingDocument.pageCount;
+      if (!pdfInput) {
+        throw new Error("PDF input is required to read page count.");
+      }
       if (!pageCountPromise) {
         pageCountPromise = getPdfSlicer().then((slicer) => slicer.getPageCount()).catch(() => getPdfPageCount(pdfInput));
       }
       return pageCountPromise;
     }
-    async function getFullPdfProviderOptions() {
+    async function getFullDocumentProviderOptions() {
+      if (doclingDocument) {
+        return buildDoclingProviderOptions(doclingDocument, activeProviderOptions);
+      }
+      if (!pdfInput) {
+        return activeProviderOptions ?? {};
+      }
       if (!fullPdfProviderOptionsPromise) {
         fullPdfProviderOptionsPromise = buildPdfProviderOptions(pdfInput, activeProviderOptions);
       }
       return fullPdfProviderOptionsPromise;
     }
     async function getPdfSlicer() {
+      if (!pdfInput) {
+        throw new Error("PDF input is not available for Docling extraction.");
+      }
       if (!pdfSlicerPromise) {
         pdfSlicerPromise = createPdfPageSlicer(pdfInput);
       }
@@ -8103,6 +8425,23 @@ function createExtractor(config) {
       pageRangeImageCache.set(cacheKey, promise);
       return promise;
     }
+    async function getPageRangeText(startPage, endPage) {
+      return doclingDocument ? getDoclingPageRangeText(doclingDocument, startPage, endPage) : "";
+    }
+    function withFullDocumentTextContext(prompt) {
+      if (!doclingDocument) return prompt;
+      return `${prompt}
+
+DOCLING DOCUMENT TEXT:
+${doclingDocument.fullText}`;
+    }
+    function withPageRangeTextContext(prompt, startPage, endPage, pageText) {
+      if (!doclingDocument) return prompt;
+      return `${prompt}
+
+DOCLING DOCUMENT PAGES ${startPage}-${endPage}:
+${pageText || "(No Docling text was available for this page range.)"}`;
+    }
     let classifyResult;
     if (resumed?.classifyResult && pipelineCtx.isPhaseComplete("classify")) {
       classifyResult = resumed.classifyResult;
@@ -8115,12 +8454,12 @@ function createExtractor(config) {
       const classifyResponse = await safeGenerateObject(
         generateObject,
         {
-          prompt: buildClassifyPrompt(),
+          prompt: withFullDocumentTextContext(buildClassifyPrompt()),
           schema: ClassifyResultSchema,
           maxTokens: budget.maxTokens,
           taskKind: "extraction_classify",
           budgetDiagnostics: budget,
-          providerOptions: await getFullPdfProviderOptions()
+          providerOptions: await getFullDocumentProviderOptions()
         },
         {
           fallback: { documentType: "policy", policyTypes: ["other"], confidence: 0 },
@@ -8165,12 +8504,12 @@ function createExtractor(config) {
       const formInventoryResponse = await safeGenerateObject(
         generateObject,
         {
-          prompt: buildFormInventoryPrompt(templateHints),
+          prompt: withFullDocumentTextContext(buildFormInventoryPrompt(templateHints)),
           schema: FormInventorySchema,
           maxTokens: budget.maxTokens,
           taskKind: "extraction_form_inventory",
           budgetDiagnostics: budget,
-          providerOptions: await getFullPdfProviderOptions()
+          providerOptions: await getFullDocumentProviderOptions()
         },
         {
           fallback: { forms: [] },
@@ -8213,18 +8552,24 @@ function createExtractor(config) {
       const pageMapResults = await Promise.all(
         pageMapChunks.map(
           ({ startPage, endPage }) => pageMapLimit(async () => {
-            const pagesPdf = await getPageRangePdf(startPage, endPage);
+            const pagesPdf = doclingDocument ? void 0 : await getPageRangePdf(startPage, endPage);
+            const pagesText = doclingDocument ? await getPageRangeText(startPage, endPage) : "";
             const budget = resolveBudget("extraction_page_map", 2048);
             const startedAt = Date.now();
             const mapResponse = await safeGenerateObject(
               generateObject,
               {
-                prompt: buildPageMapPrompt(templateHints, startPage, endPage, formInventoryHint),
+                prompt: withPageRangeTextContext(
+                  buildPageMapPrompt(templateHints, startPage, endPage, formInventoryHint),
+                  startPage,
+                  endPage,
+                  pagesText
+                ),
                 schema: PageMapChunkSchema,
                 maxTokens: budget.maxTokens,
                 taskKind: "extraction_page_map",
                 budgetDiagnostics: budget,
-                providerOptions: { ...activeProviderOptions, pdfBase64: pagesPdf }
+                providerOptions: doclingDocument ? { ...activeProviderOptions, doclingText: pagesText, doclingPageRange: { startPage, endPage } } : { ...activeProviderOptions, pdfBase64: pagesPdf }
               },
               {
                 fallback: {
@@ -8302,7 +8647,7 @@ function createExtractor(config) {
         }))
       ];
       onProgress?.(`Dispatching ${tasks.length} extractors...`);
-      const extractionPdfInput = await getPdfBase64ForExtraction();
+      const extractionPdfInput = doclingDocument ? void 0 : await getPdfBase64ForExtraction();
       const extractorResults = await Promise.all(
         tasks.map(
           (task) => extractorLimit(async () => {
@@ -8313,7 +8658,8 @@ function createExtractor(config) {
               memory,
               completedPageRangePdfCache,
               getPageRangePdf,
-              convertPdfToImages ? getPageImages : void 0
+              convertPdfToImages ? getPageImages : void 0,
+              doclingDocument ? getPageRangeText : void 0
             );
           })
         )
@@ -8345,7 +8691,8 @@ function createExtractor(config) {
             providerOptions: activeProviderOptions,
             pageRangeCache: completedPageRangePdfCache,
             getPageRangePdf,
-            getPageImages: convertPdfToImages ? getPageImages : void 0
+            getPageImages: convertPdfToImages ? getPageImages : void 0,
+            getPageRangeText: doclingDocument ? getPageRangeText : void 0
           });
           trackUsage(supplementaryResult.usage, {
             taskKind: "extraction_focused",
@@ -8381,6 +8728,7 @@ function createExtractor(config) {
           concurrency,
           getPageRangePdf,
           getPageImages: convertPdfToImages ? getPageImages : void 0,
+          getPageRangeText: doclingDocument ? getPageRangeText : void 0,
           providerOptions: activeProviderOptions,
           modelCapabilities,
           modelBudgetConstraints,
@@ -8429,12 +8777,12 @@ function createExtractor(config) {
           const reviewResponse = await safeGenerateObject(
             generateObject,
             {
-              prompt: buildReviewPrompt(template.required, extractedKeys, extractionSummary, pageMapSummary, extractorCatalog),
+              prompt: withFullDocumentTextContext(buildReviewPrompt(template.required, extractedKeys, extractionSummary, pageMapSummary, extractorCatalog)),
               schema: ReviewResultSchema,
               maxTokens: budget.maxTokens,
               taskKind: "extraction_review",
               budgetDiagnostics: budget,
-              providerOptions: await getFullPdfProviderOptions()
+              providerOptions: await getFullDocumentProviderOptions()
             },
             {
               fallback: {
@@ -8464,7 +8812,7 @@ function createExtractor(config) {
             break;
           }
           onProgress?.(`Review round ${round + 1}: dispatching ${reviewResponse.object.additionalTasks.length} follow-up extractors...`);
-          const extractionPdfInput = await getPdfBase64ForExtraction();
+          const extractionPdfInput = doclingDocument ? void 0 : await getPdfBase64ForExtraction();
           const followUpResults = await Promise.all(
             reviewResponse.object.additionalTasks.map(
               (task) => extractorLimit(async () => {
@@ -8474,7 +8822,8 @@ function createExtractor(config) {
                   memory,
                   completedPageRangePdfCache,
                   getPageRangePdf,
-                  convertPdfToImages ? getPageImages : void 0
+                  convertPdfToImages ? getPageImages : void 0,
+                  doclingDocument ? getPageRangeText : void 0
                 );
               })
             )
@@ -12473,6 +12822,7 @@ export {
   buildConfirmationSummaryPrompt,
   buildConversationMemoryGuidance,
   buildCoverageGapPrompt,
+  buildDoclingProviderOptions,
   buildFieldExplanationPrompt,
   buildFieldExtractionPrompt,
   buildFlatPdfMappingPrompt,
@@ -12514,12 +12864,16 @@ export {
   fillAcroForm,
   generateNextMessage,
   getAcroFormFields,
+  getDoclingPageRangeText,
   getExtractor,
   getFileIdentifier,
   getPdfPageCount,
   getTemplate,
+  isDoclingExtractionInput,
   isFileReference,
   mergeQuestionAnswers,
+  mergeSourceSpans,
+  normalizeDoclingDocument,
   normalizeForMatch,
   orderSourceEvidence,
   overlayTextOnPdf,