@claritylabs/cl-sdk 1.0.3 → 1.2.1

package/dist/index.mjs

@@ -2471,6 +2471,254 @@ async function overlayTextOnPdf(pdfBytes, overlays) {
   return await pdfDoc.save();
 }
 
+// src/extraction/docling.ts
+function isDoclingExtractionInput(input) {
+  return Boolean(
+    input && typeof input === "object" && input.kind === "docling_document" && input.document && typeof input.document === "object"
+  );
+}
+function normalizeDoclingDocument(document, options) {
+  const itemMap = buildItemMap(document);
+  const orderedRefs = getOrderedBodyRefs(document, itemMap);
+  const orderedItems = orderedRefs.length > 0 ? orderedRefs.map((ref) => itemMap.get(ref)).filter((item) => Boolean(item)) : getFallbackOrderedItems(document, itemMap);
+  const units = orderedItems.map(({ ref, item }) => normalizeItem(ref, item)).filter((unit) => Boolean(unit && unit.text.trim()));
+  const pageCount = inferPageCount(document, units);
+  const pageTexts = /* @__PURE__ */ new Map();
+  for (const unit of units) {
+    const page = clampPage(unit.pageStart ?? 1, pageCount);
+    pageTexts.set(page, appendText(pageTexts.get(page), unit.text));
+  }
+  const fullText = Array.from({ length: pageCount }, (_, index) => {
+    const pageNumber = index + 1;
+    const text = pageTexts.get(pageNumber)?.trim();
+    return text ? `Page ${pageNumber}
+${text}` : "";
+  }).filter(Boolean).join("\n\n");
+  const sourceKind = options.sourceKind ?? "policy_pdf";
+  const sourceSpans = units.map((unit, index) => {
+    const span = buildSourceSpan(
+      {
+        documentId: options.documentId,
+        sourceKind,
+        text: unit.text,
+        pageStart: unit.pageStart,
+        pageEnd: unit.pageEnd,
+        sectionId: unit.label,
+        metadata: {
+          sourceSystem: "docling",
+          sourceUnit: "docling_item",
+          doclingRef: unit.ref,
+          ...unit.label ? { doclingLabel: unit.label } : {}
+        }
+      },
+      index
+    );
+    return {
+      ...span,
+      kind: "plain_text",
+      bbox: unit.bboxes?.length ? unit.bboxes : void 0
+    };
+  });
+  return {
+    pageCount,
+    fullText,
+    pageTexts,
+    units,
+    sourceSpans
+  };
+}
+function getDoclingPageRangeText(normalized, startPage, endPage) {
+  const start = clampPage(startPage, normalized.pageCount);
+  const end = clampPage(endPage, normalized.pageCount);
+  const lines = [];
+  for (let page = start; page <= end; page++) {
+    const text = normalized.pageTexts.get(page)?.trim();
+    if (text) {
+      lines.push(`Page ${page}
+${text}`);
+    }
+  }
+  return lines.join("\n\n");
+}
+function buildDoclingProviderOptions(normalized, existingOptions) {
+  return {
+    ...existingOptions,
+    doclingText: normalized.fullText,
+    doclingPageCount: normalized.pageCount
+  };
+}
+function mergeSourceSpans(spans) {
+  const seen = /* @__PURE__ */ new Set();
+  const merged = [];
+  for (const span of spans) {
+    const key = [
+      span.documentId,
+      span.pageStart ?? span.location?.startPage ?? span.location?.page ?? "na",
+      span.pageEnd ?? span.location?.endPage ?? span.pageStart ?? "na",
+      span.sectionId ?? span.location?.fieldPath ?? "na",
+      span.textHash ?? sourceSpanTextHash(span.text)
+    ].join(":");
+    if (seen.has(key)) continue;
+    seen.add(key);
+    merged.push(span);
+  }
+  return merged;
+}
+function buildItemMap(document) {
+  const map = /* @__PURE__ */ new Map();
+  addItems(map, "#/texts", document.texts ?? []);
+  addItems(map, "#/tables", document.tables ?? []);
+  addItems(map, "#/key_value_items", document.key_value_items ?? document.keyValueItems ?? []);
+  addItems(map, "#/pictures", document.pictures ?? []);
+  return map;
+}
+function addItems(map, baseRef, items) {
+  items.forEach((item, index) => {
+    const ref = getSelfRef(item) ?? `${baseRef}/${index}`;
+    map.set(ref, { ref, item });
+  });
+}
+function getFallbackOrderedItems(document, itemMap) {
+  const refs = [
+    ...(document.texts ?? []).map((item, index) => getSelfRef(item) ?? `#/texts/${index}`),
+    ...(document.tables ?? []).map((item, index) => getSelfRef(item) ?? `#/tables/${index}`),
+    ...(document.key_value_items ?? document.keyValueItems ?? []).map((item, index) => getSelfRef(item) ?? `#/key_value_items/${index}`)
+  ];
+  return refs.map((ref) => itemMap.get(ref)).filter((item) => Boolean(item));
+}
+function getOrderedBodyRefs(document, itemMap) {
+  const groupMap = /* @__PURE__ */ new Map();
+  (document.groups ?? []).forEach((group, index) => {
+    groupMap.set(getSelfRef(group) ?? `#/groups/${index}`, group);
+  });
+  const refs = [];
+  const visited = /* @__PURE__ */ new Set();
+  const visitRef = (ref) => {
+    const itemEntry = itemMap.get(ref);
+    if (itemEntry) {
+      if (!visited.has(ref)) {
+        visited.add(ref);
+        refs.push(ref);
+      }
+      visitNode(itemEntry.item);
+      return;
+    }
+    visitNode(groupMap.get(ref));
+  };
+  const visitNode = (node) => {
+    for (const child of node?.children ?? []) {
+      const ref = getRef(child);
+      if (!ref) continue;
+      visitRef(ref);
+    }
+  };
+  visitNode(document.body);
+  return refs;
+}
+function normalizeItem(ref, item) {
+  const text = getItemText(item).trim();
+  if (!text) return void 0;
+  const pages = (item.prov ?? []).map((prov) => getPageNumber(prov)).filter((page) => typeof page === "number" && page > 0);
+  const pageStart = pages.length ? Math.min(...pages) : void 0;
+  const pageEnd = pages.length ? Math.max(...pages) : pageStart;
+  const bboxes = (item.prov ?? []).map((prov) => toSourceSpanBBox(prov)).filter((bbox) => Boolean(bbox));
+  return {
+    ref,
+    label: typeof item.label === "string" ? item.label : void 0,
+    text,
+    pageStart,
+    pageEnd,
+    bboxes: bboxes.length ? bboxes : void 0
+  };
+}
+function getItemText(item) {
+  if (typeof item.text === "string" && item.text.trim()) return item.text;
+  if (typeof item.orig === "string" && item.orig.trim()) return item.orig;
+  const table = tableToMarkdown(item.data);
+  if (table) return table;
+  return "";
+}
+function tableToMarkdown(data) {
+  const record = asRecord(data);
+  const cells = Array.isArray(record?.table_cells) ? record.table_cells : Array.isArray(record?.tableCells) ? record.tableCells : void 0;
+  if (!cells) return void 0;
+  const parsedCells = cells.map((cell) => asRecord(cell)).filter((cell) => Boolean(cell)).map((cell) => ({
+    row: firstNumber2([cell.start_row_offset, cell.row_header, cell.row, cell.rowIndex]) ?? 0,
+    col: firstNumber2([cell.start_col_offset, cell.col, cell.colIndex]) ?? 0,
+    text: firstString([cell.text, cell.orig, cell.content])
+  })).filter((cell) => cell.text);
+  if (parsedCells.length === 0) return void 0;
+  const maxRow = Math.max(...parsedCells.map((cell) => cell.row));
+  const maxCol = Math.max(...parsedCells.map((cell) => cell.col));
+  const rows = Array.from({ length: maxRow + 1 }, () => Array.from({ length: maxCol + 1 }, () => ""));
+  for (const cell of parsedCells) {
+    rows[cell.row][cell.col] = cell.text;
+  }
+  if (rows.length === 1) return rows[0].filter(Boolean).join(" | ");
+  const header = rows[0];
+  const separator = header.map(() => "---");
+  return [header, separator, ...rows.slice(1)].map((row) => `| ${row.map((value) => value.trim()).join(" | ")} |`).join("\n");
+}
+function inferPageCount(document, units) {
+  const pages = document.pages;
+  if (Array.isArray(pages)) return Math.max(1, pages.length);
+  if (pages && typeof pages === "object") {
+    const keys = Object.keys(pages);
+    const numericMax = Math.max(0, ...keys.map((key) => Number(key)).filter((value) => Number.isFinite(value)));
+    return Math.max(1, numericMax || keys.length);
+  }
+  return Math.max(1, ...units.flatMap((unit) => [unit.pageStart ?? 0, unit.pageEnd ?? 0]));
+}
+function getSelfRef(value) {
+  return value.self_ref ?? value.selfRef;
+}
+function getRef(value) {
+  if (typeof value === "string") return value;
+  return value.$ref ?? value.ref;
+}
+function getPageNumber(prov) {
+  return prov.page_no ?? prov.pageNo ?? prov.page;
+}
+function toSourceSpanBBox(prov) {
+  const page = getPageNumber(prov);
+  const bbox = asRecord(prov.bbox);
+  if (!page || !bbox) return void 0;
+  const x = firstNumber2([bbox.x, bbox.l, bbox.left]);
+  const y = firstNumber2([bbox.y, bbox.t, bbox.top]);
+  const width = firstNumber2([bbox.width]);
+  const height = firstNumber2([bbox.height]);
+  const right = firstNumber2([bbox.r, bbox.right]);
+  const bottom = firstNumber2([bbox.b, bbox.bottom]);
+  if (x == null || y == null) return void 0;
+  const resolvedWidth = width ?? (right != null ? right - x : void 0);
+  const resolvedHeight = height ?? (bottom != null ? bottom - y : void 0);
+  if (resolvedWidth == null || resolvedHeight == null) return void 0;
+  return { page, x, y, width: resolvedWidth, height: resolvedHeight };
+}
+function clampPage(page, pageCount) {
+  return Math.max(1, Math.min(pageCount, page));
+}
+function appendText(existing, next) {
+  return existing ? `${existing}
+
+${next}` : next;
+}
+function asRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function firstString(values) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return "";
+}
+function firstNumber2(values) {
+  for (const value of values) {
+    if (typeof value === "number" && Number.isFinite(value)) return value;
+  }
+  return void 0;
+}
+
 // src/extraction/extractor.ts
 function sourceSpansForPageRange(providerOptions, startPage, endPage) {
   const sourceSpans = providerOptions?.sourceSpans;
@@ -2512,20 +2760,38 @@ async function runExtractor(params) {
     generateObject,
     convertPdfToImages,
     maxTokens = 4096,
+    taskKind,
+    budgetDiagnostics,
     providerOptions,
     pageRangeCache
   } = params;
   const extractorProviderOptions = { ...providerOptions };
   let fullPrompt;
-  const needsPdfBase64 = convertPdfToImages && !params.getPageImages || !convertPdfToImages && !params.getPageRangePdf;
-  const pdfBase64 = needsPdfBase64 ? await pdfInputToBase64(pdfInput) : void 0;
-  if (convertPdfToImages) {
+  if (params.getPageRangeText) {
+    const pageText = await params.getPageRangeText(startPage, endPage);
+    extractorProviderOptions.doclingText = pageText;
+    extractorProviderOptions.doclingPageRange = { startPage, endPage };
+    fullPrompt = `${prompt}
+
+[Document pages ${startPage}-${endPage} are provided below as Docling-extracted text.]
+
+${pageText || "(No Docling text was available for this page range.)"}`;
+  } else if (convertPdfToImages) {
+    if (!pdfInput) {
+      throw new Error("pdfInput is required when extracting page images.");
+    }
+    const needsPdfBase64 = !params.getPageImages;
+    const pdfBase64 = needsPdfBase64 ? await pdfInputToBase64(pdfInput) : void 0;
     const images = params.getPageImages ? await params.getPageImages(startPage, endPage) : await convertPdfToImages(pdfBase64, startPage, endPage);
     extractorProviderOptions.images = images;
     fullPrompt = `${prompt}
 
 [Document pages ${startPage}-${endPage} are provided as images.]`;
   } else {
+    if (!pdfInput) {
+      throw new Error("pdfInput is required when extracting page PDFs.");
+    }
+    const pdfBase64 = params.getPageRangePdf ? void 0 : await pdfInputToBase64(pdfInput);
     const cacheKey = `${startPage}-${endPage}`;
     const cachedPagesPdf = pageRangeCache?.get(cacheKey);
     const pagesPdf = cachedPagesPdf ?? (params.getPageRangePdf ? await params.getPageRangePdf(startPage, endPage) : await extractPageRange(pdfBase64, startPage, endPage));
@@ -2545,6 +2811,8 @@ async function runExtractor(params) {
       prompt: fullPrompt,
       schema: strictSchema,
       maxTokens,
+      taskKind,
+      budgetDiagnostics,
       providerOptions: extractorProviderOptions
     })
   );
@@ -3524,6 +3792,8 @@ async function formatDocumentContent(doc, generateText, options) {
           () => generateText({
             prompt,
             maxTokens: options?.maxTokens ?? 16384,
+            taskKind: options?.taskKind,
+            budgetDiagnostics: options?.budgetDiagnostics,
             providerOptions: options?.providerOptions
           })
         );
@@ -3561,7 +3831,7 @@ function formatAddress(addr) {
 function asRecordArray(value) {
   return Array.isArray(value) ? value.filter((item) => Boolean(item) && typeof item === "object" && !Array.isArray(item)) : [];
 }
-function firstString(item, keys) {
+function firstString2(item, keys) {
   for (const key of keys) {
     const value = item[key];
     if (typeof value === "string" && value.trim()) return value;
@@ -3918,32 +4188,32 @@ ${exc.content}`.trim(), {
     );
   });
   asRecordArray(extendedDoc.definitions).forEach((definition, i) => {
-    const term = firstString(definition, ["term", "name", "title"]) ?? `Definition ${i + 1}`;
-    const body = firstString(definition, ["definition", "content", "text", "meaning"]);
+    const term = firstString2(definition, ["term", "name", "title"]) ?? `Definition ${i + 1}`;
+    const body = firstString2(definition, ["definition", "content", "text", "meaning"]);
     pushChunk(
       `definition:${i}`,
       "definition",
       lines([
         `Definition: ${term}`,
         body,
-        firstString(definition, ["originalContent", "source"]) ? `Source: ${firstString(definition, ["originalContent", "source"])}` : null
+        firstString2(definition, ["originalContent", "source"]) ? `Source: ${firstString2(definition, ["originalContent", "source"])}` : null
       ]),
       {
         term,
-        formNumber: firstString(definition, ["formNumber"]),
-        formTitle: firstString(definition, ["formTitle"]),
+        formNumber: firstString2(definition, ["formNumber"]),
+        formTitle: firstString2(definition, ["formTitle"]),
         pageNumber: typeof definition.pageNumber === "number" ? definition.pageNumber : void 0,
-        sectionRef: firstString(definition, ["sectionRef", "sectionTitle"]),
+        sectionRef: firstString2(definition, ["sectionRef", "sectionTitle"]),
         documentType: doc.type
       }
     );
   });
   const coveredReasons = asRecordArray(extendedDoc.coveredReasons ?? extendedDoc.covered_reasons);
   coveredReasons.forEach((coveredReason, i) => {
-    const title = firstString(coveredReason, ["title", "name", "reason", "peril", "cause"]) ?? `Covered Reason ${i + 1}`;
-    const coverageName = firstString(coveredReason, ["coverageName", "coverage", "coveragePart"]);
-    const reasonNumber = firstString(coveredReason, ["reasonNumber", "number"]);
-    const body = firstString(coveredReason, ["content", "description", "text", "coverageGrant"]);
+    const title = firstString2(coveredReason, ["title", "name", "reason", "peril", "cause"]) ?? `Covered Reason ${i + 1}`;
+    const coverageName = firstString2(coveredReason, ["coverageName", "coverage", "coveragePart"]);
+    const reasonNumber = firstString2(coveredReason, ["reasonNumber", "number"]);
+    const body = firstString2(coveredReason, ["content", "description", "text", "coverageGrant"]);
     pushChunk(
       `covered_reason:${i}`,
       "covered_reason",
@@ -3952,16 +4222,16 @@ ${exc.content}`.trim(), {
         reasonNumber ? `Reason Number: ${reasonNumber}` : null,
         `Covered Reason: ${title}`,
         body,
-        firstString(coveredReason, ["originalContent", "source"]) ? `Source: ${firstString(coveredReason, ["originalContent", "source"])}` : null
+        firstString2(coveredReason, ["originalContent", "source"]) ? `Source: ${firstString2(coveredReason, ["originalContent", "source"])}` : null
       ]),
       {
         coverageName,
         reasonNumber,
         title,
-        formNumber: firstString(coveredReason, ["formNumber"]),
-        formTitle: firstString(coveredReason, ["formTitle"]),
+        formNumber: firstString2(coveredReason, ["formNumber"]),
+        formTitle: firstString2(coveredReason, ["formTitle"]),
         pageNumber: typeof coveredReason.pageNumber === "number" ? coveredReason.pageNumber : void 0,
-        sectionRef: firstString(coveredReason, ["sectionRef", "sectionTitle"]),
+        sectionRef: firstString2(coveredReason, ["sectionRef", "sectionTitle"]),
         documentType: doc.type
       }
     );
@@ -3981,10 +4251,10 @@ ${exc.content}`.trim(), {
           reasonNumber,
           title,
           conditionIndex,
-          formNumber: firstString(coveredReason, ["formNumber"]),
-          formTitle: firstString(coveredReason, ["formTitle"]),
+          formNumber: firstString2(coveredReason, ["formNumber"]),
+          formTitle: firstString2(coveredReason, ["formTitle"]),
           pageNumber: typeof coveredReason.pageNumber === "number" ? coveredReason.pageNumber : void 0,
-          sectionRef: firstString(coveredReason, ["sectionRef", "sectionTitle"]),
+          sectionRef: firstString2(coveredReason, ["sectionRef", "sectionTitle"]),
           documentType: doc.type
         }
       );
@@ -6455,21 +6725,21 @@ Return JSON only.`;
 }
 
 // src/prompts/extractors/index.ts
-function asRecord(data) {
+function asRecord2(data) {
   return data && typeof data === "object" ? data : void 0;
 }
 function getSections2(data) {
-  const sections = asRecord(data)?.sections;
+  const sections = asRecord2(data)?.sections;
   return Array.isArray(sections) ? sections : [];
 }
 function isCoveredReasonsEmpty(data) {
-  const record = asRecord(data);
+  const record = asRecord2(data);
   if (!record) return true;
   const coveredReasons = Array.isArray(record.coveredReasons) ? record.coveredReasons : Array.isArray(record.covered_reasons) ? record.covered_reasons : [];
   return coveredReasons.length === 0;
 }
 function isDefinitionsEmpty(data) {
-  const definitions = asRecord(data)?.definitions;
+  const definitions = asRecord2(data)?.definitions;
   return !Array.isArray(definitions) || definitions.length === 0;
 }
 function sectionLooksLikeCoveredReason(section) {
@@ -6703,6 +6973,14 @@ function decideReferentialResolutionAction(params) {
 }
 
 // src/extraction/resolve-referential.ts
+function formatDoclingTextContext(providerOptions) {
+  const doclingText = providerOptions?.doclingText;
+  if (typeof doclingText !== "string" || !doclingText.trim()) return "";
+  return `
+
+DOCLING DOCUMENT TEXT:
+${doclingText}`;
+}
 function parseReferenceTarget(text) {
   if (typeof text !== "string") return void 0;
   const normalized = text.trim();
@@ -6784,10 +7062,12 @@ Return the page range (1-indexed) where this section is located. If the section
 
 If you cannot find the section, return startPage: 0 and endPage: 0.
 
-Return JSON only.`,
+Return JSON only.${formatDoclingTextContext(providerOptions)}`,
         schema: PageLocationSchema,
         maxTokens: budget.maxTokens,
-        providerOptions: await buildPdfProviderOptions(pdfInput, providerOptions)
+        taskKind: "extraction_referential_lookup",
+        budgetDiagnostics: budget,
+        providerOptions: pdfInput ? await buildPdfProviderOptions(pdfInput, providerOptions) : providerOptions
       },
       {
         fallback: { startPage: 0, endPage: 0 },
@@ -6821,6 +7101,7 @@ async function resolveReferentialCoverages(params) {
     convertPdfToImages,
     getPageRangePdf,
     getPageImages,
+    getPageRangeText,
     concurrency = 2,
     providerOptions,
     modelCapabilities,
@@ -6932,7 +7213,10 @@ async function resolveReferentialCoverages(params) {
             convertPdfToImages,
             getPageRangePdf,
             getPageImages,
+            getPageRangeText,
             maxTokens: budget.maxTokens,
+            taskKind: "extraction_referential_lookup",
+            budgetDiagnostics: budget,
             providerOptions
           });
           trackUsage(result.usage);
@@ -7025,6 +7309,7 @@ async function runFocusedExtractorWithFallback(params) {
     pageRangeCache,
     getPageRangePdf,
     getPageImages,
+    getPageRangeText,
     trackUsage,
     resolveBudget,
     log
@@ -7049,10 +7334,13 @@ async function runFocusedExtractorWithFallback(params) {
       generateObject,
       convertPdfToImages,
       maxTokens: budget.maxTokens,
+      taskKind,
+      budgetDiagnostics: budget,
       providerOptions,
       pageRangeCache,
       getPageRangePdf,
-      getPageImages
+      getPageImages,
+      getPageRangeText
     });
     trackUsage(result.usage, {
       taskKind,
@@ -7092,10 +7380,13 @@ async function runFocusedExtractorWithFallback(params) {
       generateObject,
       convertPdfToImages,
       maxTokens: budget.maxTokens,
+      taskKind,
+      budgetDiagnostics: budget,
       providerOptions,
       pageRangeCache,
       getPageRangePdf,
-      getPageImages
+      getPageImages,
+      getPageRangeText
     });
     trackUsage(fallbackResult.usage, {
       taskKind,
@@ -7939,7 +8230,7 @@ function createExtractor(config) {
     }
     return lines.length > 0 ? lines.join("\n") : "";
   }
-  async function runFocusedExtractorTask(task, pdfInput, memory, pageRangeCache, getPageRangePdf, getPageImages) {
+  async function runFocusedExtractorTask(task, pdfInput, memory, pageRangeCache, getPageRangePdf, getPageImages, getPageRangeText) {
     if (task.extractorName === "supplementary") {
       const alreadyExtractedSummary = buildAlreadyExtractedSummary(memory);
       const budget = resolveBudget("extraction_focused", 4096);
@@ -7954,10 +8245,13 @@ function createExtractor(config) {
         generateObject,
         convertPdfToImages,
         maxTokens: budget.maxTokens,
+        taskKind: "extraction_focused",
+        budgetDiagnostics: budget,
         providerOptions: activeProviderOptions,
         pageRangeCache,
         getPageRangePdf,
-        getPageImages
+        getPageImages,
+        getPageRangeText
       });
       trackUsage(result.usage, {
         taskKind: "extraction_focused",
@@ -7976,6 +8270,7 @@ function createExtractor(config) {
       pageRangeCache,
       getPageRangePdf,
       getPageImages,
+      getPageRangeText,
       trackUsage,
       resolveBudget,
       log
@@ -7991,8 +8286,14 @@ function createExtractor(config) {
     if (extractorPages.size === 0) return "No page assignments available.";
     return [...extractorPages.entries()].map(([extractorName, pages]) => `${extractorName}: ${pages.length} page(s), pages ${pages.join(", ")}`).join("\n");
   }
-  async function extract(pdfInput, documentId, options) {
+  async function extract(input, documentId, options) {
     const id = documentId ?? `doc-${Date.now()}`;
+    const isDoclingInput = isDoclingExtractionInput(input);
+    const pdfInput = isDoclingInput ? void 0 : input;
+    const doclingDocument = isDoclingInput ? normalizeDoclingDocument(input.document, {
+      documentId: id,
+      sourceKind: input.sourceKind
+    }) : void 0;
     const memory = /* @__PURE__ */ new Map();
     totalUsage = { inputTokens: 0, outputTokens: 0 };
     modelCalls = 0;
@@ -8002,7 +8303,10 @@ function createExtractor(config) {
       modelCalls: [],
       totalModelCallDurationMs: 0
     };
-    const sourceSpans = options?.sourceSpans ?? [];
+    const sourceSpans = mergeSourceSpans([
+      ...doclingDocument?.sourceSpans ?? [],
+      ...options?.sourceSpans ?? []
+    ]);
     const sourceChunks = sourceSpans.length ? chunkSourceSpans(sourceSpans) : [];
     activeProviderOptions = sourceSpans.length ? { ...providerOptions, sourceSpans, sourceChunks } : providerOptions;
     if (sourceStore && sourceSpans.length > 0) {
@@ -8031,24 +8335,40 @@ function createExtractor(config) {
     let fullPdfProviderOptionsPromise;
     let pageCountPromise;
     async function getPdfBase64ForExtraction() {
+      if (!pdfInput) {
+        throw new Error("PDF input is not available for Docling extraction.");
+      }
       if (pdfBase64Cache === void 0) {
         pdfBase64Cache = await pdfInputToBase64(pdfInput);
       }
       return pdfBase64Cache;
     }
     async function getCachedPageCount() {
+      if (doclingDocument) return doclingDocument.pageCount;
+      if (!pdfInput) {
+        throw new Error("PDF input is required to read page count.");
+      }
       if (!pageCountPromise) {
         pageCountPromise = getPdfSlicer().then((slicer) => slicer.getPageCount()).catch(() => getPdfPageCount(pdfInput));
       }
       return pageCountPromise;
     }
-    async function getFullPdfProviderOptions() {
+    async function getFullDocumentProviderOptions() {
+      if (doclingDocument) {
+        return buildDoclingProviderOptions(doclingDocument, activeProviderOptions);
+      }
+      if (!pdfInput) {
+        return activeProviderOptions ?? {};
+      }
       if (!fullPdfProviderOptionsPromise) {
         fullPdfProviderOptionsPromise = buildPdfProviderOptions(pdfInput, activeProviderOptions);
       }
       return fullPdfProviderOptionsPromise;
     }
     async function getPdfSlicer() {
+      if (!pdfInput) {
+        throw new Error("PDF input is not available for Docling extraction.");
+      }
       if (!pdfSlicerPromise) {
         pdfSlicerPromise = createPdfPageSlicer(pdfInput);
       }
@@ -8087,6 +8407,23 @@ function createExtractor(config) {
       pageRangeImageCache.set(cacheKey, promise);
       return promise;
     }
+    async function getPageRangeText(startPage, endPage) {
+      return doclingDocument ? getDoclingPageRangeText(doclingDocument, startPage, endPage) : "";
+    }
+    function withFullDocumentTextContext(prompt) {
+      if (!doclingDocument) return prompt;
+      return `${prompt}
+
+DOCLING DOCUMENT TEXT:
+${doclingDocument.fullText}`;
+    }
+    function withPageRangeTextContext(prompt, startPage, endPage, pageText) {
+      if (!doclingDocument) return prompt;
+      return `${prompt}
+
+DOCLING DOCUMENT PAGES ${startPage}-${endPage}:
+${pageText || "(No Docling text was available for this page range.)"}`;
+    }
     let classifyResult;
     if (resumed?.classifyResult && pipelineCtx.isPhaseComplete("classify")) {
       classifyResult = resumed.classifyResult;
@@ -8099,10 +8436,12 @@ function createExtractor(config) {
       const classifyResponse = await safeGenerateObject(
         generateObject,
         {
-          prompt: buildClassifyPrompt(),
+          prompt: withFullDocumentTextContext(buildClassifyPrompt()),
           schema: ClassifyResultSchema,
           maxTokens: budget.maxTokens,
-          providerOptions: await getFullPdfProviderOptions()
+          taskKind: "extraction_classify",
+          budgetDiagnostics: budget,
+          providerOptions: await getFullDocumentProviderOptions()
         },
         {
           fallback: { documentType: "policy", policyTypes: ["other"], confidence: 0 },
@@ -8147,10 +8486,12 @@ function createExtractor(config) {
       const formInventoryResponse = await safeGenerateObject(
         generateObject,
         {
-          prompt: buildFormInventoryPrompt(templateHints),
+          prompt: withFullDocumentTextContext(buildFormInventoryPrompt(templateHints)),
           schema: FormInventorySchema,
           maxTokens: budget.maxTokens,
-          providerOptions: await getFullPdfProviderOptions()
+          taskKind: "extraction_form_inventory",
+          budgetDiagnostics: budget,
+          providerOptions: await getFullDocumentProviderOptions()
         },
         {
           fallback: { forms: [] },
@@ -8193,16 +8534,24 @@ function createExtractor(config) {
       const pageMapResults = await Promise.all(
         pageMapChunks.map(
           ({ startPage, endPage }) => pageMapLimit(async () => {
-            const pagesPdf = await getPageRangePdf(startPage, endPage);
+            const pagesPdf = doclingDocument ? void 0 : await getPageRangePdf(startPage, endPage);
+            const pagesText = doclingDocument ? await getPageRangeText(startPage, endPage) : "";
             const budget = resolveBudget("extraction_page_map", 2048);
             const startedAt = Date.now();
             const mapResponse = await safeGenerateObject(
               generateObject,
               {
-                prompt: buildPageMapPrompt(templateHints, startPage, endPage, formInventoryHint),
+                prompt: withPageRangeTextContext(
+                  buildPageMapPrompt(templateHints, startPage, endPage, formInventoryHint),
+                  startPage,
+                  endPage,
+                  pagesText
+                ),
                 schema: PageMapChunkSchema,
                 maxTokens: budget.maxTokens,
-                providerOptions: { ...activeProviderOptions, pdfBase64: pagesPdf }
+                taskKind: "extraction_page_map",
+                budgetDiagnostics: budget,
+                providerOptions: doclingDocument ? { ...activeProviderOptions, doclingText: pagesText, doclingPageRange: { startPage, endPage } } : { ...activeProviderOptions, pdfBase64: pagesPdf }
               },
               {
                 fallback: {
@@ -8280,7 +8629,7 @@ function createExtractor(config) {
         }))
       ];
       onProgress?.(`Dispatching ${tasks.length} extractors...`);
-      const extractionPdfInput = await getPdfBase64ForExtraction();
+      const extractionPdfInput = doclingDocument ? void 0 : await getPdfBase64ForExtraction();
       const extractorResults = await Promise.all(
         tasks.map(
           (task) => extractorLimit(async () => {
@@ -8291,7 +8640,8 @@ function createExtractor(config) {
               memory,
               completedPageRangePdfCache,
               getPageRangePdf,
-              convertPdfToImages ? getPageImages : void 0
+              convertPdfToImages ? getPageImages : void 0,
+              doclingDocument ? getPageRangeText : void 0
             );
           })
         )
@@ -8318,10 +8668,13 @@ function createExtractor(config) {
             generateObject,
             convertPdfToImages,
             maxTokens: budget.maxTokens,
+            taskKind: "extraction_focused",
+            budgetDiagnostics: budget,
             providerOptions: activeProviderOptions,
             pageRangeCache: completedPageRangePdfCache,
             getPageRangePdf,
-            getPageImages: convertPdfToImages ? getPageImages : void 0
+            getPageImages: convertPdfToImages ? getPageImages : void 0,
+            getPageRangeText: doclingDocument ? getPageRangeText : void 0
           });
           trackUsage(supplementaryResult.usage, {
             taskKind: "extraction_focused",
@@ -8357,6 +8710,7 @@ function createExtractor(config) {
           concurrency,
           getPageRangePdf,
           getPageImages: convertPdfToImages ? getPageImages : void 0,
+          getPageRangeText: doclingDocument ? getPageRangeText : void 0,
           providerOptions: activeProviderOptions,
           modelCapabilities,
           modelBudgetConstraints,
@@ -8405,13 +8759,22 @@ function createExtractor(config) {
           const reviewResponse = await safeGenerateObject(
             generateObject,
             {
-              prompt: buildReviewPrompt(template.required, extractedKeys, extractionSummary, pageMapSummary, extractorCatalog),
+              prompt: withFullDocumentTextContext(buildReviewPrompt(template.required, extractedKeys, extractionSummary, pageMapSummary, extractorCatalog)),
               schema: ReviewResultSchema,
               maxTokens: budget.maxTokens,
-              providerOptions: await getFullPdfProviderOptions()
+              taskKind: "extraction_review",
+              budgetDiagnostics: budget,
+              providerOptions: await getFullDocumentProviderOptions()
             },
             {
-              fallback: { complete: true, missingFields: [], qualityIssues: [], additionalTasks: [] },
+              fallback: {
+                complete: false,
+                missingFields: ["llm_review_unavailable"],
+                qualityIssues: [
+                  "LLM extraction review failed; deterministic review was used and the result needs review."
+                ],
+                additionalTasks: []
+              },
               log,
               onError: (err, attempt) => log?.(`Review round ${round + 1} attempt ${attempt + 1} failed: ${err}`)
             }
@@ -8431,7 +8794,7 @@ function createExtractor(config) {
             break;
           }
           onProgress?.(`Review round ${round + 1}: dispatching ${reviewResponse.object.additionalTasks.length} follow-up extractors...`);
-          const extractionPdfInput = await getPdfBase64ForExtraction();
+          const extractionPdfInput = doclingDocument ? void 0 : await getPdfBase64ForExtraction();
           const followUpResults = await Promise.all(
             reviewResponse.object.additionalTasks.map(
               (task) => extractorLimit(async () => {
@@ -8441,7 +8804,8 @@ function createExtractor(config) {
                   memory,
                   completedPageRangePdfCache,
                   getPageRangePdf,
-                  convertPdfToImages ? getPageImages : void 0
+                  convertPdfToImages ? getPageImages : void 0,
+                  doclingDocument ? getPageRangeText : void 0
                 );
               })
             )
@@ -8512,6 +8876,8 @@ function createExtractor(config) {
             prompt: buildSummaryPrompt(document),
             schema: SummaryResultSchema,
             maxTokens: budget.maxTokens,
+            taskKind: "extraction_summary",
+            budgetDiagnostics: budget,
             providerOptions: activeProviderOptions
           },
           {
@@ -8539,6 +8905,8 @@ function createExtractor(config) {
     const formatResult = await formatDocumentContent(document, generateText, {
       providerOptions: activeProviderOptions,
       maxTokens: formatBudget.maxTokens,
+      taskKind: "extraction_format",
+      budgetDiagnostics: formatBudget,
       concurrency: formatConcurrency ?? concurrency,
       onProgress,
       log
@@ -8941,6 +9309,7 @@ async function classifyApplication(pdfContent, generateObject, providerOptions,
 Analyze the attached insurance document. If text source units are provided in provider options, use them as supporting context. Do not infer from base64 text.`,
       schema: ApplicationClassifyResultSchema,
       maxTokens,
+      taskKind: "application_classify",
       providerOptions: {
         ...providerOptions,
         pdfBase64: providerOptions?.pdfBase64 ?? pdfContent
@@ -9043,6 +9412,7 @@ Extract fields from the attached application PDF. Use provider-supplied source u
       prompt,
       schema: FieldExtractionResultSchema,
       maxTokens,
+      taskKind: "application_extract_fields",
       providerOptions: {
         ...providerOptions,
         pdfBase64: providerOptions?.pdfBase64 ?? pdfContent
@@ -9096,6 +9466,7 @@ async function autoFillFromContext(fields, orgContext, generateObject, providerO
       prompt,
       schema: AutoFillResultSchema,
       maxTokens,
+      taskKind: "application_auto_fill",
       providerOptions
     })
   );
@@ -9166,6 +9537,7 @@ async function batchQuestions(unfilledFields, generateObject, providerOptions, m
       prompt,
       schema: QuestionBatchResultSchema,
       maxTokens,
+      taskKind: "application_batch",
       providerOptions
     })
   );
@@ -9217,6 +9589,7 @@ async function classifyReplyIntent(fields, replyText, generateObject, providerOp
       prompt,
       schema: ReplyIntentSchema,
       maxTokens,
+      taskKind: "application_classify",
       providerOptions
     })
   );
@@ -9276,6 +9649,7 @@ async function parseAnswers(fields, replyText, generateObject, providerOptions,
       prompt,
       schema: AnswerParsingResultSchema,
       maxTokens,
+      taskKind: "application_parse_answers",
       providerOptions
     })
   );
@@ -9405,6 +9779,7 @@ async function fillFromLookup(requests, targetFields, availableData, generateObj
       prompt,
       schema: LookupFillResultSchema,
       maxTokens,
+      taskKind: "application_lookup",
       providerOptions
     })
   );
@@ -9487,6 +9862,7 @@ async function generateBatchEmail(batchFields, batchIndex, totalBatches, opts, g
     () => generateText({
       prompt,
       maxTokens,
+      taskKind: "application_email",
       providerOptions
     })
   );
@@ -10009,11 +10385,14 @@ function createApplicationPipeline(config) {
     }
     if (replyPlan.answerQuestion && intent.questionText) {
       try {
+        const budget = resolveBudget("application_email", 512);
         const { text, usage } = await generateText({
           prompt: `The user is filling out an insurance application and asked: "${intent.questionText}"
 
 Provide a brief, helpful explanation (2-3 sentences). End with "Just reply with the answer when you're ready and I'll fill it in."`,
-          maxTokens: resolveBudget("application_email", 512).maxTokens,
+          maxTokens: budget.maxTokens,
+          taskKind: "application_email",
+          budgetDiagnostics: budget,
           providerOptions
         });
         trackUsage(usage);
@@ -10138,6 +10517,7 @@ ${emailText}`;
     if (!state) throw new Error(`Application ${applicationId} not found`);
     const filledFields = state.fields.filter((f) => f.value);
     const fieldSummary = filledFields.map((f) => `${f.section} > ${f.label}: ${f.value} (source: ${f.source ?? "unknown"})`).join("\n");
+    const budget = resolveBudget("application_email", 4096);
     const { text, usage } = await generateText({
       prompt: `Format these filled insurance application fields as a clean confirmation summary for the user to review. Group by section, show each field as "Label: Value". End with a note asking them to confirm or request changes.
 
@@ -10145,7 +10525,9 @@ Application: ${state.title ?? "Insurance Application"}
 
 Fields:
 ${fieldSummary}`,
-      maxTokens: resolveBudget("application_email", 4096).maxTokens,
+      maxTokens: budget.maxTokens,
+      taskKind: "application_email",
+      budgetDiagnostics: budget,
       providerOptions
     });
     trackUsage(usage);
@@ -10619,6 +11001,8 @@ ${e.text}`;
       prompt,
       schema: SubAnswerSchema,
       maxTokens: budget.maxTokens,
+      taskKind: "query_reason",
+      budgetDiagnostics: budget,
       providerOptions
     })
   );
@@ -10842,6 +11226,8 @@ async function verify(originalQuestion, subAnswers, allEvidence, config) {
       prompt,
       schema: VerifyResultSchema,
       maxTokens: budget.maxTokens,
+      taskKind: "query_verify",
+      budgetDiagnostics: budget,
       providerOptions
     })
   );
@@ -10984,6 +11370,8 @@ async function interpretAttachments(params) {
         prompt,
         schema: AttachmentInterpretationSchema,
         maxTokens: budget.maxTokens,
+        taskKind: "query_attachment",
+        budgetDiagnostics: budget,
         providerOptions: buildAttachmentProviderOptions(attachment, providerOptions)
       },
       {
@@ -11321,6 +11709,8 @@ function createQueryAgent(config) {
         prompt,
         schema: QueryClassifyResultSchema,
         maxTokens: budget.maxTokens,
+        taskKind: "query_classify",
+        budgetDiagnostics: budget,
         providerOptions
       },
       {
@@ -11372,6 +11762,8 @@ function createQueryAgent(config) {
         prompt,
         schema: QueryResultSchema,
         maxTokens: budget.maxTokens,
+        taskKind: "query_respond",
+        budgetDiagnostics: budget,
         providerOptions
       },
       {
@@ -11467,6 +11859,8 @@ function createPceAgent(config = {}) {
           prompt: buildPceNormalizePrompt({ requestText: input.requestText, evidenceSources }),
           schema: PceNormalizationResultSchema,
           maxTokens: budget.maxTokens,
+          taskKind: "pce_impact_analysis",
+          budgetDiagnostics: budget,
           providerOptions: config.providerOptions
         },
         { fallback, maxRetries: 1, log: config.log }
@@ -11528,6 +11922,8 @@ function createPceAgent(config = {}) {
           }),
           schema: ReplyAnswersSchema,
           maxTokens: budget.maxTokens,
+          taskKind: "pce_reply_parse",
+          budgetDiagnostics: budget,
           providerOptions: config.providerOptions
         },
         { fallback: { answers }, maxRetries: 1, log: config.log }
@@ -12408,6 +12804,7 @@ export {
   buildConfirmationSummaryPrompt,
   buildConversationMemoryGuidance,
   buildCoverageGapPrompt,
+  buildDoclingProviderOptions,
   buildFieldExplanationPrompt,
   buildFieldExtractionPrompt,
   buildFlatPdfMappingPrompt,
@@ -12449,12 +12846,16 @@ export {
   fillAcroForm,
   generateNextMessage,
   getAcroFormFields,
+  getDoclingPageRangeText,
   getExtractor,
   getFileIdentifier,
   getPdfPageCount,
   getTemplate,
+  isDoclingExtractionInput,
   isFileReference,
   mergeQuestionAnswers,
+  mergeSourceSpans,
+  normalizeDoclingDocument,
   normalizeForMatch,
   orderSourceEvidence,
   overlayTextOnPdf,