@claritylabs/cl-sdk 0.15.0 → 0.16.1

package/dist/index.js

@@ -1,7 +1,9 @@
 "use strict";
+var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __export = (target, all) => {
   for (var name in all)
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
   }
   return to;
 };
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/index.ts
@@ -221,6 +231,7 @@ __export(index_exports, {
   buildIntentPrompt: () => buildIntentPrompt,
   buildInterpretAttachmentPrompt: () => buildInterpretAttachmentPrompt,
   buildLookupFillPrompt: () => buildLookupFillPrompt,
+  buildPdfProviderOptions: () => buildPdfProviderOptions,
   buildQueryClassifyPrompt: () => buildQueryClassifyPrompt,
   buildQuestionBatchPrompt: () => buildQuestionBatchPrompt,
   buildQuotesPoliciesPrompt: () => buildQuotesPoliciesPrompt,
@@ -238,10 +249,14 @@ __export(index_exports, {
   fillAcroForm: () => fillAcroForm,
   getAcroFormFields: () => getAcroFormFields,
   getExtractor: () => getExtractor,
+  getFileIdentifier: () => getFileIdentifier,
   getPdfPageCount: () => getPdfPageCount,
   getTemplate: () => getTemplate,
+  isFileReference: () => isFileReference,
   overlayTextOnPdf: () => overlayTextOnPdf,
   pLimit: () => pLimit,
+  pdfInputToBase64: () => pdfInputToBase64,
+  pdfInputToBytes: () => pdfInputToBytes,
   safeGenerateObject: () => safeGenerateObject,
   sanitizeNulls: () => sanitizeNulls,
   stripFences: () => stripFences,
@@ -1667,34 +1682,134 @@ var CONTEXT_KEY_MAP = [
 
 // src/extraction/pdf.ts
 var import_pdf_lib = require("pdf-lib");
-async function extractPageRange(pdfBase64, startPage, endPage) {
-  const srcBytes = typeof Buffer !== "undefined" ? Buffer.from(pdfBase64, "base64") : Uint8Array.from(atob(pdfBase64), (c) => c.charCodeAt(0));
+function isFileIdRef(input) {
+  return typeof input === "object" && input !== null && "fileId" in input;
+}
+function isUrl(input) {
+  return input instanceof URL;
+}
+function isBytes(input) {
+  return input instanceof Uint8Array;
+}
+async function pdfInputToBytes(input) {
+  if (isFileIdRef(input)) {
+    throw new Error(
+      "Cannot convert fileId reference to bytes. Pass the fileId directly to your provider callback instead."
+    );
+  }
+  if (isUrl(input)) {
+    if (input.protocol === "file:") {
+      if (typeof process !== "undefined" && process.versions?.node) {
+        const fs = await import("fs/promises");
+        const buffer = await fs.readFile(input.pathname);
+        return new Uint8Array(buffer);
+      }
+      throw new Error("File URLs not supported in browser environment");
+    }
+    const response = await fetch(input.toString());
+    if (!response.ok) {
+      throw new Error(`Failed to fetch PDF: ${response.status} ${response.statusText}`);
+    }
+    const arrayBuffer = await response.arrayBuffer();
+    return new Uint8Array(arrayBuffer);
+  }
+  if (isBytes(input)) {
+    return input;
+  }
+  if (typeof Buffer !== "undefined") {
+    return new Uint8Array(Buffer.from(input, "base64"));
+  }
+  return Uint8Array.from(atob(input), (c) => c.charCodeAt(0));
+}
+async function pdfInputToBase64(input) {
+  if (isFileIdRef(input)) {
+    throw new Error(
+      "Cannot convert fileId reference to base64. Pass the fileId directly to your provider callback instead."
+    );
+  }
+  if (isUrl(input)) {
+    const bytes = await pdfInputToBytes(input);
+    return bytesToBase64(bytes);
+  }
+  if (isBytes(input)) {
+    return bytesToBase64(input);
+  }
+  return input;
+}
+function bytesToBase64(bytes) {
+  if (typeof Buffer !== "undefined") {
+    return Buffer.from(bytes).toString("base64");
+  }
+  let binary = "";
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary);
+}
+function isFileReference(input) {
+  return isFileIdRef(input) || isUrl(input);
+}
+function getFileIdentifier(input) {
+  if (isFileIdRef(input)) {
+    return { fileId: input.fileId };
+  }
+  if (isUrl(input)) {
+    return { url: input.toString() };
+  }
+  return void 0;
+}
+async function getPdfPageCount(input) {
+  const bytes = await pdfInputToBytes(input);
+  const doc = await import_pdf_lib.PDFDocument.load(bytes, { ignoreEncryption: true });
+  return doc.getPageCount();
+}
+async function extractPageRange(input, startPage, endPage) {
+  if (isFileIdRef(input)) {
+    throw new Error(
+      "Cannot extract page range from fileId reference. The provider must handle fileId inputs directly or you must pass the full PDF as base64/bytes."
+    );
+  }
+  if (isUrl(input) && (input.protocol === "http:" || input.protocol === "https:")) {
+    throw new Error(
+      "Cannot extract page range from remote URL. Either pass the full PDF as base64/bytes, or download it first."
+    );
+  }
+  const srcBytes = await pdfInputToBytes(input);
   const srcDoc = await import_pdf_lib.PDFDocument.load(srcBytes, { ignoreEncryption: true });
   const totalPages = srcDoc.getPageCount();
   const start = Math.max(startPage - 1, 0);
   const end = Math.min(endPage, totalPages) - 1;
   if (start === 0 && end >= totalPages - 1) {
-    return pdfBase64;
+    if (isBytes(input)) {
+      return bytesToBase64(input);
+    }
+    if (typeof input === "string") {
+      return input;
+    }
+    return bytesToBase64(srcBytes);
   }
   const newDoc = await import_pdf_lib.PDFDocument.create();
   const indices = Array.from({ length: end - start + 1 }, (_, i) => start + i);
   const pages = await newDoc.copyPages(srcDoc, indices);
   pages.forEach((page) => newDoc.addPage(page));
   const bytes = await newDoc.save();
-  if (typeof Buffer !== "undefined") {
-    return Buffer.from(bytes).toString("base64");
+  return bytesToBase64(new Uint8Array(bytes));
+}
+async function buildPdfProviderOptions(input, existingOptions) {
+  const options = { ...existingOptions };
+  if (isFileIdRef(input)) {
+    options.fileId = input.fileId;
+    if (input.mimeType) {
+      options.fileMimeType = input.mimeType;
+    }
+    return options;
   }
-  let binary = "";
-  const uint8 = new Uint8Array(bytes);
-  for (let i = 0; i < uint8.length; i++) {
-    binary += String.fromCharCode(uint8[i]);
+  if (isUrl(input)) {
+    options.pdfUrl = input;
+    return options;
   }
-  return btoa(binary);
-}
-async function getPdfPageCount(pdfBase64) {
-  const srcBytes = typeof Buffer !== "undefined" ? Buffer.from(pdfBase64, "base64") : Uint8Array.from(atob(pdfBase64), (c) => c.charCodeAt(0));
-  const doc = await import_pdf_lib.PDFDocument.load(srcBytes, { ignoreEncryption: true });
-  return doc.getPageCount();
+  options.pdfBase64 = await pdfInputToBase64(input);
+  return options;
 }
 function getAcroFormFields(pdfDoc) {
   const form = pdfDoc.getForm();
@@ -1787,7 +1902,7 @@ async function runExtractor(params) {
     name,
     prompt,
     schema,
-    pdfBase64,
+    pdfInput,
     startPage,
     endPage,
     generateObject,
@@ -1797,6 +1912,7 @@ async function runExtractor(params) {
   } = params;
   const extractorProviderOptions = { ...providerOptions };
   let fullPrompt;
+  const pdfBase64 = await pdfInputToBase64(pdfInput);
   if (convertPdfToImages) {
     const images = await convertPdfToImages(pdfBase64, startPage, endPage);
     extractorProviderOptions.images = images;
@@ -5249,7 +5365,7 @@ async function findReferencedPages(params) {
     referenceTarget,
     sections,
     formInventory,
-    pdfBase64,
+    pdfInput,
     pageCount,
     generateObject,
     providerOptions,
@@ -5289,7 +5405,7 @@ If you cannot find the section, return startPage: 0 and endPage: 0.
 Return JSON only.`,
         schema: PageLocationSchema,
         maxTokens: 256,
-        providerOptions: { ...providerOptions, pdfBase64 }
+        providerOptions: await buildPdfProviderOptions(pdfInput, providerOptions)
       },
       {
         fallback: { startPage: 0, endPage: 0 },
@@ -5316,7 +5432,7 @@ Return JSON only.`,
 async function resolveReferentialCoverages(params) {
   const {
     memory,
-    pdfBase64,
+    pdfInput,
     pageCount,
     generateObject,
     convertPdfToImages,
@@ -5377,7 +5493,7 @@ async function resolveReferentialCoverages(params) {
           referenceTarget: target,
           sections,
           formInventory,
-          pdfBase64,
+          pdfInput,
           pageCount,
           generateObject,
           providerOptions,
@@ -5411,7 +5527,7 @@ async function resolveReferentialCoverages(params) {
             name: "referential_lookup",
             prompt: buildReferentialLookupPrompt(promptCoverages),
             schema: ReferentialLookupSchema,
-            pdfBase64,
+            pdfInput,
             startPage: pageRange.startPage,
             endPage: pageRange.endPage,
             generateObject,
@@ -6033,7 +6149,7 @@ function createExtractor(config) {
       }))
     };
   }
-  async function extract(pdfBase64, documentId, options) {
+  async function extract(pdfInput, documentId, options) {
     const id = documentId ?? `doc-${Date.now()}`;
     const memory = /* @__PURE__ */ new Map();
     totalUsage = { inputTokens: 0, outputTokens: 0 };
@@ -6051,20 +6167,27 @@ function createExtractor(config) {
         memory.set(k, v);
       }
     }
+    let pdfBase64Cache;
+    async function getPdfBase64ForExtraction() {
+      if (pdfBase64Cache === void 0) {
+        pdfBase64Cache = await pdfInputToBase64(pdfInput);
+      }
+      return pdfBase64Cache;
+    }
     let classifyResult;
     if (resumed?.classifyResult && pipelineCtx.isPhaseComplete("classify")) {
       classifyResult = resumed.classifyResult;
       onProgress?.("Resuming from checkpoint (classify complete)...");
     } else {
       onProgress?.("Classifying document...");
-      const pageCount2 = await getPdfPageCount(pdfBase64);
+      const pageCount2 = await getPdfPageCount(pdfInput);
       const classifyResponse = await safeGenerateObject(
         generateObject,
         {
           prompt: buildClassifyPrompt(),
           schema: ClassifyResultSchema,
           maxTokens: 512,
-          providerOptions: { ...providerOptions, pdfBase64 }
+          providerOptions: await buildPdfProviderOptions(pdfInput, providerOptions)
         },
         {
           fallback: { documentType: "policy", policyTypes: ["other"], confidence: 0 },
@@ -6089,7 +6212,7 @@ function createExtractor(config) {
     const { documentType, policyTypes } = classifyResult;
     const primaryType = policyTypes[0] ?? "other";
     const template = getTemplate(primaryType);
-    const pageCount = resumed?.pageCount ?? await getPdfPageCount(pdfBase64);
+    const pageCount = resumed?.pageCount ?? await getPdfPageCount(pdfInput);
     const templateHints = buildTemplateHints(primaryType, documentType, pageCount, template);
     let formInventory;
     if (resumed?.formInventory && pipelineCtx.isPhaseComplete("form_inventory")) {
@@ -6104,7 +6227,7 @@ function createExtractor(config) {
           prompt: buildFormInventoryPrompt(templateHints),
           schema: FormInventorySchema,
           maxTokens: 2048,
-          providerOptions: { ...providerOptions, pdfBase64 }
+          providerOptions: await buildPdfProviderOptions(pdfInput, providerOptions)
         },
         {
           fallback: { forms: [] },
@@ -6132,9 +6255,10 @@ function createExtractor(config) {
       const chunkSize = 8;
       const collectedAssignments = [];
       const formInventoryHint = formInventory?.forms.length ? formatFormInventoryForPageMap(formInventory.forms) : void 0;
+      const extractionBase64 = await getPdfBase64ForExtraction();
       for (let startPage = 1; startPage <= pageCount; startPage += chunkSize) {
         const endPage = Math.min(pageCount, startPage + chunkSize - 1);
-        const pagesPdf = await extractPageRange(pdfBase64, startPage, endPage);
+        const pagesPdf = await extractPageRange(extractionBase64, startPage, endPage);
         const mapResponse = await safeGenerateObject(
           generateObject,
           {
@@ -6214,7 +6338,7 @@ function createExtractor(config) {
                 name: task.extractorName,
                 prompt: ext.buildPrompt(),
                 schema: ext.schema,
-                pdfBase64,
+                pdfInput,
                 startPage: task.startPage,
                 endPage: task.endPage,
                 generateObject,
@@ -6244,7 +6368,7 @@ function createExtractor(config) {
             name: "supplementary",
             prompt: buildSupplementaryPrompt(alreadyExtractedSummary),
             schema: SupplementarySchema,
-            pdfBase64,
+            pdfInput,
             startPage: 1,
             endPage: pageCount,
             generateObject,
@@ -6273,7 +6397,7 @@ function createExtractor(config) {
       try {
         const resolution = await resolveReferentialCoverages({
           memory,
-          pdfBase64,
+          pdfInput,
           pageCount,
           generateObject,
           convertPdfToImages,
@@ -6313,7 +6437,7 @@ function createExtractor(config) {
             prompt: buildReviewPrompt(template.required, extractedKeys, extractionSummary, pageMapSummary),
             schema: ReviewResultSchema,
             maxTokens: 1536,
-            providerOptions: { ...providerOptions, pdfBase64 }
+            providerOptions: await buildPdfProviderOptions(pdfInput, providerOptions)
           },
           {
             fallback: { complete: true, missingFields: [], qualityIssues: [], additionalTasks: [] },
@@ -6341,7 +6465,7 @@ function createExtractor(config) {
                   name: task.extractorName,
                   prompt: ext.buildPrompt(),
                   schema: ext.schema,
-                  pdfBase64,
+                  pdfInput,
                   startPage: task.startPage,
                   endPage: task.endPage,
                   generateObject,
@@ -9187,6 +9311,7 @@ var AGENT_TOOLS = [
   buildIntentPrompt,
   buildInterpretAttachmentPrompt,
   buildLookupFillPrompt,
+  buildPdfProviderOptions,
   buildQueryClassifyPrompt,
   buildQuestionBatchPrompt,
   buildQuotesPoliciesPrompt,
@@ -9204,10 +9329,14 @@ var AGENT_TOOLS = [
   fillAcroForm,
   getAcroFormFields,
   getExtractor,
+  getFileIdentifier,
   getPdfPageCount,
   getTemplate,
+  isFileReference,
   overlayTextOnPdf,
   pLimit,
+  pdfInputToBase64,
+  pdfInputToBytes,
   safeGenerateObject,
   sanitizeNulls,
   stripFences,