@truto/truto-jsonata 1.0.17 → 1.0.19

package/dist/main.cjs

@@ -11,9 +11,6 @@ var $dxT2C$json2md = require("json2md");
 var $dxT2C$mime = require("mime");
 var $dxT2C$pmap = require("p-map");
 var $dxT2C$pretry = require("p-retry");
-var $dxT2C$officeparser = require("officeparser");
-var $dxT2C$pdfjsserverless = require("pdfjs-serverless");
-var $dxT2C$xlsx = require("xlsx");
 var $dxT2C$langchaintextsplitters = require("@langchain/textsplitters");
 
 function $parcel$interopDefault(a) {
@@ -1401,11 +1398,16 @@ async function $caaee789061bb8bb$var$generateEmbeddingsCohere(body, api_key) {
                         texts: chunk
                     })
                 });
-                if (!response.ok) throw new Error(await response.text());
+                if (!response.ok) {
+                    if (response.status === 429) throw new Error("Rate limit exceeded");
+                    if (response.status >= 500) throw new Error("Server error");
+                    throw new (0, $dxT2C$pretry.AbortError)(await response.text());
+                }
                 return await response.json();
             }, {
-                retries: 10,
-                maxTimeout: 600000
+                retries: 5,
+                maxTimeout: 5000,
+                minTimeout: 2500
             });
         }, {
             concurrency: 1
@@ -1421,89 +1423,49 @@ async function $caaee789061bb8bb$var$generateEmbeddingsCohere(body, api_key) {
             },
             body: JSON.stringify(body)
         });
-        if (!response.ok) throw new Error(await response.text());
+        if (!response.ok) {
+            if (response.status === 429) throw new Error("Rate limit exceeded");
+            if (response.status >= 500) throw new Error("Server error");
+            throw new (0, $dxT2C$pretry.AbortError)(await response.text());
+        }
         return await response.json();
     }, {
-        retries: 10,
-        maxTimeout: 600000
+        retries: 5,
+        maxTimeout: 5000,
+        minTimeout: 2500
     });
 }
 var $caaee789061bb8bb$export$2e2bcd8739ae039 = $caaee789061bb8bb$var$generateEmbeddingsCohere;
 
 
 
-
-
-async function $0f748d6318103cdf$var$bufferToString(value, encoding) {
-    if (value instanceof ReadableStream) {
-        const chunks = [];
-        for await (const chunk of value)chunks.push(Buffer.from(chunk));
-        const buffer = Buffer.concat(chunks);
-        return buffer.toString(encoding);
-    }
-    return value.toString(encoding);
-}
-var $0f748d6318103cdf$export$2e2bcd8739ae039 = $0f748d6318103cdf$var$bufferToString;
-
-
-
-async function $05e3378f7c17d263$var$parsePdf(buffer) {
-    const data = buffer.buffer.slice(buffer?.byteOffset, buffer?.byteOffset + buffer?.byteLength);
-    const { getDocument: getDocument  } = await (0, $dxT2C$pdfjsserverless.resolvePDFJS)();
-    const doc = await getDocument({
-        data: data,
-        useSystemFonts: true
-    }).promise;
-    // Get metadata and initialize output object
-    const output = [];
-    // Iterate through each page and fetch the text content
-    for(let i = 1; i <= doc.numPages; i++){
-        const page = await doc.getPage(i);
-        const textContent = await page.getTextContent();
-        const contents = textContent.items.map((item)=>(0, $dxT2C$lodashes.get)(item, "str")).join(" ");
-        // Add page content to output
-        if (contents) output.push((0, $dxT2C$lodashes.join)([
-            `Page Number: ${i}`,
-            contents
-        ], "/\n"));
-    }
-    // Return the results as JSON
-    return (0, $dxT2C$lodashes.join)(output, "\n");
-}
 async function $05e3378f7c17d263$var$parseDocument(file, fileType) {
-    let buffer;
-    if (file instanceof ReadableStream) {
-        const chunks = [];
-        for await (const chunk of file)chunks.push(Buffer.from(chunk));
-        buffer = Buffer.concat(chunks);
-    }
-    if ((0, $dxT2C$lodashes.includes)([
-        "application/vnd.ms-excel",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-    ], fileType)) {
-        const workbook = $dxT2C$xlsx.read(buffer, {
-            type: "buffer"
-        });
-        const allData = workbook.SheetNames.flatMap((sheetName)=>{
-            const sheetData = $dxT2C$xlsx.utils.sheet_to_json(workbook.Sheets[sheetName]);
-            return sheetData.map((record)=>{
-                const newRecord = {
-                    sheetName: sheetName
-                };
-                return Object.assign(newRecord, record);
-            });
+    const documentParserApiUrl = this.environment.lookup("documentParserApiUrl");
+    const documentParserApiKey = this.environment.lookup("documentParserApiKey");
+    if (!documentParserApiKey) throw new Error("API key not found in environment");
+    return await (0, ($parcel$interopDefault($dxT2C$pretry)))(async ()=>{
+        const response = await fetch(`${documentParserApiUrl}/parse`, {
+            method: "POST",
+            headers: {
+                accept: "application/json",
+                "content-type": fileType,
+                "user-agent": "truto",
+                Authorization: `Bearer ${documentParserApiKey}`
+            },
+            body: file
         });
-        return allData;
-    }
-    if ((0, $dxT2C$lodashes.includes)([
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "application/vnd.oasis.opendocument.text",
-        "application/vnd.oasis.opendocument.spreadsheet",
-        "application/vnd.oasis.opendocument.presentation"
-    ], fileType)) return await (0, $dxT2C$officeparser.parseOfficeAsync)(buffer);
-    if (fileType === "application/pdf") return await $05e3378f7c17d263$var$parsePdf(buffer);
-    return (0, $0f748d6318103cdf$export$2e2bcd8739ae039)(buffer, "utf-8");
+        if (!response.ok) {
+            if (response.status === 429) throw new Error("Rate limit exceeded");
+            if (response.status >= 500) throw new Error("Server error");
+            throw new (0, $dxT2C$pretry.AbortError)(await response.text());
+        }
+        const data = await response.json();
+        return data.content;
+    }, {
+        retries: 5,
+        maxTimeout: 5000,
+        minTimeout: 2500
+    });
 }
 var $05e3378f7c17d263$export$2e2bcd8739ae039 = $05e3378f7c17d263$var$parseDocument;
 
@@ -1547,6 +1509,17 @@ async function $9a2529096849a04f$var$teeStream(stream) {
 var $9a2529096849a04f$export$2e2bcd8739ae039 = $9a2529096849a04f$var$teeStream;
 
 
+async function $0f748d6318103cdf$var$bufferToString(value, encoding) {
+    if (value instanceof ReadableStream) {
+        const chunks = [];
+        for await (const chunk of value)chunks.push(Buffer.from(chunk));
+        const buffer = Buffer.concat(chunks);
+        return buffer.toString(encoding);
+    }
+    return value.toString(encoding);
+}
+var $0f748d6318103cdf$export$2e2bcd8739ae039 = $0f748d6318103cdf$var$bufferToString;
+
 
 function $af351c41b7fd6f79$export$2e2bcd8739ae039(expression) {
     expression.registerFunction("dtFromIso", (0, $bab42b5e4be720d3$export$2e2bcd8739ae039));