@truto/truto-jsonata 1.0.10 → 1.0.12

package/dist/module.js

@@ -1,5 +1,5 @@
 import $hgUW1$jsonata from "jsonata";
-import {difference as $hgUW1$difference, groupBy as $hgUW1$groupBy, castArray as $hgUW1$castArray, keyBy as $hgUW1$keyBy, pick as $hgUW1$pick, omit as $hgUW1$omit, compact as $hgUW1$compact, join as $hgUW1$join, orderBy as $hgUW1$orderBy, find as $hgUW1$find, filter as $hgUW1$filter, values as $hgUW1$values, reduce as $hgUW1$reduce, each as $hgUW1$each, get as $hgUW1$get, isEmpty as $hgUW1$isEmpty, concat as $hgUW1$concat, isEqual as $hgUW1$isEqual, flatMap as $hgUW1$flatMap, map as $hgUW1$map, set as $hgUW1$set, isString as $hgUW1$isString, flattenDeep as $hgUW1$flattenDeep, sortBy as $hgUW1$sortBy, repeat as $hgUW1$repeat, flatten as $hgUW1$flatten, reject as $hgUW1$reject, isNil as $hgUW1$isNil, isFunction as $hgUW1$isFunction, toNumber as $hgUW1$toNumber} from "lodash-es";
+import {difference as $hgUW1$difference, groupBy as $hgUW1$groupBy, castArray as $hgUW1$castArray, keyBy as $hgUW1$keyBy, pick as $hgUW1$pick, omit as $hgUW1$omit, compact as $hgUW1$compact, join as $hgUW1$join, orderBy as $hgUW1$orderBy, find as $hgUW1$find, filter as $hgUW1$filter, values as $hgUW1$values, chunk as $hgUW1$chunk, reduce as $hgUW1$reduce, each as $hgUW1$each, get as $hgUW1$get, isEmpty as $hgUW1$isEmpty, concat as $hgUW1$concat, isEqual as $hgUW1$isEqual, flatMap as $hgUW1$flatMap, map as $hgUW1$map, set as $hgUW1$set, isString as $hgUW1$isString, flattenDeep as $hgUW1$flattenDeep, sortBy as $hgUW1$sortBy, repeat as $hgUW1$repeat, flatten as $hgUW1$flatten, reject as $hgUW1$reject, isNil as $hgUW1$isNil, isFunction as $hgUW1$isFunction, toNumber as $hgUW1$toNumber} from "lodash-es";
 import {DateTime as $hgUW1$DateTime} from "luxon";
 import {Lexer as $hgUW1$Lexer, marked as $hgUW1$marked} from "marked";
 import {diceCoefficient as $hgUW1$diceCoefficient} from "dice-coefficient";
@@ -9,6 +9,13 @@ import {gfm as $hgUW1$gfm} from "turndown-plugin-gfm";
 import {xml2js as $hgUW1$xml2js, js2xml as $hgUW1$js2xml} from "xml-js";
 import $hgUW1$json2md from "json2md";
 import $hgUW1$mime from "mime";
+import $hgUW1$pmap from "p-map";
+import $hgUW1$pretry from "p-retry";
+import {Buffer as $hgUW1$Buffer} from "buffer";
+import {parseOfficeAsync as $hgUW1$parseOfficeAsync} from "officeparser";
+import {fileTypeFromBuffer as $hgUW1$fileTypeFromBuffer} from "file-type";
+import {resolvePDFJS as $hgUW1$resolvePDFJS} from "pdfjs-serverless";
+import {RecursiveCharacterTextSplitter as $hgUW1$RecursiveCharacterTextSplitter} from "@langchain/textsplitters";
 
 function $parcel$interopDefault(a) {
   return a && a.__esModule ? a.default : a;
@@ -1339,23 +1346,138 @@ async function $22f22d9ed76050e5$var$convertMarkdownToHtml(markdown, options) {
 var $22f22d9ed76050e5$export$2e2bcd8739ae039 = $22f22d9ed76050e5$var$convertMarkdownToHtml;
 
 
+
+
+
 async function $bf2fdaa075f6a5e5$var$generateEmbeddingsCohere(body, api_key) {
-    const response = await fetch("https://api.cohere.com/v2/embed", {
-        method: "POST",
-        headers: {
-            accept: "application/json",
-            "content-type": "application/json",
-            "user-agent": "truto",
-            Authorization: `Bearer ${api_key}`
-        },
-        body: JSON.stringify(body)
+    if (!(0, $hgUW1$isEmpty)(body.texts)) {
+        const chunks = (0, $hgUW1$chunk)((0, $hgUW1$castArray)(body.texts), 20);
+        return await (0, $hgUW1$pmap)(chunks, async (chunk)=>{
+            return await (0, $hgUW1$pretry)(async ()=>{
+                const response = await fetch("https://api.cohere.com/v2/embed", {
+                    method: "POST",
+                    headers: {
+                        accept: "application/json",
+                        "content-type": "application/json",
+                        "user-agent": "truto",
+                        Authorization: `Bearer ${api_key}`
+                    },
+                    body: JSON.stringify({
+                        ...body,
+                        texts: chunk
+                    })
+                });
+                if (!response.ok) throw new Error(await response.text());
+                return await response.json();
+            }, {
+                retries: 10,
+                maxTimeout: 600000
+            });
+        }, {
+            concurrency: 1
+        });
+    } else if (!(0, $hgUW1$isEmpty)(body.images)) return await (0, $hgUW1$pretry)(async ()=>{
+        const response = await fetch("https://api.cohere.com/v2/embed", {
+            method: "POST",
+            headers: {
+                accept: "application/json",
+                "content-type": "application/json",
+                "user-agent": "truto",
+                Authorization: `Bearer ${api_key}`
+            },
+            body: JSON.stringify(body)
+        });
+        if (!response.ok) throw new Error(await response.text());
+        return await response.json();
+    }, {
+        retries: 10,
+        maxTimeout: 600000
     });
-    if (!response.ok) throw new Error(await response.text());
-    return await response.json();
 }
 var $bf2fdaa075f6a5e5$export$2e2bcd8739ae039 = $bf2fdaa075f6a5e5$var$generateEmbeddingsCohere;
 
 
+
+
+
+
+
+var $f29f00a7536a3143$require$Buffer = $hgUW1$Buffer;
+async function $f29f00a7536a3143$var$parsePdf(buffer) {
+    const data = buffer.buffer.slice(buffer?.byteOffset, buffer?.byteOffset + buffer?.byteLength);
+    const { getDocument: getDocument  } = await (0, $hgUW1$resolvePDFJS)();
+    const doc = await getDocument({
+        data: data,
+        useSystemFonts: true
+    }).promise;
+    // Get metadata and initialize output object
+    const output = [];
+    // Iterate through each page and fetch the text content
+    for(let i = 1; i <= doc.numPages; i++){
+        const page = await doc.getPage(i);
+        const textContent = await page.getTextContent();
+        const contents = textContent.items.map((item)=>(0, $hgUW1$get)(item, "str")).join(" ");
+        // Add page content to output
+        if (contents) output.push((0, $hgUW1$join)([
+            `Page Number: ${i}`,
+            contents
+        ], "/\n"));
+    }
+    // Return the results as JSON
+    return (0, $hgUW1$join)(output, "\n");
+}
+async function $f29f00a7536a3143$var$parseDocument(file) {
+    if (file instanceof ReadableStream) {
+        const chunks = [];
+        for await (const chunk of file)chunks.push($f29f00a7536a3143$require$Buffer.from(chunk));
+        const buffer = $f29f00a7536a3143$require$Buffer.concat(chunks);
+        const fileExt = (await (0, $hgUW1$fileTypeFromBuffer)(buffer))?.ext;
+        if (fileExt === "pdf") return await $f29f00a7536a3143$var$parsePdf(buffer);
+        return await (0, $hgUW1$parseOfficeAsync)(buffer);
+    }
+    return await (0, $hgUW1$parseOfficeAsync)(file);
+}
+var $f29f00a7536a3143$export$2e2bcd8739ae039 = $f29f00a7536a3143$var$parseDocument;
+
+
+
+function $a12871c9a32cbf3b$export$e600492876ee595b(text, options = {
+    chunkSize: 200,
+    chunkOverlap: 60
+}) {
+    const splitter = new (0, $hgUW1$RecursiveCharacterTextSplitter)(options);
+    return splitter.splitText(text);
+}
+
+
+
+
+var $6b4f9d8bcf8ab446$require$Buffer = $hgUW1$Buffer;
+async function $6b4f9d8bcf8ab446$var$getDataUri(file) {
+    if (file instanceof ReadableStream) {
+        const chunks = [];
+        for await (const chunk of file)chunks.push($6b4f9d8bcf8ab446$require$Buffer.from(chunk));
+        const buffer = $6b4f9d8bcf8ab446$require$Buffer.concat(chunks);
+        const base64Image = buffer.toString("base64");
+        const mimeType = (await (0, $hgUW1$fileTypeFromBuffer)(buffer))?.mime;
+        // Construct the data URI for a PNG image
+        return `data:${mimeType};base64,${base64Image}`;
+    }
+    const arrayBuffer = file instanceof Blob ? await file.arrayBuffer() : file;
+    const buffer = $6b4f9d8bcf8ab446$require$Buffer.from(arrayBuffer);
+    const base64Image = buffer.toString("base64");
+    const mimeType = (await (0, $hgUW1$fileTypeFromBuffer)(buffer))?.mime;
+    return `data:${mimeType};base64,${base64Image}`;
+}
+var $6b4f9d8bcf8ab446$export$2e2bcd8739ae039 = $6b4f9d8bcf8ab446$var$getDataUri;
+
+
+async function $5e08a3608e719b52$var$teeStream(stream) {
+    return stream.tee();
+}
+var $5e08a3608e719b52$export$2e2bcd8739ae039 = $5e08a3608e719b52$var$teeStream;
+
+
 function $3e1c03caddb74419$export$2e2bcd8739ae039(expression) {
     expression.registerFunction("dtFromIso", (0, $9e00534fc92334a4$export$2e2bcd8739ae039));
     expression.registerFunction("base64decode", (0, $52da679a67e0b066$export$2e2bcd8739ae039));
@@ -1423,6 +1545,9 @@ function $3e1c03caddb74419$export$2e2bcd8739ae039(expression) {
     expression.registerFunction("values", function(obj) {
         return (0, $hgUW1$values)(obj);
     });
+    expression.registerFunction("chunk", function(arr, size) {
+        return (0, $hgUW1$chunk)((0, $hgUW1$castArray)(arr), size);
+    });
     expression.registerFunction("wrap", function(value, wrapper, endWrapper) {
         return (0, $hgUW1$join)([
             wrapper,
@@ -1430,6 +1555,10 @@ function $3e1c03caddb74419$export$2e2bcd8739ae039(expression) {
             endWrapper || wrapper
         ], "");
     });
+    expression.registerFunction("parseDocument", (0, $f29f00a7536a3143$export$2e2bcd8739ae039));
+    expression.registerFunction("recursiveCharacterTextSplitter", (0, $a12871c9a32cbf3b$export$e600492876ee595b));
+    expression.registerFunction("getDataUri", (0, $6b4f9d8bcf8ab446$export$2e2bcd8739ae039));
+    expression.registerFunction("teeStream", (0, $5e08a3608e719b52$export$2e2bcd8739ae039));
     return expression;
 }