@truto/truto-jsonata 1.0.10 → 1.0.12

package/README.md

@@ -881,6 +881,23 @@ expression.evaluate({ file}).then(result => { console.log(result); });
 
 </details>
 
+<details>
+<summary>getDataUri(file)</summary>
+
+Converts a `Blob` or `Buffer` or `Readable Stream` to a data URI.
+
+**Example:**
+
+```javascript
+import trutoJsonata from '@truto/truto-jsonata';
+const file = new Blob(['Hello, World!'], { type: 'text/plain' });
+const expression = trutoJsonata("getDataUri(file)");
+expression.evaluate({ file}).then(result => { console.log(result); });
+// Output: 'data:text/plain;base64,SGVsbG8sIFdvcmxkIQ=='
+```
+
+</details>
+
 <details>
 <summary>blob(content, options)</summary>
 
@@ -1241,6 +1258,25 @@ expression3.evaluate({ jsonData: jsonData3, options: options3 }).then(result =>
 
 </details>
 
+<details>
+<summary>parseDocument(file)</summary>
+
+Parses a document file (e.g., PDF, DOCX) and extracts text content.
+
+**Example:**
+
+```javascript
+import trutoJsonata from '@truto/truto-jsonata';
+const file = new Blob(['Hello, World!'], { type: 'application/pdf' });
+const buffer = await file.arrayBuffer();
+const expression = trutoJsonata("$parseDocument(buffer)");
+expression.evaluate({ file}).then(result => { console.log(result); });
+// Output: 'Hello, World!'
+
+````
+
+</details>
+
 ### Markdown and Text Conversion
 
 <details>
@@ -1855,6 +1891,35 @@ expression.evaluate({ body, api_key }).then(result => { console.log(result); });
 
 </details>
 
+<details>
+<summary>recursiveCharacterTextSplitter(text, options)</summary>
+
+Splits a text into an array of characters, words, or sentences, recursively.
+
+**Parameters:**
+
+- **text**: The input text to split.
+- **options**: An object containing the following properties:
+  - **chunkSize**: The maximum number of characters, words, or sentences per chunk (default is `200`).
+  - **chunkOverlap**: The number of characters, words, or sentences to overlap between chunks (default is `60`).
+
+**Example Usage:**
+
+```javascript
+import trutoJsonata from '@truto/truto-jsonata';
+
+const text = "Hello, World! This is a sample text.";
+const options = {
+    chunkSize: 10,
+    chunkOverlap: 3
+};
+const expression = trutoJsonata("$recursiveCharacterTextSplitter(text, options)");
+expression.evaluate(text).then(result => { console.log(result); });
+// Output: ["Hello, Wo", "lo, World", "rld! This", "is a samp", "ample text", "text."]
+```
+
+</details>
+
 
 ### Miscellaneous
 
@@ -2015,3 +2080,23 @@ expression.evaluate({}).then(result => { console.log(result); });
 ```
 
 </details>
+
+
+
+<details>
+<summary>chunk(arr,size)</summary>
+
+Chunks an array into smaller arrays of a specified size.
+
+**Example:**
+
+```javascript
+import trutoJsonata from '@truto/truto-jsonata';
+
+
+const expression = trutoJsonata("$chunk([1, 2, 3, 4, 5], 2)");
+expression.evaluate({}).then(result => { console.log(result); });
+// Output: [[1,2],[3,4],[5]]
+```
+
+</details>

package/dist/main.cjs

@@ -9,6 +9,12 @@ var $dxT2C$turndownplugingfm = require("turndown-plugin-gfm");
 var $dxT2C$xmljs = require("xml-js");
 var $dxT2C$json2md = require("json2md");
 var $dxT2C$mime = require("mime");
+var $dxT2C$pmap = require("p-map");
+var $dxT2C$pretry = require("p-retry");
+var $dxT2C$officeparser = require("officeparser");
+var $dxT2C$filetype = require("file-type");
+var $dxT2C$pdfjsserverless = require("pdfjs-serverless");
+var $dxT2C$langchaintextsplitters = require("@langchain/textsplitters");
 
 function $parcel$interopDefault(a) {
   return a && a.__esModule ? a.default : a;
@@ -1349,23 +1355,134 @@ async function $2c639152271462a6$var$convertMarkdownToHtml(markdown, options) {
 var $2c639152271462a6$export$2e2bcd8739ae039 = $2c639152271462a6$var$convertMarkdownToHtml;
 
 
+
+
+
 async function $caaee789061bb8bb$var$generateEmbeddingsCohere(body, api_key) {
-    const response = await fetch("https://api.cohere.com/v2/embed", {
-        method: "POST",
-        headers: {
-            accept: "application/json",
-            "content-type": "application/json",
-            "user-agent": "truto",
-            Authorization: `Bearer ${api_key}`
-        },
-        body: JSON.stringify(body)
+    if (!(0, $dxT2C$lodashes.isEmpty)(body.texts)) {
+        const chunks = (0, $dxT2C$lodashes.chunk)((0, $dxT2C$lodashes.castArray)(body.texts), 20);
+        return await (0, ($parcel$interopDefault($dxT2C$pmap)))(chunks, async (chunk)=>{
+            return await (0, ($parcel$interopDefault($dxT2C$pretry)))(async ()=>{
+                const response = await fetch("https://api.cohere.com/v2/embed", {
+                    method: "POST",
+                    headers: {
+                        accept: "application/json",
+                        "content-type": "application/json",
+                        "user-agent": "truto",
+                        Authorization: `Bearer ${api_key}`
+                    },
+                    body: JSON.stringify({
+                        ...body,
+                        texts: chunk
+                    })
+                });
+                if (!response.ok) throw new Error(await response.text());
+                return await response.json();
+            }, {
+                retries: 10,
+                maxTimeout: 600000
+            });
+        }, {
+            concurrency: 1
+        });
+    } else if (!(0, $dxT2C$lodashes.isEmpty)(body.images)) return await (0, ($parcel$interopDefault($dxT2C$pretry)))(async ()=>{
+        const response = await fetch("https://api.cohere.com/v2/embed", {
+            method: "POST",
+            headers: {
+                accept: "application/json",
+                "content-type": "application/json",
+                "user-agent": "truto",
+                Authorization: `Bearer ${api_key}`
+            },
+            body: JSON.stringify(body)
+        });
+        if (!response.ok) throw new Error(await response.text());
+        return await response.json();
+    }, {
+        retries: 10,
+        maxTimeout: 600000
     });
-    if (!response.ok) throw new Error(await response.text());
-    return await response.json();
 }
 var $caaee789061bb8bb$export$2e2bcd8739ae039 = $caaee789061bb8bb$var$generateEmbeddingsCohere;
 
 
+
+
+
+
+async function $05e3378f7c17d263$var$parsePdf(buffer) {
+    const data = buffer.buffer.slice(buffer?.byteOffset, buffer?.byteOffset + buffer?.byteLength);
+    const { getDocument: getDocument  } = await (0, $dxT2C$pdfjsserverless.resolvePDFJS)();
+    const doc = await getDocument({
+        data: data,
+        useSystemFonts: true
+    }).promise;
+    // Get metadata and initialize output object
+    const output = [];
+    // Iterate through each page and fetch the text content
+    for(let i = 1; i <= doc.numPages; i++){
+        const page = await doc.getPage(i);
+        const textContent = await page.getTextContent();
+        const contents = textContent.items.map((item)=>(0, $dxT2C$lodashes.get)(item, "str")).join(" ");
+        // Add page content to output
+        if (contents) output.push((0, $dxT2C$lodashes.join)([
+            `Page Number: ${i}`,
+            contents
+        ], "/\n"));
+    }
+    // Return the results as JSON
+    return (0, $dxT2C$lodashes.join)(output, "\n");
+}
+async function $05e3378f7c17d263$var$parseDocument(file) {
+    if (file instanceof ReadableStream) {
+        const chunks = [];
+        for await (const chunk of file)chunks.push(Buffer.from(chunk));
+        const buffer = Buffer.concat(chunks);
+        const fileExt = (await (0, $dxT2C$filetype.fileTypeFromBuffer)(buffer))?.ext;
+        if (fileExt === "pdf") return await $05e3378f7c17d263$var$parsePdf(buffer);
+        return await (0, $dxT2C$officeparser.parseOfficeAsync)(buffer);
+    }
+    return await (0, $dxT2C$officeparser.parseOfficeAsync)(file);
+}
+var $05e3378f7c17d263$export$2e2bcd8739ae039 = $05e3378f7c17d263$var$parseDocument;
+
+
+
+function $fe4dcef142601b8c$export$e600492876ee595b(text, options = {
+    chunkSize: 200,
+    chunkOverlap: 60
+}) {
+    const splitter = new (0, $dxT2C$langchaintextsplitters.RecursiveCharacterTextSplitter)(options);
+    return splitter.splitText(text);
+}
+
+
+
+async function $15c432f5f036a88a$var$getDataUri(file) {
+    if (file instanceof ReadableStream) {
+        const chunks = [];
+        for await (const chunk of file)chunks.push(Buffer.from(chunk));
+        const buffer = Buffer.concat(chunks);
+        const base64Image = buffer.toString("base64");
+        const mimeType = (await (0, $dxT2C$filetype.fileTypeFromBuffer)(buffer))?.mime;
+        // Construct the data URI for a PNG image
+        return `data:${mimeType};base64,${base64Image}`;
+    }
+    const arrayBuffer = file instanceof Blob ? await file.arrayBuffer() : file;
+    const buffer = Buffer.from(arrayBuffer);
+    const base64Image = buffer.toString("base64");
+    const mimeType = (await (0, $dxT2C$filetype.fileTypeFromBuffer)(buffer))?.mime;
+    return `data:${mimeType};base64,${base64Image}`;
+}
+var $15c432f5f036a88a$export$2e2bcd8739ae039 = $15c432f5f036a88a$var$getDataUri;
+
+
+async function $9a2529096849a04f$var$teeStream(stream) {
+    return stream.tee();
+}
+var $9a2529096849a04f$export$2e2bcd8739ae039 = $9a2529096849a04f$var$teeStream;
+
+
 function $af351c41b7fd6f79$export$2e2bcd8739ae039(expression) {
     expression.registerFunction("dtFromIso", (0, $bab42b5e4be720d3$export$2e2bcd8739ae039));
     expression.registerFunction("base64decode", (0, $77081a2d6d46cd50$export$2e2bcd8739ae039));
@@ -1433,6 +1550,9 @@ function $af351c41b7fd6f79$export$2e2bcd8739ae039(expression) {
     expression.registerFunction("values", function(obj) {
         return (0, $dxT2C$lodashes.values)(obj);
     });
+    expression.registerFunction("chunk", function(arr, size) {
+        return (0, $dxT2C$lodashes.chunk)((0, $dxT2C$lodashes.castArray)(arr), size);
+    });
     expression.registerFunction("wrap", function(value, wrapper, endWrapper) {
         return (0, $dxT2C$lodashes.join)([
             wrapper,
@@ -1440,6 +1560,10 @@ function $af351c41b7fd6f79$export$2e2bcd8739ae039(expression) {
             endWrapper || wrapper
         ], "");
     });
+    expression.registerFunction("parseDocument", (0, $05e3378f7c17d263$export$2e2bcd8739ae039));
+    expression.registerFunction("recursiveCharacterTextSplitter", (0, $fe4dcef142601b8c$export$e600492876ee595b));
+    expression.registerFunction("getDataUri", (0, $15c432f5f036a88a$export$2e2bcd8739ae039));
+    expression.registerFunction("teeStream", (0, $9a2529096849a04f$export$2e2bcd8739ae039));
     return expression;
 }