@truto/truto-jsonata 1.0.12 → 1.0.14

package/dist/main.cjs

@@ -12,7 +12,6 @@ var $dxT2C$mime = require("mime");
 var $dxT2C$pmap = require("p-map");
 var $dxT2C$pretry = require("p-retry");
 var $dxT2C$officeparser = require("officeparser");
-var $dxT2C$filetype = require("file-type");
 var $dxT2C$pdfjsserverless = require("pdfjs-serverless");
 var $dxT2C$langchaintextsplitters = require("@langchain/textsplitters");
 
@@ -459,6 +458,7 @@ var $90c566adb85cb52e$export$2e2bcd8739ae039 = $90c566adb85cb52e$var$convertMark
 
 
 
+
 const $bc8e7b2fdb012b3a$var$parseMarkedTokenToNotionRequest = (tokens, acc = [])=>{
     return (0, $dxT2C$lodashes.reduce)(tokens, (acc, token)=>{
         const childTokens = token.type === "blockquote" ? (0, $dxT2C$lodashes.get)(token, "tokens[0].tokens") : (0, $dxT2C$lodashes.get)(token, "tokens");
@@ -520,17 +520,25 @@ const $bc8e7b2fdb012b3a$var$parseMarkedTokenToNotionRequest = (tokens, acc = [])
             return acc;
         }
         if (token.type === "code") {
+            const textSplitByNewLine = token.text.split("\n");
+            const chunksOfChunks = (0, $dxT2C$lodashes.compact)((0, $dxT2C$lodashes.map)(textSplitByNewLine, (chunk)=>$bc8e7b2fdb012b3a$var$chunkText(chunk)));
+            const chunksWithNewLines = (0, $dxT2C$lodashes.flattenDeep)((0, $233848446ca33fe6$export$2e2bcd8739ae039)((0, $dxT2C$lodashes.map)(chunksOfChunks, (chunk)=>{
+                return (0, $dxT2C$lodashes.map)(chunk, (_chunk)=>({
+                        type: "text",
+                        text: {
+                            content: _chunk
+                        }
+                    }));
+            }), {
+                type: "text",
+                text: {
+                    content: "\n\n"
+                }
+            }));
             acc.push({
                 type: "code",
                 code: {
-                    rich_text: [
-                        {
-                            type: "text",
-                            text: {
-                                content: token.text
-                            }
-                        }
-                    ],
+                    rich_text: chunksWithNewLines,
                     language: token.lang || "plain text"
                 }
             });
@@ -633,20 +641,27 @@ const $bc8e7b2fdb012b3a$var$parseMarkedTokenToNotionRequest = (tokens, acc = [])
         const text = (0, $dxT2C$lodashes.get)(token, "text");
         if (!text) return acc;
         const textToInsert = (0, $5a7cb266718aeaae$export$2e2bcd8739ae039)(text);
-        acc.push({
-            type: "text",
-            text: {
-                content: textToInsert
-            },
-            ...token.type === "codespan" ? {
-                annotations: {
-                    code: true
-                }
-            } : {}
+        // chunk the text into 2000 character chunks, should handle emojis and multi-byte characters
+        const chunks = $bc8e7b2fdb012b3a$var$chunkText(textToInsert);
+        (0, $dxT2C$lodashes.each)(chunks, (chunk)=>{
+            acc.push({
+                type: "text",
+                text: {
+                    content: chunk
+                },
+                ...token.type === "codespan" ? {
+                    annotations: {
+                        code: true
+                    }
+                } : {}
+            });
         });
         return acc;
     }, acc);
 };
+const $bc8e7b2fdb012b3a$var$chunkText = (text, numChars = 2000)=>{
+    return text.match(new RegExp(`.{1,${numChars}}`, "g"));
+};
 const $bc8e7b2fdb012b3a$var$convertMarkdownToNotion = (text)=>{
     const tokens = (0, $dxT2C$marked.Lexer).lex(text);
     const parsedTokens = $bc8e7b2fdb012b3a$var$parseMarkedTokenToNotionRequest(tokens);
@@ -1409,6 +1424,17 @@ var $caaee789061bb8bb$export$2e2bcd8739ae039 = $caaee789061bb8bb$var$generateEmb
 
 
 
+async function $0f748d6318103cdf$var$bufferToString(value, encoding) {
+    if (value instanceof ReadableStream) {
+        const chunks = [];
+        for await (const chunk of value)chunks.push(Buffer.from(chunk));
+        const buffer = Buffer.concat(chunks);
+        return buffer.toString(encoding);
+    }
+    return value.toString(encoding);
+}
+var $0f748d6318103cdf$export$2e2bcd8739ae039 = $0f748d6318103cdf$var$bufferToString;
+
 
 async function $05e3378f7c17d263$var$parsePdf(buffer) {
     const data = buffer.buffer.slice(buffer?.byteOffset, buffer?.byteOffset + buffer?.byteLength);
@@ -1433,16 +1459,23 @@ async function $05e3378f7c17d263$var$parsePdf(buffer) {
     // Return the results as JSON
     return (0, $dxT2C$lodashes.join)(output, "\n");
 }
-async function $05e3378f7c17d263$var$parseDocument(file) {
+async function $05e3378f7c17d263$var$parseDocument(file, fileType) {
+    let buffer;
     if (file instanceof ReadableStream) {
         const chunks = [];
         for await (const chunk of file)chunks.push(Buffer.from(chunk));
-        const buffer = Buffer.concat(chunks);
-        const fileExt = (await (0, $dxT2C$filetype.fileTypeFromBuffer)(buffer))?.ext;
-        if (fileExt === "pdf") return await $05e3378f7c17d263$var$parsePdf(buffer);
-        return await (0, $dxT2C$officeparser.parseOfficeAsync)(buffer);
+        buffer = Buffer.concat(chunks);
     }
-    return await (0, $dxT2C$officeparser.parseOfficeAsync)(file);
+    if ((0, $dxT2C$lodashes.includes)([
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "application/vnd.oasis.opendocument.text",
+        "application/vnd.oasis.opendocument.presentation",
+        "application/vnd.oasis.opendocument.spreadsheet"
+    ], fileType)) return await (0, $dxT2C$officeparser.parseOfficeAsync)(buffer);
+    if (fileType === "application/pdf") return await $05e3378f7c17d263$var$parsePdf(buffer);
+    return (0, $0f748d6318103cdf$export$2e2bcd8739ae039)(buffer, "utf-8");
 }
 var $05e3378f7c17d263$export$2e2bcd8739ae039 = $05e3378f7c17d263$var$parseDocument;
 
@@ -1457,21 +1490,19 @@ function $fe4dcef142601b8c$export$e600492876ee595b(text, options = {
 }
 
 
-
-async function $15c432f5f036a88a$var$getDataUri(file) {
+async function $15c432f5f036a88a$var$getDataUri(file, mimeType) {
+    if (!mimeType) throw new Error("Mime type is required");
     if (file instanceof ReadableStream) {
         const chunks = [];
         for await (const chunk of file)chunks.push(Buffer.from(chunk));
         const buffer = Buffer.concat(chunks);
         const base64Image = buffer.toString("base64");
-        const mimeType = (await (0, $dxT2C$filetype.fileTypeFromBuffer)(buffer))?.mime;
         // Construct the data URI for a PNG image
         return `data:${mimeType};base64,${base64Image}`;
     }
     const arrayBuffer = file instanceof Blob ? await file.arrayBuffer() : file;
     const buffer = Buffer.from(arrayBuffer);
     const base64Image = buffer.toString("base64");
-    const mimeType = (await (0, $dxT2C$filetype.fileTypeFromBuffer)(buffer))?.mime;
     return `data:${mimeType};base64,${base64Image}`;
 }
 var $15c432f5f036a88a$export$2e2bcd8739ae039 = $15c432f5f036a88a$var$getDataUri;
@@ -1483,6 +1514,7 @@ async function $9a2529096849a04f$var$teeStream(stream) {
 var $9a2529096849a04f$export$2e2bcd8739ae039 = $9a2529096849a04f$var$teeStream;
 
 
+
 function $af351c41b7fd6f79$export$2e2bcd8739ae039(expression) {
     expression.registerFunction("dtFromIso", (0, $bab42b5e4be720d3$export$2e2bcd8739ae039));
     expression.registerFunction("base64decode", (0, $77081a2d6d46cd50$export$2e2bcd8739ae039));
@@ -1564,6 +1596,7 @@ function $af351c41b7fd6f79$export$2e2bcd8739ae039(expression) {
     expression.registerFunction("recursiveCharacterTextSplitter", (0, $fe4dcef142601b8c$export$e600492876ee595b));
     expression.registerFunction("getDataUri", (0, $15c432f5f036a88a$export$2e2bcd8739ae039));
     expression.registerFunction("teeStream", (0, $9a2529096849a04f$export$2e2bcd8739ae039));
+    expression.registerFunction("bufferToString", (0, $0f748d6318103cdf$export$2e2bcd8739ae039));
     return expression;
 }