@truto/truto-jsonata 1.0.14 → 1.0.16

package/dist/main.cjs

@@ -13,6 +13,7 @@ var $dxT2C$pmap = require("p-map");
 var $dxT2C$pretry = require("p-retry");
 var $dxT2C$officeparser = require("officeparser");
 var $dxT2C$pdfjsserverless = require("pdfjs-serverless");
+var $dxT2C$xlsx = require("xlsx");
 var $dxT2C$langchaintextsplitters = require("@langchain/textsplitters");
 
 function $parcel$interopDefault(a) {
@@ -642,19 +643,28 @@ const $bc8e7b2fdb012b3a$var$parseMarkedTokenToNotionRequest = (tokens, acc = [])
         if (!text) return acc;
         const textToInsert = (0, $5a7cb266718aeaae$export$2e2bcd8739ae039)(text);
         // chunk the text into 2000 character chunks, should handle emojis and multi-byte characters
-        const chunks = $bc8e7b2fdb012b3a$var$chunkText(textToInsert);
-        (0, $dxT2C$lodashes.each)(chunks, (chunk)=>{
-            acc.push({
-                type: "text",
-                text: {
-                    content: chunk
-                },
-                ...token.type === "codespan" ? {
-                    annotations: {
-                        code: true
-                    }
-                } : {}
-            });
+        const textSplitByNewLine = textToInsert.split("\n");
+        const chunksOfChunks = (0, $dxT2C$lodashes.compact)((0, $dxT2C$lodashes.map)(textSplitByNewLine, (chunk)=>$bc8e7b2fdb012b3a$var$chunkText(chunk)));
+        const chunksWithNewLines = (0, $dxT2C$lodashes.flattenDeep)((0, $233848446ca33fe6$export$2e2bcd8739ae039)((0, $dxT2C$lodashes.map)(chunksOfChunks, (chunk)=>{
+            return (0, $dxT2C$lodashes.map)(chunk, (_chunk)=>({
+                    type: "text",
+                    text: {
+                        content: _chunk
+                    },
+                    ...token.type === "codespan" ? {
+                        annotations: {
+                            code: true
+                        }
+                    } : {}
+                }));
+        }), {
+            type: "text",
+            text: {
+                content: "\n"
+            }
+        }));
+        (0, $dxT2C$lodashes.each)(chunksWithNewLines, (chunk)=>{
+            acc.push(chunk);
         });
         return acc;
     }, acc);
@@ -1436,6 +1446,7 @@ async function $0f748d6318103cdf$var$bufferToString(value, encoding) {
 var $0f748d6318103cdf$export$2e2bcd8739ae039 = $0f748d6318103cdf$var$bufferToString;
 
 
+
 async function $05e3378f7c17d263$var$parsePdf(buffer) {
     const data = buffer.buffer.slice(buffer?.byteOffset, buffer?.byteOffset + buffer?.byteLength);
     const { getDocument: getDocument  } = await (0, $dxT2C$pdfjsserverless.resolvePDFJS)();
@@ -1466,13 +1477,30 @@ async function $05e3378f7c17d263$var$parseDocument(file, fileType) {
         for await (const chunk of file)chunks.push(Buffer.from(chunk));
         buffer = Buffer.concat(chunks);
     }
+    if ((0, $dxT2C$lodashes.includes)([
+        "application/vnd.ms-excel",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+    ], fileType)) {
+        const workbook = $dxT2C$xlsx.read(buffer, {
+            type: "buffer"
+        });
+        const allData = workbook.SheetNames.flatMap((sheetName)=>{
+            const sheetData = $dxT2C$xlsx.utils.sheet_to_json(workbook.Sheets[sheetName]);
+            return sheetData.map((record)=>{
+                const newRecord = {
+                    sheetName: sheetName
+                };
+                return Object.assign(newRecord, record);
+            });
+        });
+        return allData;
+    }
     if ((0, $dxT2C$lodashes.includes)([
         "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
         "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
         "application/vnd.oasis.opendocument.text",
-        "application/vnd.oasis.opendocument.presentation",
-        "application/vnd.oasis.opendocument.spreadsheet"
+        "application/vnd.oasis.opendocument.spreadsheet",
+        "application/vnd.oasis.opendocument.presentation"
     ], fileType)) return await (0, $dxT2C$officeparser.parseOfficeAsync)(buffer);
     if (fileType === "application/pdf") return await $05e3378f7c17d263$var$parsePdf(buffer);
     return (0, $0f748d6318103cdf$export$2e2bcd8739ae039)(buffer, "utf-8");
@@ -1481,12 +1509,17 @@ var $05e3378f7c17d263$export$2e2bcd8739ae039 = $05e3378f7c17d263$var$parseDocume
 
 
 
-function $fe4dcef142601b8c$export$e600492876ee595b(text, options = {
+
+async function $fe4dcef142601b8c$export$e600492876ee595b(text, options = {
     chunkSize: 200,
     chunkOverlap: 60
 }) {
+    let textToSplit;
+    if ((0, $dxT2C$lodashes.isString)(text)) textToSplit = text;
+    else if ((0, $dxT2C$lodashes.isPlainObject)(text) || (0, $dxT2C$lodashes.isArray)(text)) textToSplit = JSON.stringify(text, null, 2);
+    else textToSplit = (0, $dxT2C$lodashes.toString)(text);
     const splitter = new (0, $dxT2C$langchaintextsplitters.RecursiveCharacterTextSplitter)(options);
-    return splitter.splitText(text);
+    return splitter.splitText(textToSplit);
 }