@aj-archipelago/cortex 1.0.11 → 1.0.13

package/config.js

@@ -87,7 +87,27 @@ var config = convict({
                 "params": {
                     "model": "whisper-1"
                 },
-            }
+            },
+            "azure-cognitive": {
+                "type": "AZURE-COGNITIVE",
+                "url": "https://archipelago-cognitive-search.search.windows.net/indexes/indexcortex/docs/search?api-version=2023-07-01-Preview",
+                "headers": {
+                    "api-key": "{{AZURE_COGNITIVE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "oai-embeddings": {
+                "type": "OPENAI-EMBEDDINGS",
+                "url": "https://archipelago-openai.openai.azure.com/openai/deployments/archipelago-embedding/embeddings?api-version=2023-05-15",
+                "headers": {
+                    "api-key": "{{ARCHIPELAGO_OPENAI_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "text-embedding-ada-002"
+                },
+                "maxTokenLength": 8192,
+            },
         },
         env: 'CORTEX_MODELS'
     },

package/helper_apps/MediaFileChunker/docHelper.js

@@ -0,0 +1,94 @@
+import pdfjsLib from 'pdfjs-dist';
+import fs from 'fs/promises';
+import mammoth from 'mammoth';
+import XLSX from 'xlsx';
+
+export async function txtToText(filePath) {
+    const text = await fs.readFile(filePath, 'utf-8');
+    return text;
+}
+
+export async function docxToText(filePath) {
+    const buffer = await fs.readFile(filePath);
+    const result = await mammoth.extractRawText({ buffer: buffer });
+    return result.value;
+}
+
+export async function xlsxToText(filePath) {
+    const workbook = XLSX.readFile(filePath);
+    let finalText = '';
+
+    workbook.SheetNames.forEach(sheetName => {
+        const sheet = workbook.Sheets[sheetName];
+        const sheetAsJson = XLSX.utils.sheet_to_json(sheet, { header: 1 });
+        sheetAsJson.forEach(row => {
+            finalText += row.join(' ') + '\n';
+        });
+    });
+
+    return finalText;
+}
+
+export async function pdfToText(filePath) {
+    const pdf = await pdfjsLib.getDocument(filePath).promise;
+    let finalText = '';
+
+    for(let i = 1; i <= pdf.numPages; i++) {
+        const page = await pdf.getPage(i);
+        const textContent = await page.getTextContent();
+        const strings = textContent.items.map(item => item.str);
+        finalText += strings.join(' ');
+    }
+
+    return finalText;
+}
+
+export async function documentToText(filePath) {
+    const fileExtension = filePath.split('.').pop();
+
+    switch (fileExtension) {
+        case 'pdf':
+            return pdfToText(filePath);
+        case 'txt':
+            return txtToText(filePath);
+        case 'docx':
+            return docxToText(filePath);
+        case 'xlsx':
+            return xlsxToText(filePath);
+        default:
+            throw new Error(`Unsupported file type: ${fileExtension}`);
+    }
+}
+
+export function easyChunker(text) {
+    const result = [];
+    const n = 10000;
+
+    // If the text is less than n characters, just process it as is
+    if (text.length <= n) {
+        return [text];
+    }
+
+    let startIndex = 0;
+    while (startIndex < text.length) {
+        let endIndex = Math.min(startIndex + n, text.length);
+
+        // Make sure we don't split in the middle of a sentence
+        while (endIndex > startIndex && text[endIndex] !== '.' && text[endIndex] !== ' ') {
+            endIndex--;
+        }
+
+        // If we didn't find a sentence break, just split at n characters
+        if (endIndex === startIndex) {
+            endIndex = startIndex + n;
+        }
+
+        // Push the chunk to the result array
+        result.push(text.substring(startIndex, endIndex));
+
+        // Move the start index to the next chunk
+        startIndex = endIndex;
+    }
+
+    return result;
+}

package/helper_apps/MediaFileChunker/fileChunker.js

@@ -163,5 +163,5 @@ const processYoutubeUrl = async (url) => {
 }
 
 export {
-    splitMediaFile, processYoutubeUrl
+    splitMediaFile, processYoutubeUrl, downloadFile
 };

package/helper_apps/MediaFileChunker/index.js

@@ -1,8 +1,12 @@
-import { processYoutubeUrl, splitMediaFile } from './fileChunker.js';
+import { downloadFile, processYoutubeUrl, splitMediaFile } from './fileChunker.js';
 import { saveFileToBlob, deleteBlob, uploadBlob } from './blobHandler.js';
 import { publishRequestProgress } from './redis.js';
 import { deleteTempPath, ensureEncoded, isValidYoutubeUrl } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder } from './localFileHandler.js';
+import { documentToText, easyChunker } from './docHelper.js';
+import path from 'path';
+import os from 'os';
+import { v4 as uuidv4 } from 'uuid';
 
 const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
 console.log(useAzure ? 'Using Azure Storage' : 'Using local file system');
@@ -59,42 +63,51 @@ async function main(context, req) {
         await publishRequestProgress({ requestId, progress, completedCount, totalCount, numberOfChunks, data });
     }
 
-    try {
-        if (isYoutubeUrl) {
-            // totalCount += 1; // extra 1 step for youtube download
-            file = await processYoutubeUrl(file);
-        }
-
-        const { chunkPromises, uniqueOutputPath } = await splitMediaFile(file);
-        folder = uniqueOutputPath;
+    const isDocument = ['.pdf', '.txt', '.docx', '.xlsx'].some(ext => uri.toLowerCase().endsWith(ext));
 
-        numberOfChunks = chunkPromises.length; // for progress reporting
-        totalCount += chunkPromises.length * 4; // 4 steps for each chunk (download and upload)
-        // isYoutubeUrl && sendProgress(); // send progress for youtube download after total count is calculated
-
-        // sequential download of chunks
-        const chunks = [];
-        for (const chunkPromise of chunkPromises) {
-            chunks.push(await chunkPromise);
-            sendProgress();
-        }
-
-        // sequential processing of chunks
-        for (const chunk of chunks) {
-            const blobName = useAzure ? await saveFileToBlob(chunk, requestId) : await moveFileToPublicFolder(chunk, requestId);
-            result.push(blobName);
-            context.log(`Saved chunk as: ${blobName}`);
-            sendProgress();
+    try {
+        if (isDocument) {
+            const extension = path.extname(uri).toLowerCase();
+            const file = path.join(os.tmpdir(), `${uuidv4()}${extension}`);
+            await downloadFile(uri,file)
+            result.push(...easyChunker(await documentToText(file)));
+        }else{
+
+            if (isYoutubeUrl) {
+                // totalCount += 1; // extra 1 step for youtube download
+                file = await processYoutubeUrl(file);
+            }
+
+            const { chunkPromises, uniqueOutputPath } = await splitMediaFile(file);
+            folder = uniqueOutputPath;
+
+            numberOfChunks = chunkPromises.length; // for progress reporting
+            totalCount += chunkPromises.length * 4; // 4 steps for each chunk (download and upload)
+            // isYoutubeUrl && sendProgress(); // send progress for youtube download after total count is calculated
+
+            // sequential download of chunks
+            const chunks = [];
+            for (const chunkPromise of chunkPromises) {
+                chunks.push(await chunkPromise);
+                sendProgress();
+            }
+
+            // sequential processing of chunks
+            for (const chunk of chunks) {
+                const blobName = useAzure ? await saveFileToBlob(chunk, requestId) : await moveFileToPublicFolder(chunk, requestId);
+                result.push(blobName);
+                context.log(`Saved chunk as: ${blobName}`);
+                sendProgress();
+            }
+
+            // parallel processing, dropped 
+            // result = await Promise.all(mediaSplit.chunks.map(processChunk));
         }
-
-        // parallel processing, dropped 
-        // result = await Promise.all(mediaSplit.chunks.map(processChunk));
-
     } catch (error) {
         console.error("An error occurred:", error);
     } finally {
         try {
-            isYoutubeUrl && (await deleteTempPath(file));
+            (isYoutubeUrl||isDocument) && (await deleteTempPath(file));
             folder && (await deleteTempPath(folder));
         } catch (error) {
             console.error("An error occurred while deleting:", error);