@aj-archipelago/cortex 1.0.16 → 1.0.18

package/helper_apps/MediaFileChunker/docHelper.js

@@ -38,10 +38,10 @@ export async function pdfToText(filePath) {
         const page = await pdf.getPage(i);
         const textContent = await page.getTextContent();
         const strings = textContent.items.map(item => item.str);
-        finalText += strings.join(' ');
+        finalText += strings.join(' ') + '\n';
     }
 
-    return finalText;
+    return finalText.trim();
 }
 
 export async function csvToText(filePath) {

package/helper_apps/MediaFileChunker/index.js

@@ -9,6 +9,8 @@ import os from 'os';
 import { v4 as uuidv4 } from 'uuid';
 import fs from 'fs';
 
+const DOC_EXTENSIONS =  [".txt", ".json", ".csv", ".md", ".xml", ".js", ".html", ".css", '.pdf', '.docx', '.xlsx', '.csv'];
+
 const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
 console.log(useAzure ? 'Using Azure Storage' : 'Using local file system');
 
@@ -64,7 +66,7 @@ async function main(context, req) {
         await publishRequestProgress({ requestId, progress, completedCount, totalCount, numberOfChunks, data });
     }
 
-    const isDocument = ['.pdf', '.txt', '.docx', '.xlsx', '.csv'].some(ext => uri.toLowerCase().endsWith(ext));
+    const isDocument = DOC_EXTENSIONS.some(ext => uri.toLowerCase().endsWith(ext));
 
     try {
         if (isDocument) {
@@ -72,20 +74,47 @@ async function main(context, req) {
             const file = path.join(os.tmpdir(), `${uuidv4()}${extension}`);
             await downloadFile(uri, file)
             const text = await documentToText(file);
-            if (save) {
-                const fileName = `${uuidv4()}.txt`; // generate unique file name
-                const filePath = path.join(os.tmpdir(), fileName);
-                const tmpPath = filePath;
-                fs.writeFileSync(filePath, text); // write text to file
-        
-                // save file to the cloud or local file system
-                const saveResult = useAzure ? await saveFileToBlob(filePath, requestId) : await moveFileToPublicFolder(filePath, requestId);
-                result.push(saveResult);
-        
-                // delete temporary file
-                fs.unlinkSync(tmpPath);
-            } else {
-                result.push(...easyChunker(text));
+            let tmpPath;
+
+            try{
+                if (save) {
+                    const fileName = `${uuidv4()}.txt`; // generate unique file name
+                    const filePath = path.join(os.tmpdir(), fileName);
+                    tmpPath = filePath;
+                    fs.writeFileSync(filePath, text); // write text to file
+            
+                    // save file to the cloud or local file system
+                    const saveResult = useAzure ? await saveFileToBlob(filePath, requestId) : await moveFileToPublicFolder(filePath, requestId);
+                    result.push(saveResult);
+
+                } else {
+                    result.push(...easyChunker(text));
+                }
+            }catch(err){
+                console.log(`Error saving file ${uri} with request id ${requestId}:`, err);
+            }finally{
+                try{
+                    // delete temporary files
+                    tmpPath && fs.unlinkSync(tmpPath);
+                    file && fs.unlinkSync(file);
+                    console.log(`Cleaned temp files ${tmpPath}, ${file}`);
+                }catch(err){
+                    console.log(`Error cleaning temp files ${tmpPath}, ${file}:`, err);
+                }
+                
+                try{
+                    //delete uploaded prev nontext file
+                    //check cleanup for whisper temp uploaded files url
+                    const regex = /whispertempfiles\/([a-z0-9-]+)/;
+                    const match = uri.match(regex);
+                    if (match && match[1]) {
+                        const extractedValue = match[1];
+                        useAzure ? await deleteBlob(extractedValue) : await deleteFolder(extractedValue);
+                        console.log(`Cleaned temp file ${uri} with request id ${extractedValue}`);
+                    }
+                }catch(err){
+                    console.log(`Error cleaning temp file ${uri}:`, err);
+                }
             }
         }else{
 
@@ -123,7 +152,7 @@ async function main(context, req) {
         console.error("An error occurred:", error);
     } finally {
         try {
-            (isYoutubeUrl||isDocument) && (await deleteTempPath(file));
+            (isYoutubeUrl) && (await deleteTempPath(file));
             folder && (await deleteTempPath(folder));
         } catch (error) {
             console.error("An error occurred while deleting:", error);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.0.16",
+  "version": "1.0.18",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",

package/server/plugins/azureCognitivePlugin.js

@@ -10,6 +10,8 @@ const API_URL = config.get('whisperMediaApiUrl');
 
 const TOP = 1000;
 
+let DIRECT_FILE_EXTENSIONS = [".txt", ".json", ".csv", ".md", ".xml", ".js", ".html", ".css"];
+
 class AzureCognitivePlugin extends ModelPlugin {
     constructor(config, pathway, modelName, model) {
         super(config, pathway, modelName, model);
@@ -51,14 +53,24 @@ class AzureCognitivePlugin extends ModelPlugin {
         }
 
         if (mode == 'index') {
+
+            /*
             const calculateInputVector = async () => {
-                return JSON.parse(await callPathway(this.config, 'embeddings', { text }))[0];
+                try{
+                    if(!text || !text.trim()){
+                        return;
+                    }
+                    return JSON.parse(await callPathway(this.config, 'embeddings', { text }))[0];
+                }catch(err){
+                    console.log(`Error in calculating input vector for text: ${text}, error: ${err}`);
+                }
             }
+            */
 
             const doc = {
                 id: uuidv4(),
                 content: text,
-                contentVector: inputVector || (await calculateInputVector()),
+                //contentVector: inputVector || (await calculateInputVector()),
                 owner: savedContextId,
                 docId: docId || uuidv4(),
                 createdAt: new Date().toISOString()
@@ -141,7 +153,7 @@ class AzureCognitivePlugin extends ModelPlugin {
             let url = file;
             //if not txt file, use helper app to convert to txt
             const extension = path.extname(file).toLowerCase();
-            if (extension !== '.txt') {
+            if (!DIRECT_FILE_EXTENSIONS.includes(extension)) {
                 try {
                     const {data} = await axios.get(API_URL, { params: { uri: file, requestId, save: true } });
                     url = data[0]
@@ -154,10 +166,18 @@ class AzureCognitivePlugin extends ModelPlugin {
             const { data } = await axios.get(url);
             await this.markCompletedForCleanUp(requestId);
 
+            if(!data){
+                throw Error(`No data can be extracted out of file!`);
+            }
+
             //return await this.execute(data, {...parameters, file:null}, prompt, pathwayResolver); 
             return await callPathway(this.config, 'cognitive_insert', {...parameters, file:null, text:data });
         }
 
+        if (mode === 'index' && (!text || !text.trim()) ){
+            return; // nothing to index
+        }
+
         const { data, params } = await this.getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId, {headers, requestId, pathway, url});
 
         // update contextid last used