@aj-archipelago/cortex 1.0.12 → 1.0.13

package/helper_apps/MediaFileChunker/package.json

@@ -13,11 +13,15 @@
     "@ffprobe-installer/ffprobe": "^2.0.0",
     "axios": "^1.3.6",
     "busboy": "^1.6.0",
+    "cors": "^2.8.5",
     "express": "^4.18.2",
     "fluent-ffmpeg": "^2.1.2",
     "ioredis": "^5.3.1",
+    "mammoth": "^1.6.0",
+    "pdfjs-dist": "^3.9.179",
     "public-ip": "^6.0.1",
     "uuid": "^9.0.0",
-    "ytdl-core": "^4.11.5"
+    "ytdl-core": "git+ssh://git@github.com:khlevon/node-ytdl-core.git#v4.11.4-patch.2",
+    "xlsx": "^0.18.5"
   }
 }

package/helper_apps/MediaFileChunker/start.js

@@ -2,6 +2,7 @@ import MediaFileChunker from "./index.js";
 import express from "express";
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
+import cors from 'cors';
 
 import { publicIpv4 } from 'public-ip';
 const ipAddress = await publicIpv4();
@@ -10,7 +11,7 @@ const app = express();
 const port = process.env.PORT || 7071;
 const publicFolder = join(dirname(fileURLToPath(import.meta.url)), 'files');
 
-
+app.use(cors());
 // Serve static files from the public folder
 app.use('/files', express.static(publicFolder));
 

package/lib/request.js

@@ -70,16 +70,16 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
     let promises = [];
     for (let i = 0; i < MAX_RETRY; i++) {
         const modelProperties = config.get('models')[model];
-        const enableDuplicateRequests = pathway.enableDuplicateRequests !== undefined ? pathway.enableDuplicateRequests : config.get('enableDuplicateRequests');
+        const enableDuplicateRequests = pathway?.enableDuplicateRequests !== undefined ? pathway.enableDuplicateRequests : config.get('enableDuplicateRequests');
         let maxDuplicateRequests = enableDuplicateRequests ? MAX_DUPLICATE_REQUESTS : 1;
-        let duplicateRequestAfter = (pathway.duplicateRequestAfter || DUPLICATE_REQUEST_AFTER) * 1000;
+        let duplicateRequestAfter = (pathway?.duplicateRequestAfter || DUPLICATE_REQUEST_AFTER) * 1000;
 
         if (enableDuplicateRequests) {
             //console.log(`>>> [${requestId}] Duplicate requests enabled after ${duplicateRequestAfter / 1000} seconds`);
         }
 
         const axiosConfigObj = { params, headers, cache };
-        const streamRequested = (params.stream || data.stream);
+        const streamRequested = (params?.stream || data?.stream);
         if (streamRequested && modelProperties.supportsStreaming) {
             axiosConfigObj.responseType = 'stream';
             promises.push(limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj)));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.0.12",
+  "version": "1.0.13",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",

package/pathways/cognitive_delete.js

@@ -0,0 +1,10 @@
+export default {
+    // prompt: `{{text}}`,
+    model: 'azure-cognitive',
+    inputParameters: {
+        docId: ``
+    },
+    mode: 'delete', 
+    enableDuplicateRequests: false,
+    timeout: 300,
+};

package/pathways/cognitive_insert.js

@@ -0,0 +1,14 @@
+export default {
+    prompt: `{{text}}`,
+    model: 'azure-cognitive',
+    inputParameters: {
+        inputVector: ``,
+        file: ``,
+        privateData: true,
+        docId: ``,
+    },
+    mode: 'index', // 'index' or 'search',
+    inputChunkSize:  500,
+    enableDuplicateRequests: false,
+    timeout: 300,
+};

package/pathways/cognitive_search.js

@@ -0,0 +1,13 @@
+export default {
+    prompt: `{{text}}`,
+    model: 'azure-cognitive',
+    inputParameters: {
+        inputVector: ``,
+        privateData: false,
+        filter: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+    },
+    enableDuplicateRequests: false,
+    timeout: 300,
+};

package/pathways/embeddings.js

@@ -0,0 +1,14 @@
+// embeddings.js
+// Embeddings module that returns the embeddings for the text. 
+
+export default {
+    prompt: `{{text}}`,
+    model: 'oai-embeddings',
+    enableCache: true,
+    inputParameters: {
+        input: [],
+    },
+    enableDuplicateRequests: false,
+    timeout: 300,
+};
+

package/pathways/index.js

@@ -1,6 +1,9 @@
 import edit from './edit.js';
 import chat from './chat.js';
 import bias from './bias.js';
+import cognitive_delete from './cognitive_delete.js';
+import cognitive_insert from './cognitive_insert.js';
+import cognitive_search from './cognitive_search.js';
 import complete from './complete.js';
 import entities from './entities.js';
 import paraphrase from './paraphrase.js';
@@ -13,12 +16,17 @@ import test_langchain from './test_langchain.mjs';
 import test_palm_chat from './test_palm_chat.js';
 import transcribe from './transcribe.js';
 import translate from './translate.js';
+import embeddings from './embeddings.js';
 
 export {
     edit,
     chat,
     bias,
+    cognitive_delete,
+    cognitive_insert,
+    cognitive_search,
     complete,
+    embeddings,
     entities,
     paraphrase,
     sentiment,

package/pathways/language.js

@@ -0,0 +1,10 @@
+// language.js
+// Analyze the language of a given text and return the language code.
+
+export default {
+    // Uncomment the following line to enable caching for this prompt, if desired.
+    enableCache: true,
+    temperature: 0,
+    
+    prompt: `{{text}}\n\nPick one language that best represents what the text above is written in. Please return the ISO 639-1 two letter language code:\n`
+};

package/server/pathwayPrompter.js

@@ -10,6 +10,8 @@ import PalmCompletionPlugin from './plugins/palmCompletionPlugin.js';
 import PalmCodeCompletionPlugin from './plugins/palmCodeCompletionPlugin.js';
 import CohereGeneratePlugin from './plugins/cohereGeneratePlugin.js';
 import CohereSummarizePlugin from './plugins/cohereSummarizePlugin.js';
+import AzureCognitivePlugin from './plugins/azureCognitivePlugin.js';
+import OpenAiEmbeddingsPlugin from './plugins/openAiEmbeddingsPlugin.js';
 
 class PathwayPrompter {
     constructor(config, pathway, modelName, model) {
@@ -26,6 +28,12 @@ class PathwayPrompter {
             case 'AZURE-TRANSLATE':
                 plugin = new AzureTranslatePlugin(config, pathway, modelName, model);
                 break;
+            case 'AZURE-COGNITIVE':
+                plugin = new AzureCognitivePlugin(config, pathway, modelName, model);
+                break;
+            case 'OPENAI-EMBEDDINGS':
+                plugin = new OpenAiEmbeddingsPlugin(config, pathway, modelName, model);
+                break;
             case 'OPENAI-COMPLETION':
                 plugin = new OpenAICompletionPlugin(config, pathway, modelName, model);
                 break;

package/server/pathwayResolver.js

@@ -382,7 +382,7 @@ class PathwayResolver {
         let result = '';
 
         // If this text is empty, skip applying the prompt as it will likely be a nonsensical result
-        if (!/^\s*$/.test(text) || parameters?.file) {
+        if (!/^\s*$/.test(text) || parameters?.file || parameters?.inputVector || this?.modelName.includes('cognitive')) {
             result = await this.pathwayPrompter.execute(text, { ...parameters, ...this.savedContext }, prompt, this);
         } else {
             result = text;

package/server/plugins/azureCognitivePlugin.js

@@ -0,0 +1,143 @@
+// Azure Cognitive Services plugin for the server
+import { callPathway } from '../../lib/pathwayTools.js';
+import ModelPlugin from './modelPlugin.js';
+import { v4 as uuidv4 } from 'uuid';
+
+const TOP = 1000;
+
+class AzureCognitivePlugin extends ModelPlugin {
+    constructor(config, pathway, modelName, model) {
+        super(config, pathway, modelName, model);
+    }
+
+    // Set up parameters specific to the Azure Cognitive API
+    async getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId,  {headers, requestId, pathway, url}) {
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+        const { modelPromptText } = this.getCompiledPrompt(text, combinedParameters, prompt);
+        const { inputVector, filter, docId } = combinedParameters;
+        const data = {};
+
+        if (mode == 'delete') {
+            const searchUrl = this.ensureMode(this.requestUrl(text), 'search');
+            let searchQuery = `owner:${savedContextId}`;
+            
+            if (docId) {
+                searchQuery += ` AND docId:'${docId}'`;
+            }
+
+            const docsToDelete = JSON.parse(await this.executeRequest(searchUrl,
+                { search: searchQuery,  
+                    "searchMode": "all",
+                    "queryType": "full",
+                    select: 'id', top: TOP 
+                },
+                {}, headers, prompt, requestId, pathway));
+        
+            const value = docsToDelete.value.map(({id}) => ({
+                id,
+                "@search.action": "delete"
+            }));
+        
+            return {
+                data: {
+                    value
+                }
+            };
+        }
+
+        if (mode == 'index') {
+            const calculateInputVector = async () => {
+                return JSON.parse(await callPathway(this.config, 'embeddings', { text }))[0];
+            }
+
+            const doc = {
+                id: uuidv4(),
+                content: text,
+                contentVector: inputVector || (await calculateInputVector()),
+                owner: savedContextId,
+                docId: docId || uuidv4()
+            }
+            // if(!privateData){
+            //     delete doc.owner;
+            // }
+            data.value = [doc];
+            return { data };
+        }
+
+        //default mode, 'search'
+        if (inputVector) {
+            data.vectors = [
+                {
+                    "value": typeof inputVector === 'string' ? JSON.parse(inputVector) : inputVector,
+                    "fields": "contentVector",
+                    "k": 20
+                }
+            ];
+        } else {
+            data.search = modelPromptText;
+        }
+
+        filter && (data.filter = filter);
+        if (indexName == 'indexcortex') { //if private, filter by owner via contextId //privateData && 
+            data.filter && (data.filter = data.filter + ' and ');
+            data.filter = `owner eq '${savedContextId}'`;
+        }
+
+        return { data };
+    }
+
+    ensureMode(url, mode) {
+        const pattern = new RegExp(`indexes\/.*\/docs\/${mode}`);
+        if (pattern.test(url)) {
+            // if the URL is already in the correct form, return it as is
+            return url;
+        } else {
+            // otherwise, perform the replacement
+            return url.replace(/(indexes\/.*\/docs\/)([^?]+)/, `$1${mode}`);
+        }
+    }
+
+    ensureIndex(url, indexName) {
+        const pattern = new RegExp(`indexes\/${indexName}\/docs\/search`);
+        if (pattern.test(url)) {
+            // if the URL is already in the correct form, return it as is
+            return url;
+        } else {
+            // otherwise, perform the replacement
+            return url.replace(/(indexes\/)([^\/]+)/, `$1${indexName}`);
+        }
+    }
+
+    // Execute the request to the Azure Cognitive API
+    async execute(text, parameters, prompt, pathwayResolver) {
+        const { requestId, pathway, savedContextId } = pathwayResolver;
+        const mode = this.promptParameters.mode || 'search';
+        let url = this.ensureMode(this.requestUrl(text), mode == 'delete' ? 'index' : mode);
+        const indexName = parameters.indexName || 'indexcortex';
+        url = this.ensureIndex(url, indexName);
+        const headers = this.model.headers;
+
+        const { data, params } = await this.getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId, {headers, requestId, pathway, url});
+
+        if (mode === 'delete' && data.value.length == 0){
+            return; // nothing to delete
+        }
+
+        // execute the request
+        const result = await this.executeRequest(url, data || {}, params || {}, headers || {}, prompt, requestId, pathway);
+
+        // if still has more to delete
+        if (mode === 'delete' && data?.value?.length == TOP) { 
+            return await this.execute(text, parameters, prompt, pathwayResolver);
+        }
+        
+        return result;
+    }
+
+    parseResponse(data) {
+        return JSON.stringify(data || {});
+    }
+
+}
+
+export default AzureCognitivePlugin;

package/server/plugins/openAiEmbeddingsPlugin.js

@@ -0,0 +1,38 @@
+// OpenAiEmbeddingsPlugin.js
+import ModelPlugin from './modelPlugin.js';
+
+class OpenAiEmbeddingsPlugin extends ModelPlugin {
+    constructor(config, pathway, modelName, model) {
+        super(config, pathway, modelName, model);
+    }
+
+    getRequestParameters(text, parameters, prompt) {
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+        const { modelPromptText } = this.getCompiledPrompt(text, combinedParameters, prompt);
+        const requestParameters = {
+            data:  {
+                input: combinedParameters?.input?.length ? combinedParameters.input :  modelPromptText || text,
+            }
+        };
+        return requestParameters;
+    }
+
+    async execute(text, parameters, prompt, pathwayResolver) {
+        const { requestId, pathway} = pathwayResolver;
+        const requestParameters = this.getRequestParameters(text, parameters, prompt);
+        const url = this.requestUrl();
+
+        const data = requestParameters.data || {};
+        const params = requestParameters.params || {};
+        const headers = this.model.headers || {};
+
+        return this.executeRequest(url, data, params, headers, prompt, requestId, pathway);
+    }
+
+    parseResponse(data) {
+        return JSON.stringify(data?.data?.map( ({embedding}) => embedding) || []);
+    }
+
+}
+
+export default OpenAiEmbeddingsPlugin;