@aj-archipelago/cortex 1.0.18 → 1.0.20

package/config.js

@@ -95,6 +95,7 @@ var config = convict({
                     "api-key": "{{AZURE_COGNITIVE_API_KEY}}",
                     "Content-Type": "application/json"
                 },
+                "requestsPerSecond": 6
             },
             "oai-embeddings": {
                 "type": "OPENAI-EMBEDDINGS",
@@ -148,6 +149,11 @@ var config = convict({
         sensitive: true,
         env: 'STORAGE_CONNECTION_STRING'
     },
+    dalleImageApiUrl: {
+        format: String,
+        default: 'null',
+        env: 'DALLE_IMAGE_API_URL'
+    },
     whisperMediaApiUrl: {
         format: String,
         default: 'null',

package/helper_apps/MediaFileChunker/docHelper.js

@@ -30,17 +30,49 @@ export async function xlsxToText(filePath) {
     return finalText;
 }
 
-export async function pdfToText(filePath) {
+async function pdfToText(filePath) {
     const pdf = await pdfjsLib.getDocument(filePath).promise;
+    const meta = await pdf.getMetadata();
+
+    // Check if pdf is scanned
+    if (meta && meta.metadata && meta.metadata._metadataMap && meta.metadata._metadataMap.has('dc:format')) {
+        const format = meta.metadata._metadataMap.get('dc:format');
+        if (format && format._value && format._value.toLowerCase() === 'application/pdf; version=1.3') {
+            throw new Error('Scanned PDFs are not supported');
+        }
+    }
+
+    // Check if pdf is encrypted
+    if (pdf._pdfInfo && pdf._pdfInfo.encrypt) {
+        throw new Error('Encrypted PDFs are not supported');
+    }
+
+    // Check if pdf is password protected
+    if (pdf._passwordNeeded) {
+        throw new Error('Password protected PDFs are not supported');
+    }
+
     let finalText = '';
+    let ocrNeeded = true; // Initialize the variable as true
 
-    for(let i = 1; i <= pdf.numPages; i++) {
+    for (let i = 1; i <= pdf.numPages; i++) {
         const page = await pdf.getPage(i);
+        const operatorList = await page.getOperatorList();
+
+        // Check if there are any fonts used in the PDF
+        if (operatorList.fnArray.some(fn => fn === pdfjsLib.OPS.setFont)) {
+            ocrNeeded = false; // Set ocrNeeded to false if fonts are found
+        }
+
         const textContent = await page.getTextContent();
         const strings = textContent.items.map(item => item.str);
         finalText += strings.join(' ') + '\n';
     }
 
+    if (ocrNeeded) {
+        throw new Error('OCR might be needed for this document!');
+    }
+
     return finalText.trim();
 }
 

package/helper_apps/MediaFileChunker/index.js

@@ -150,6 +150,9 @@ async function main(context, req) {
         }
     } catch (error) {
         console.error("An error occurred:", error);
+        context.res.status(500);
+        context.res.body = error.message || error;
+        return;
     } finally {
         try {
             (isYoutubeUrl) && (await deleteTempPath(file));
@@ -159,13 +162,11 @@ async function main(context, req) {
         }
     }
 
-
     console.log(`result: ${result}`);
-
     context.res = {
-        // status: 200, /* Defaults to 200 */
         body: result
     };
+
 }
 
 

package/lib/pathwayTools.js

@@ -1,4 +1,5 @@
 // pathwayTools.js
+import { encode , decode } from 'gpt-3-encoder';
 
 // callPathway - call a pathway from another pathway
 const callPathway = async (config, pathwayName, args) => {
@@ -12,4 +13,12 @@ const callPathway = async (config, pathwayName, args) => {
     return data?.result;
 };
 
-export { callPathway };
+const gpt3Encode = (text) => {
+    return encode(text);
+}
+
+const gpt3Decode = (text) => {
+    return decode(text);
+}
+
+export { callPathway, gpt3Encode, gpt3Decode };

package/lib/request.js

@@ -169,7 +169,8 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
         try {
             const response = await Promise.race(promises);
 
-            if (response.status === 200) {
+            // if response status is 2xx
+            if (response.status >= 200 && response.status < 300) {
                 return response;
             } else {
                 throw new Error(`Received error response: ${response.status}`);

package/lib/requestDurationEstimator.js

@@ -0,0 +1,90 @@
+/**
+ * A class to get request durations and estimate their average.
+ */
+export default class RequestDurationEstimator {
+    // Initializing the class with given number of durations to track.
+    constructor(n = 10) {
+        this.n = n;  // Number of last durations to consider
+        this.durations = [];  // List to keep track of last n durations
+    }
+
+    /**
+     * Private method to add a request duration to the durations list.
+     * If the list is full (n durations already), the oldest duration is removed.
+     * @param {number} duration - The duration of the request
+     */
+    #add(duration) {
+        this.durations.push(duration);
+        // Remove the oldest duration if we have stored n durations
+        if (this.durations.length > this.n) {
+            this.durations.shift();
+        }
+    }
+
+    /**
+     * To be invoked when a request starts.
+     * If there is an ongoing request, it ends that request.
+     * @param {string} requestId - The ID of the request
+     */
+    startRequest(requestId) {
+        // If there is an ongoing request, end it
+        if (this.requestId) {
+            this.endRequest();
+        }
+
+        // Store the starting details of the new request
+        this.requestId = requestId;
+        this.startTime = Date.now();
+    }
+
+    /**
+     * To be invoked when a request ends.
+     * Calculates the duration of the request and adds it to the durations list.
+     */
+    endRequest() {
+        // If there is an ongoing request, add its duration to the durations list
+        if (this.requestId) {
+            this.#add(Date.now() - this.startTime);
+            this.requestId = null;
+        }
+    }
+
+    /**
+     * Calculate and return the average of the request durations.
+     * @return {number} The average request duration
+     */
+    getAverage() {
+        // If no duration is stored, return 0
+        if (!this.durations.length) {
+            return 0;
+        }
+
+        // Calculate the sum of the durations and divide by the number of durations to get the average
+        return this.durations.reduce((a, b) => a + b) / this.durations.length;
+    }
+
+    /**
+     * Calculate the percentage completion of the current request based on the average of past durations.
+     * @return {number} The estimated percent completion of the ongoing request
+     */
+    calculatePercentComplete() {
+        // If no duration is stored, return 0
+        if (!this.durations.length) {
+            return 0;
+        }
+
+
+        // Calculate the duration of the current request
+        const duration = Date.now() - this.startTime;
+        // Get the average of the durations
+        const average = this.getAverage();
+        // Calculate the percentage completion
+        let percentComplete = duration / average;
+
+        if (percentComplete > .8) {
+            percentComplete = 0.8;
+        }
+
+        return percentComplete;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.0.18",
+  "version": "1.0.20",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",

package/pathways/basePathway.js

@@ -26,5 +26,7 @@ export default {
     // args: the input arguments to the pathway
     // runAllPrompts: a function that runs all prompts in the pathway and returns the result
     executePathway: undefined,
+    // Set the temperature to 0 to favor more deterministic output when generating entity extraction.
+    temperature: undefined,
 };
 

package/pathways/cognitive_search.js

@@ -1,5 +1,5 @@
 export default {
-    prompt: `{{text}}`,
+    prompt: `{{{text}}}`,
     model: 'azure-cognitive',
     inputParameters: {
         inputVector: ``,

package/pathways/image.js

@@ -0,0 +1,4 @@
+export default {
+    prompt:["{{text}}"],
+    model: 'azure-dalle',
+}

package/server/graphql.js

@@ -72,6 +72,7 @@ const getTypedefs = (pathways) => {
     type RequestSubscription {
         requestId: String
         progress: Float
+        status: String
         data: String
     }
 

package/server/pathwayPrompter.js

@@ -12,16 +12,20 @@ import CohereGeneratePlugin from './plugins/cohereGeneratePlugin.js';
 import CohereSummarizePlugin from './plugins/cohereSummarizePlugin.js';
 import AzureCognitivePlugin from './plugins/azureCognitivePlugin.js';
 import OpenAiEmbeddingsPlugin from './plugins/openAiEmbeddingsPlugin.js';
+import OpenAIImagePlugin from './plugins/openAiImagePlugin.js';
 
 class PathwayPrompter {
     constructor(config, pathway, modelName, model) {
-        
+
         let plugin;
 
         switch (model.type) {
             case 'OPENAI-CHAT':
                 plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
                 break;
+            case 'OPENAI-IMAGE':
+                plugin = new OpenAIImagePlugin(config, pathway, modelName, model);
+                break;
             case 'OPENAI-CHAT-EXTENSION':
                 plugin = new OpenAIChatExtensionPlugin(config, pathway, modelName, model);
                 break;

package/server/pathwayResolver.js

@@ -10,6 +10,8 @@ import { getv, setv } from '../lib/keyValueStorageClient.js';
 import { requestState } from './requestState.js';
 import { callPathway } from '../lib/pathwayTools.js';
 
+const modelTypesExcludedFromProgressUpdates = ['OPENAI-IMAGE'];
+
 class PathwayResolver {
     constructor({ config, pathway, args }) {
         this.config = config;
@@ -78,13 +80,17 @@ class PathwayResolver {
             if (args.async || typeof responseData === 'string') {
                 const { completedCount, totalCount } = requestState[this.requestId];
                 requestState[this.requestId].data = responseData;
-                pubsub.publish('REQUEST_PROGRESS', {
-                    requestProgress: {
-                        requestId: this.requestId,
-                        progress: completedCount / totalCount,
-                        data: JSON.stringify(responseData),
-                    }
-                });
+                
+                // if model type is OPENAI-IMAGE
+                if (!modelTypesExcludedFromProgressUpdates.includes(this.model.type)) {
+                    pubsub.publish('REQUEST_PROGRESS', {
+                        requestProgress: {
+                            requestId: this.requestId,
+                            progress: completedCount / totalCount,
+                            data: JSON.stringify(responseData),
+                        }
+                    });
+                }
             } else {
                 try {
                     const incomingMessage = responseData;
@@ -204,7 +210,7 @@ class PathwayResolver {
     async promptAndParse(args) {
         // Get saved context from contextId or change contextId if needed
         const { contextId } = args;
-        this.savedContextId = contextId ? contextId : null;
+        this.savedContextId = contextId ? contextId : uuidv4();
         this.savedContext = contextId ? (getv && (await getv(contextId)) || {}) : {};
 
         // Save the context before processing the request
@@ -290,7 +296,7 @@ class PathwayResolver {
         text = await this.summarizeIfEnabled({ text, ...parameters }); // summarize if flag enabled
         const chunks = this.processInputText(text);
 
-        const anticipatedRequestCount = chunks.length * this.prompts.length;
+        let anticipatedRequestCount = chunks.length * this.prompts.length   
 
         if ((requestState[this.requestId] || {}).canceled) {
             throw new Error('Request canceled');

package/server/plugins/azureCognitivePlugin.js

@@ -155,11 +155,12 @@ class AzureCognitivePlugin extends ModelPlugin {
             const extension = path.extname(file).toLowerCase();
             if (!DIRECT_FILE_EXTENSIONS.includes(extension)) {
                 try {
-                    const {data} = await axios.get(API_URL, { params: { uri: file, requestId, save: true } });
-                    url = data[0]
+                    const { data }  = await axios.get(API_URL, { params: { uri: file, requestId, save: true } });
+                    url = data[0];
                 } catch (error) {
-                    console.log(`Error converting file ${file} to txt:`, error);
-                    throw error;
+                    console.error(`Error converting file ${file} to txt:`, error);
+                    await this.markCompletedForCleanUp(requestId);
+                    throw Error(error?.response?.data || error?.message || error);
                 }
             }
  

package/server/plugins/modelPlugin.js

@@ -197,6 +197,13 @@ class ModelPlugin {
             }
         });
      
+        // Clean up any null messages if they exist
+        expandedMessages.forEach((message) => {
+            if (typeof message === 'object' && message.content === null) {
+                message.content = '';
+            }
+        });
+        
         return expandedMessages;
     }
 

package/server/plugins/openAiChatPlugin.js

@@ -57,7 +57,7 @@ class OpenAIChatPlugin extends ModelPlugin {
         if (isPalmFormat) {
             const context = modelPrompt.context || '';
             const examples = modelPrompt.examples || [];
-            requestMessages = this.convertPalmToOpenAIMessages(context, examples, expandedMessages);
+            requestMessages = this.convertPalmToOpenAIMessages(context, examples, modelPromptMessages);
         }
     
         // Check if the token length exceeds the model's max token length

package/server/plugins/openAiImagePlugin.js

@@ -0,0 +1,85 @@
+// OpenAIImagePlugin.js
+import FormData from 'form-data';
+import { config } from '../../config.js';
+import ModelPlugin from './modelPlugin.js';
+import pubsub from '../pubsub.js';
+import axios from 'axios';
+import RequestDurationEstimator from '../../lib/requestDurationEstimator.js';
+
+const API_URL = config.get('dalleImageApiUrl'); // URL for the DALL-E API
+const requestDurationEstimator = new RequestDurationEstimator(10);
+
+class OpenAIImagePlugin extends ModelPlugin {
+    constructor(config, pathway, modelName, model) {
+        super(config, pathway, modelName, model);
+    }
+
+    // Implement the method to call the DALL-E API
+    async execute(text, parameters, _, pathwayResolver) {
+        const url = this.requestUrl(text);
+        const data = JSON.stringify({ prompt: text });
+
+        let id;
+        const { requestId, pathway } = pathwayResolver;
+
+        try {
+            requestDurationEstimator.startRequest(requestId);
+            id = (await this.executeRequest(url, data, {}, { ...this.model.headers }, {}, requestId, pathway))?.id;
+        } catch (error) {
+            const errMsg = `Error generating image: ${error?.message || JSON.stringify(error)}`;
+            console.error(errMsg);
+            return errMsg;
+        }
+
+        if (!parameters.async) {
+            return await this.getStatus(text, id, requestId);
+        }
+        else {
+            this.getStatus(text, id, requestId);
+        }
+    }
+
+    async getStatus(text, id, requestId) {
+        // get the post URL which is used to send the request
+        const url = this.requestUrl(text);
+
+        // conver it to the GET URL which is used to check the status
+        const statusUrl = url.replace("images/generations:submit", `operations/images/${id}`);
+        let status;
+        let attemptCount = 0;
+        let data = null;
+
+        do {
+            const response = (await axios.get(statusUrl, { cache: false, headers: { ...this.model.headers } })).data;
+            status = response.status;
+            let progress = 
+                requestDurationEstimator.calculatePercentComplete();
+
+            if (status === "succeeded") {
+                progress = 1;
+                data = JSON.stringify(response);
+            }
+
+            pubsub.publish('REQUEST_PROGRESS', {
+                requestProgress: {
+                    requestId,
+                    status,
+                    progress,
+                    data,
+                }
+            });
+
+            if (status === "succeeded") {
+                requestDurationEstimator.endRequest();
+                break;
+            }
+            // sleep for 5 seconds
+            await new Promise(resolve => setTimeout(resolve, 2000));
+        }
+        while (status !== "succeeded" && attemptCount++ < 30);
+
+        return data;
+    }
+}
+
+export default OpenAIImagePlugin;

package/server/plugins/openAiWhisperPlugin.js

@@ -21,7 +21,7 @@ const pipeline = promisify(stream.pipeline);
 const API_URL = config.get('whisperMediaApiUrl');
 const WHISPER_TS_API_URL  = config.get('whisperTSApiUrl');
 
-function alignSubtitles(subtitles) {
+function alignSubtitles(subtitles, format) {
     const result = [];
     const offset = 1000 * 60 * 10; // 10 minutes for each chunk
 
@@ -39,7 +39,7 @@ function alignSubtitles(subtitles) {
         const subtitle = subtitles[i];
         result.push(...shiftSubtitles(subtitle, i * offset));
     }
-    return subsrt.build(result);
+    return subsrt.build(result, { format: format === 'vtt' ? 'vtt' : 'srt' });
 }
 
 function generateUniqueFilename(extension) {
@@ -234,7 +234,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
         }
 
         if (['srt','vtt'].includes(responseFormat) || wordTimestamped) { // align subtitles for formats
-            return alignSubtitles(result);
+            return alignSubtitles(result, responseFormat);
         }
         return result.join(` `);
     }

package/tests/requestDurationEstimator.test.js

@@ -0,0 +1,59 @@
+import test from 'ava';
+import RequestDurationEstimator from '../lib/requestDurationEstimator.js';
+
+test('add and get average request duration', async (t) => {
+    const estimator = new RequestDurationEstimator(5);
+
+    estimator.startRequest('req1');
+    await new Promise(resolve => setTimeout(() => {
+        estimator.endRequest();
+
+        const average = estimator.calculatePercentComplete();
+
+        // An average should be calculated after the first completed request  
+        t.not(average, 0);
+        resolve();
+    }, 1000));
+});
+
+test('add more requests than size of durations array', (t) => {
+    const estimator = new RequestDurationEstimator(5);
+
+    for (let i = 0; i < 10; i++) {
+        estimator.startRequest(`req${i}`);
+        estimator.endRequest();
+    }
+
+    // Array size should not exceed maximum length (5 in this case)
+    t.is(estimator.durations.length, 5);
+});
+
+test('calculate percent complete of current request based on average of past durations', async (t) => {
+    const estimator = new RequestDurationEstimator(5);
+
+    for (let i = 0; i < 4; i++) {
+        estimator.startRequest(`req${i}`);
+        // wait 1 second
+        await new Promise(resolve => setTimeout(resolve, 1000));
+        estimator.endRequest();
+    }
+
+    estimator.startRequest('req5');
+
+    await new Promise(resolve => setTimeout(() => {
+        const percentComplete = estimator.calculatePercentComplete();
+
+        // Depending on how fast the operations are,
+        // the percentage may not be exactly 50%, but
+        // we'll affirm it should be at least partially complete.
+        t.true(percentComplete > 0);
+        resolve();
+    }, 500));
+});
+
+test('calculate percent complete based on average of past durations', async (t) => {
+    const estimator = new RequestDurationEstimator(5);
+    estimator.durations = [1000, 2000, 3000];
+    const average = estimator.getAverage();
+    t.is(average, 2000);
+});