@aj-archipelago/cortex 0.0.9 → 0.0.11

package/config.js

@@ -1,7 +1,8 @@
-const path = require('path');
-const convict = require('convict');
-const handlebars = require("handlebars");
-const fs = require('fs');
+import path from 'path';
+const __dirname = path.dirname(new URL(import.meta.url).pathname);
+import convict from 'convict';
+import handlebars from 'handlebars';
+import fs from 'fs';
 
 // Schema for config
 var config = convict({
@@ -108,7 +109,13 @@ var config = convict({
         format: String,
         default: null,
         env: 'CORTEX_CONFIG_FILE'
-    }
+    },
+    serpApiKey: {
+        format: String,
+        default: null,
+        env: 'SERPAPI_API_KEY',
+        sensitive: true
+    },
 });
 
 // Read in environment variables and set up service configuration
@@ -127,22 +134,21 @@ if (configFile && fs.existsSync(configFile)) {
     }
 }
 
-
 // Build and load pathways to config
-const buildPathways = (config) => {
+const buildPathways = async (config) => {
     const { pathwaysPath, corePathwaysPath, basePathwayPath } = config.getProperties();
 
     // Load cortex base pathway 
-    const basePathway = require(basePathwayPath);
+    const basePathway = await import(basePathwayPath).then(module => module.default);
 
     // Load core pathways, default from the Cortex package
     console.log('Loading core pathways from', corePathwaysPath)
-    let loadedPathways = require(corePathwaysPath);
+    let loadedPathways = await import(`${corePathwaysPath}/index.js`).then(module => module);
 
     // Load custom pathways and override core pathways if same
     if (pathwaysPath && fs.existsSync(pathwaysPath)) {
         console.log('Loading custom pathways from', pathwaysPath)
-        const customPathways = require(pathwaysPath);
+        const customPathways = await import(`${pathwaysPath}/index.js`).then(module => module);
         loadedPathways = { ...loadedPathways, ...customPathways };
     }
 
@@ -191,4 +197,4 @@ const buildModels = (config) => {
 // TODO: Perform validation
 // config.validate({ allowed: 'strict' });
 
-module.exports = { config, buildPathways, buildModels };
+export { config, buildPathways, buildModels };

package/graphql/chunker.js

@@ -1,21 +1,4 @@
-const { encode, decode } = require('gpt-3-encoder')
-
-const estimateCharPerToken = (text) => {
-    // check text only contains asciish characters
-    if (/^[ -~\t\n\r]+$/.test(text)) {
-        return 4;
-    }
-    return 1;
-}
-
-const getLastNChar = (text, maxLen) => {
-    if (text.length > maxLen) {
-        //slice text to avoid maxLen limit but keep the last n characters up to a \n or space to avoid cutting words
-        text = text.slice(-maxLen);
-        text = text.slice(text.search(/\s/) + 1);
-    }
-    return text;
-}
+import { encode, decode } from 'gpt-3-encoder';
 
 const getLastNToken = (text, maxTokenLen) => { 
     const encoded = encode(text);
@@ -35,113 +18,120 @@ const getFirstNToken = (text, maxTokenLen) => {
     return text;
 }
 
-const isBigChunk = ({ text, maxChunkLength, maxChunkToken }) => {
-    if (maxChunkLength && text.length > maxChunkLength) {
-        return true;
-    }
-    if (maxChunkToken && encode(text).length > maxChunkToken) {
-        return true;
-    }
-    return false;
-}
-
-const getSemanticChunks = ({ text, maxChunkLength, maxChunkToken,
-    enableParagraphChunks = true, enableSentenceChunks = true, enableLineChunks = true,
-    enableWordChunks = true, finallyMergeChunks = true }) => {
-
-    if (maxChunkLength && maxChunkLength <= 0) {
-        throw new Error(`Invalid maxChunkLength: ${maxChunkLength}`);
-    }
-    if (maxChunkToken && maxChunkToken <= 0) {
-        throw new Error(`Invalid maxChunkToken: ${maxChunkToken}`);
-    }
-
-    const isBig = (text) => {
-        return isBigChunk({ text, maxChunkLength, maxChunkToken });
-    }
-
-    // split into paragraphs
-    let paragraphChunks = enableParagraphChunks ? text.split('\n\n') : [text];
-
-    // Chunk paragraphs into sentences if needed
-    const sentenceChunks = enableSentenceChunks ? [] : paragraphChunks;
-    for (let i = 0; enableSentenceChunks && i < paragraphChunks.length; i++) {
-        if (isBig(paragraphChunks[i])) { // too long paragraph, chunk into sentences
-            sentenceChunks.push(...paragraphChunks[i].split('.\n')); // split into sentences
-        } else {
-            sentenceChunks.push(paragraphChunks[i]);
+const getSemanticChunks = (text, chunkSize) => {
+
+  const breakByRegex = (str, regex, preserveWhitespace = false) => {
+    const result = [];
+    let match;
+  
+    while ((match = regex.exec(str)) !== null) {
+      const value = str.slice(0, match.index);
+      result.push(value);
+  
+      if (preserveWhitespace || /\S/.test(match[0])) {
+        result.push(match[0]);
+      }
+  
+      str = str.slice(match.index + match[0].length);
+    }
+  
+    if (str) {
+      result.push(str);
+    }
+  
+    return result.filter(Boolean);
+  };
+
+  const breakByParagraphs = (str) => breakByRegex(str, /[\r\n]+/, true);
+  const breakBySentences = (str) => breakByRegex(str, /(?<=[.。؟！\?!\n])\s+/, true);
+  const breakByWords = (str) => breakByRegex(str, /(\s,;:.+)/);
+
+  const createChunks = (tokens) => {
+    let chunks = [];
+    let currentChunk = '';
+  
+    for (const token of tokens) {
+      const currentTokenLength = encode(currentChunk + token).length;
+      if (currentTokenLength <= chunkSize) {
+        currentChunk += token;
+      } else {
+        if (currentChunk) {
+          chunks.push(currentChunk);
         }
+        currentChunk = token;
+      }
     }
-
-    // Chunk sentences with newlines if needed
-    const newlineChunks = enableLineChunks ? [] : sentenceChunks;
-    for (let i = 0; enableLineChunks && i < sentenceChunks.length; i++) {
-        if (isBig(sentenceChunks[i])) { // too long, split into lines
-            newlineChunks.push(...sentenceChunks[i].split('\n'));
-        } else {
-            newlineChunks.push(sentenceChunks[i]);
-        }
+  
+    if (currentChunk) {
+      chunks.push(currentChunk);
     }
+  
+    return chunks;
+  };
 
-    // Chunk sentences into word chunks if needed
-    let chunks = enableWordChunks ? [] : newlineChunks;
-    for (let j = 0; enableWordChunks && j < newlineChunks.length; j++) {
-        if (isBig(newlineChunks[j])) { // too long sentence, chunk into words
-            const words = newlineChunks[j].split(' ');
-            // merge words into chunks up to max
-            let chunk = '';
-            for (let k = 0; k < words.length; k++) {
-                if (isBig( chunk + ' ' + words[k]) ) {
-                    chunks.push(chunk.trim());
-                    chunk = '';
-                }
-                chunk += words[k] + ' ';
-            }
-            if (chunk.length > 0) {
-                chunks.push(chunk.trim());
-            }
+  const combineChunks = (chunks) => {
+    let optimizedChunks = [];
+  
+    for (let i = 0; i < chunks.length; i++) {
+      if (i < chunks.length - 1) {
+        const combinedChunk = chunks[i] + chunks[i + 1];
+        const combinedLen = encode(combinedChunk).length;
+  
+        if (combinedLen <= chunkSize) {
+          optimizedChunks.push(combinedChunk);
+          i += 1;
         } else {
-            chunks.push(newlineChunks[j]);
+          optimizedChunks.push(chunks[i]);
         }
+      } else {
+        optimizedChunks.push(chunks[i]);
+      }
     }
+  
+    return optimizedChunks;
+  };
 
-    chunks = chunks.filter(Boolean).map(chunk => '\n' + chunk + '\n'); //filter empty chunks and add newlines
-
-    return finallyMergeChunks ? mergeChunks({ chunks, maxChunkLength, maxChunkToken }) : chunks;
-}
+  const breakText = (str) => {
+    const tokenLength = encode(str).length;
 
-const mergeChunks = ({ chunks, maxChunkLength, maxChunkToken }) => {
-    const isBig = (text) => {
-        return isBigChunk({ text, maxChunkLength, maxChunkToken });
+    if (tokenLength <= chunkSize) {
+      return [str];
     }
 
-    // Merge chunks into maxChunkLength chunks
-    let mergedChunks = [];
-    let chunk = '';
-    for (let i = 0; i < chunks.length; i++) {
-        if (isBig(chunk + ' ' + chunks[i])) {
-            mergedChunks.push(chunk);
-            chunk = '';
+    const breakers = [breakByParagraphs, breakBySentences, breakByWords];
+
+    for (let i = 0; i < breakers.length; i++) {
+      const tokens = breakers[i](str);
+      if (tokens.length > 1) {
+        let chunks = createChunks(tokens);
+        chunks = combineChunks(chunks);
+        const brokenChunks = chunks.flatMap(breakText);
+        if (brokenChunks.every(chunk => encode(chunk).length <= chunkSize)) {
+          return brokenChunks;
         }
-        chunk += chunks[i];
-    }
-    if (chunk.length > 0) {
-        mergedChunks.push(chunk);
+      }
     }
-    return mergedChunks;
+
+    return createChunks([...str]); // Split by characters
+  };
+
+  return breakText(text);
 }
 
 
 const semanticTruncate = (text, maxLength) => {
-    if (text.length > maxLength) {
-        text = getSemanticChunks({ text, maxChunkLength: maxLength })[0].slice(0, maxLength - 3).trim() + "...";
-    }
+  if (text.length <= maxLength) {
     return text;
-}
+  }
 
+  const truncatedText = text.slice(0, maxLength - 3).trim();
+  const lastSpaceIndex = truncatedText.lastIndexOf(" ");
 
+  return (lastSpaceIndex !== -1)
+    ? truncatedText.slice(0, lastSpaceIndex) + "..."
+    : truncatedText + "...";
+};
 
-module.exports = {
-    getSemanticChunks, semanticTruncate, mergeChunks,
-    getLastNChar, getLastNToken, getFirstNToken, estimateCharPerToken
-}
+export {
+    getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken
+};

package/graphql/graphql.js

@@ -1,23 +1,21 @@
-const { createServer } = require('http');
-const {
+import { createServer } from 'http';
+import {
     ApolloServerPluginDrainHttpServer,
     ApolloServerPluginLandingPageLocalDefault,
-} = require("apollo-server-core");
-const { makeExecutableSchema } = require('@graphql-tools/schema');
-const { WebSocketServer } = require('ws');
-const { useServer } = require('graphql-ws/lib/use/ws');
-const express = require('express');
-
-/// Create apollo graphql server
-const Keyv = require("keyv");
-const { KeyvAdapter } = require("@apollo/utils.keyvadapter");
-const responseCachePlugin = require('apollo-server-plugin-response-cache').default
-
-const subscriptions = require('./subscriptions');
-const { buildLimiters } = require('../lib/request');
-const { cancelRequestResolver } = require('./resolver');
-const { buildPathways, buildModels } = require('../config');
-const { requestState } = require('./requestState');
+} from 'apollo-server-core';
+import { makeExecutableSchema } from '@graphql-tools/schema';
+import { WebSocketServer } from 'ws';
+import { useServer } from 'graphql-ws/lib/use/ws';
+import express from 'express';
+import { ApolloServer } from 'apollo-server-express';
+import Keyv from 'keyv';
+import { KeyvAdapter } from '@apollo/utils.keyvadapter';
+import responseCachePlugin from 'apollo-server-plugin-response-cache';
+import subscriptions from './subscriptions.js';
+import { buildLimiters } from '../lib/request.js';
+import { cancelRequestResolver } from './resolver.js';
+import { buildPathways, buildModels } from '../config.js';
+import { requestState } from './requestState.js';
 
 const getPlugins = (config) => {
     // server plugins
@@ -134,9 +132,9 @@ const getResolvers = (config, pathways) => {
 }
 
 //graphql api build factory method
-const build = (config) => {
+const build = async (config) => {
     // First perform config build
-    buildPathways(config);
+    await buildPathways(config);
     buildModels(config);
 
     // build api limiters 
@@ -152,7 +150,6 @@ const build = (config) => {
 
     const { plugins, cache } = getPlugins(config);
 
-    const { ApolloServer, gql } = require('apollo-server-express');
     const app = express()
 
     const httpServer = createServer(app);
@@ -221,6 +218,6 @@ const build = (config) => {
 }
 
 
-module.exports = {
+export {
     build
 };

package/graphql/parser.js

@@ -31,7 +31,7 @@ const parseNumberedObjectList = (text, format) => {
     return result;
 }
 
-module.exports = {
+export {
     regexParser,
     parseNumberedList,
     parseNumberedObjectList,

package/graphql/pathwayPrompter.js

@@ -1,10 +1,9 @@
 // PathwayPrompter.js
-const OpenAIChatPlugin = require('./plugins/openAIChatPlugin');
-const OpenAICompletionPlugin = require('./plugins/openAICompletionPlugin');
-const AzureTranslatePlugin = require('./plugins/azureTranslatePlugin');
-const OpenAIWhisperPlugin = require('./plugins/openAiWhisperPlugin');
-const handlebars = require("handlebars");
-const { Exception } = require("handlebars");
+import OpenAIChatPlugin from './plugins/openAIChatPlugin.js';
+import OpenAICompletionPlugin from './plugins/openAICompletionPlugin.js';
+import AzureTranslatePlugin from './plugins/azureTranslatePlugin.js';
+import OpenAIWhisperPlugin from './plugins/openAiWhisperPlugin.js';
+import handlebars from 'handlebars';
 
 // register functions that can be called directly in the prompt markdown
 handlebars.registerHelper('stripHTML', function (value) {
@@ -27,7 +26,7 @@ class PathwayPrompter {
         const model = config.get('models')[modelName];
 
         if (!model) {
-            throw new Exception(`Model ${modelName} not found in config`);
+            throw new handlebars.Exception(`Model ${modelName} not found in config`);
         }
 
         let plugin;
@@ -46,7 +45,7 @@ class PathwayPrompter {
                 plugin = new OpenAIWhisperPlugin(config, pathway);
                 break;
             default:
-                throw new Exception(`Unsupported model type: ${model.type}`);
+                throw new handlebars.Exception(`Unsupported model type: ${model.type}`);
         }
 
         this.plugin = plugin;
@@ -57,6 +56,6 @@ class PathwayPrompter {
     }
 }
 
-module.exports = {
+export {
     PathwayPrompter
 };

package/graphql/pathwayResolver.js

@@ -1,14 +1,12 @@
-const { PathwayPrompter } = require('./pathwayPrompter');
-const {
-    v4: uuidv4,
-} = require('uuid');
-const pubsub = require('./pubsub');
-const { encode } = require('gpt-3-encoder')
-const { getFirstNToken, getLastNToken, getSemanticChunks } = require('./chunker');
-const { PathwayResponseParser } = require('./pathwayResponseParser');
-const { Prompt } = require('./prompt');
-const { getv, setv } = require('../lib/keyValueStorageClient');
-const { requestState } = require('./requestState');
+import { PathwayPrompter } from './pathwayPrompter.js';
+import { v4 as uuidv4 } from 'uuid';
+import pubsub from './pubsub.js';
+import { encode } from 'gpt-3-encoder';
+import { getFirstNToken, getLastNToken, getSemanticChunks } from './chunker.js';
+import { PathwayResponseParser } from './pathwayResponseParser.js';
+import { Prompt } from './prompt.js';
+import { getv, setv } from '../lib/keyValueStorageClient.js';
+import { requestState } from './requestState.js';
 
 const MAX_PREVIOUS_RESULT_TOKEN_LENGTH = 1000;
 
@@ -125,7 +123,7 @@ class PathwayResolver {
         // Get saved context from contextId or change contextId if needed
         const { contextId } = args;
         this.savedContextId = contextId ? contextId : null;
-        this.savedContext = contextId ? (getv && await getv(contextId) || {}) : {};
+        this.savedContext = contextId ? (getv && (await getv(contextId)) || {}) : {};
 
         // Save the context before processing the request
         const savedContextStr = JSON.stringify(this.savedContext);
@@ -163,7 +161,7 @@ class PathwayResolver {
         }
 
         // chunk the text and return the chunks with newline separators
-        return getSemanticChunks({ text, maxChunkToken: chunkTokenLength });
+        return getSemanticChunks(text, chunkTokenLength);
     }
 
     truncate(str, n) {
@@ -312,4 +310,4 @@ class PathwayResolver {
     }
 }
 
-module.exports = { PathwayResolver };
+export { PathwayResolver };

package/graphql/pathwayResponseParser.js

@@ -1,4 +1,4 @@
-const { parseNumberedList, parseNumberedObjectList } = require('./parser')
+import { parseNumberedList, parseNumberedObjectList } from './parser.js';
 
 class PathwayResponseParser {
     constructor(pathway) {
@@ -21,4 +21,4 @@ class PathwayResponseParser {
     }
 }
 
-module.exports = { PathwayResponseParser };
+export { PathwayResponseParser };

package/graphql/plugins/azureTranslatePlugin.js

@@ -1,5 +1,5 @@
 // AzureTranslatePlugin.js
-const ModelPlugin = require('./modelPlugin');
+import ModelPlugin from './modelPlugin.js';
 
 class AzureTranslatePlugin extends ModelPlugin {
     constructor(config, pathway) {
@@ -37,4 +37,4 @@ class AzureTranslatePlugin extends ModelPlugin {
     }
 }
 
-module.exports = AzureTranslatePlugin;
+export default AzureTranslatePlugin;

package/graphql/plugins/modelPlugin.js

@@ -1,7 +1,9 @@
 // ModelPlugin.js
-const handlebars = require('handlebars');
-const { request } = require("../../lib/request");
-const { encode } = require("gpt-3-encoder");
+import handlebars from 'handlebars';
+
+import { request } from '../../lib/request.js';
+import { encode } from 'gpt-3-encoder';
+import { getFirstNToken } from '../chunker.js';
 
 const DEFAULT_MAX_TOKENS = 4096;
 const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
@@ -38,37 +40,77 @@ class ModelPlugin {
         this.shouldCache = config.get('enableCache') && (pathway.enableCache || pathway.temperature == 0);
     }
 
-    // Function to remove non-system messages until token length is less than target
-    removeMessagesUntilTarget = (messages, targetTokenLength) => {
-        let chatML = this.messagesToChatML(messages);
-        let tokenLength = encode(chatML).length;
+    truncateMessagesToTargetLength = (messages, targetTokenLength) => {
+        // Calculate the token length of each message
+        const tokenLengths = messages.map((message) => ({
+            message,
+            tokenLength: encode(this.messagesToChatML([message], false)).length,
+        }));
     
-        while (tokenLength > targetTokenLength) {
-            for (let i = 0; i < messages.length; i++) {
-                if (messages[i].role !== 'system') {
-                    messages.splice(i, 1);
-                    chatML = this.messagesToChatML(messages);
-                    tokenLength = encode(chatML).length;
-                    break;
-                }
+        // Calculate the total token length of all messages
+        let totalTokenLength = tokenLengths.reduce(
+            (sum, { tokenLength }) => sum + tokenLength,
+            0
+        );
+    
+        // If we're already under the target token length, just bail
+        if (totalTokenLength <= targetTokenLength) return messages;
+    
+        // Remove and/or truncate messages until the target token length is reached
+        let index = 0;
+        while (totalTokenLength > targetTokenLength) {
+            const message = tokenLengths[index].message;
+
+            // Skip system messages
+            if (message.role === 'system') {
+                index++;
+                continue;
             }
-            if (messages.every(message => message.role === 'system')) {
-                break; // All remaining messages are 'system', stop removing messages
+
+            const currentTokenLength = tokenLengths[index].tokenLength;
+            
+            if (totalTokenLength - currentTokenLength >= targetTokenLength) {
+                // Remove the message entirely if doing so won't go below the target token length
+                totalTokenLength -= currentTokenLength;
+                tokenLengths.splice(index, 1);
+            } else {
+                // Truncate the message to fit the remaining target token length
+                const emptyContentLength = encode(this.messagesToChatML([{ ...message, content: '' }], false)).length;
+                const otherMessageTokens = totalTokenLength - currentTokenLength;
+                const tokensToKeep = targetTokenLength - (otherMessageTokens + emptyContentLength);
+
+                const truncatedContent = getFirstNToken(message.content, tokensToKeep);
+                const truncatedMessage = { ...message, content: truncatedContent };
+            
+                tokenLengths[index] = {
+                    message: truncatedMessage,
+                    tokenLength: encode(this.messagesToChatML([ truncatedMessage ], false)).length
+                }
+
+                // calculate the length again to keep us honest
+                totalTokenLength = tokenLengths.reduce(
+                    (sum, { tokenLength }) => sum + tokenLength,
+                    0
+                );
             }
         }
-        return messages;
-    }
-
+    
+        // Return the modified messages array
+        return tokenLengths.map(({ message }) => message);
+    };
+    
     //convert a messages array to a simple chatML format
-    messagesToChatML = (messages) => {
+    messagesToChatML(messages, addAssistant = true) {
         let output = "";
         if (messages && messages.length) {
             for (let message of messages) {
-                output += (message.role && message.content) ? `<|im_start|>${message.role}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
+                output += (message.role && (message.content || message.content === '')) ? `<|im_start|>${message.role}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
             }
             // you always want the assistant to respond next so add a
             // directive for that
-            output += "<|im_start|>assistant\n";
+            if (addAssistant) {
+                output += "<|im_start|>assistant\n";
+            }
         }
         return output;
     }
@@ -196,7 +238,7 @@ class ModelPlugin {
         const responseData = await request({ url, data, params, headers, cache: this.shouldCache }, this.modelName);
         
         if (responseData.error) {
-            throw new Exception(`An error was returned from the server: ${JSON.stringify(responseData.error)}`);
+            throw new Error(`An error was returned from the server: ${JSON.stringify(responseData.error)}`);
         }
     
         this.logRequestData(data, responseData, prompt);
@@ -205,6 +247,6 @@ class ModelPlugin {
 
 }
 
-module.exports = ModelPlugin;
+export default ModelPlugin;
 
   

package/graphql/plugins/openAiChatPlugin.js

@@ -1,5 +1,5 @@
 // OpenAIChatPlugin.js
-const ModelPlugin = require('./modelPlugin');
+import ModelPlugin from './modelPlugin.js';
 
 class OpenAIChatPlugin extends ModelPlugin {
     constructor(config, pathway) {
@@ -19,7 +19,7 @@ class OpenAIChatPlugin extends ModelPlugin {
         // Check if the token length exceeds the model's max token length
         if (tokenLength > modelMaxTokenLength) {
             // Remove older messages until the token length is within the model's limit
-            requestMessages = this.removeMessagesUntilTarget(requestMessages, modelMaxTokenLength);
+            requestMessages = this.truncateMessagesToTargetLength(requestMessages, modelMaxTokenLength);
         }
     
         const requestParameters = {
@@ -43,4 +43,4 @@ class OpenAIChatPlugin extends ModelPlugin {
     }
 }
 
-module.exports = OpenAIChatPlugin;
+export default OpenAIChatPlugin;