@aj-archipelago/cortex 0.0.9 → 0.0.10

package/graphql/chunker.js

@@ -1,22 +1,5 @@
 const { encode, decode } = require('gpt-3-encoder')
 
-const estimateCharPerToken = (text) => {
-    // check text only contains asciish characters
-    if (/^[ -~\t\n\r]+$/.test(text)) {
-        return 4;
-    }
-    return 1;
-}
-
-const getLastNChar = (text, maxLen) => {
-    if (text.length > maxLen) {
-        //slice text to avoid maxLen limit but keep the last n characters up to a \n or space to avoid cutting words
-        text = text.slice(-maxLen);
-        text = text.slice(text.search(/\s/) + 1);
-    }
-    return text;
-}
-
 const getLastNToken = (text, maxTokenLen) => { 
     const encoded = encode(text);
     if (encoded.length > maxTokenLen) {
@@ -35,113 +18,120 @@ const getFirstNToken = (text, maxTokenLen) => {
     return text;
 }
 
-const isBigChunk = ({ text, maxChunkLength, maxChunkToken }) => {
-    if (maxChunkLength && text.length > maxChunkLength) {
-        return true;
-    }
-    if (maxChunkToken && encode(text).length > maxChunkToken) {
-        return true;
-    }
-    return false;
-}
-
-const getSemanticChunks = ({ text, maxChunkLength, maxChunkToken,
-    enableParagraphChunks = true, enableSentenceChunks = true, enableLineChunks = true,
-    enableWordChunks = true, finallyMergeChunks = true }) => {
-
-    if (maxChunkLength && maxChunkLength <= 0) {
-        throw new Error(`Invalid maxChunkLength: ${maxChunkLength}`);
-    }
-    if (maxChunkToken && maxChunkToken <= 0) {
-        throw new Error(`Invalid maxChunkToken: ${maxChunkToken}`);
-    }
-
-    const isBig = (text) => {
-        return isBigChunk({ text, maxChunkLength, maxChunkToken });
-    }
-
-    // split into paragraphs
-    let paragraphChunks = enableParagraphChunks ? text.split('\n\n') : [text];
-
-    // Chunk paragraphs into sentences if needed
-    const sentenceChunks = enableSentenceChunks ? [] : paragraphChunks;
-    for (let i = 0; enableSentenceChunks && i < paragraphChunks.length; i++) {
-        if (isBig(paragraphChunks[i])) { // too long paragraph, chunk into sentences
-            sentenceChunks.push(...paragraphChunks[i].split('.\n')); // split into sentences
-        } else {
-            sentenceChunks.push(paragraphChunks[i]);
+const getSemanticChunks = (text, chunkSize) => {
+
+  const breakByRegex = (str, regex, preserveWhitespace = false) => {
+    const result = [];
+    let match;
+  
+    while ((match = regex.exec(str)) !== null) {
+      const value = str.slice(0, match.index);
+      result.push(value);
+  
+      if (preserveWhitespace || /\S/.test(match[0])) {
+        result.push(match[0]);
+      }
+  
+      str = str.slice(match.index + match[0].length);
+    }
+  
+    if (str) {
+      result.push(str);
+    }
+  
+    return result.filter(Boolean);
+  };
+
+  const breakByParagraphs = (str) => breakByRegex(str, /[\r\n]+/, true);
+  const breakBySentences = (str) => breakByRegex(str, /(?<=[.。؟！\?!\n])\s+/, true);
+  const breakByWords = (str) => breakByRegex(str, /(\s,;:.+)/);
+
+  const createChunks = (tokens) => {
+    let chunks = [];
+    let currentChunk = '';
+  
+    for (const token of tokens) {
+      const currentTokenLength = encode(currentChunk + token).length;
+      if (currentTokenLength <= chunkSize) {
+        currentChunk += token;
+      } else {
+        if (currentChunk) {
+          chunks.push(currentChunk);
         }
+        currentChunk = token;
+      }
     }
-
-    // Chunk sentences with newlines if needed
-    const newlineChunks = enableLineChunks ? [] : sentenceChunks;
-    for (let i = 0; enableLineChunks && i < sentenceChunks.length; i++) {
-        if (isBig(sentenceChunks[i])) { // too long, split into lines
-            newlineChunks.push(...sentenceChunks[i].split('\n'));
-        } else {
-            newlineChunks.push(sentenceChunks[i]);
-        }
+  
+    if (currentChunk) {
+      chunks.push(currentChunk);
     }
+  
+    return chunks;
+  };
 
-    // Chunk sentences into word chunks if needed
-    let chunks = enableWordChunks ? [] : newlineChunks;
-    for (let j = 0; enableWordChunks && j < newlineChunks.length; j++) {
-        if (isBig(newlineChunks[j])) { // too long sentence, chunk into words
-            const words = newlineChunks[j].split(' ');
-            // merge words into chunks up to max
-            let chunk = '';
-            for (let k = 0; k < words.length; k++) {
-                if (isBig( chunk + ' ' + words[k]) ) {
-                    chunks.push(chunk.trim());
-                    chunk = '';
-                }
-                chunk += words[k] + ' ';
-            }
-            if (chunk.length > 0) {
-                chunks.push(chunk.trim());
-            }
+  const combineChunks = (chunks) => {
+    let optimizedChunks = [];
+  
+    for (let i = 0; i < chunks.length; i++) {
+      if (i < chunks.length - 1) {
+        const combinedChunk = chunks[i] + chunks[i + 1];
+        const combinedLen = encode(combinedChunk).length;
+  
+        if (combinedLen <= chunkSize) {
+          optimizedChunks.push(combinedChunk);
+          i += 1;
         } else {
-            chunks.push(newlineChunks[j]);
+          optimizedChunks.push(chunks[i]);
         }
+      } else {
+        optimizedChunks.push(chunks[i]);
+      }
     }
+  
+    return optimizedChunks;
+  };
 
-    chunks = chunks.filter(Boolean).map(chunk => '\n' + chunk + '\n'); //filter empty chunks and add newlines
-
-    return finallyMergeChunks ? mergeChunks({ chunks, maxChunkLength, maxChunkToken }) : chunks;
-}
+  const breakText = (str) => {
+    const tokenLength = encode(str).length;
 
-const mergeChunks = ({ chunks, maxChunkLength, maxChunkToken }) => {
-    const isBig = (text) => {
-        return isBigChunk({ text, maxChunkLength, maxChunkToken });
+    if (tokenLength <= chunkSize) {
+      return [str];
     }
 
-    // Merge chunks into maxChunkLength chunks
-    let mergedChunks = [];
-    let chunk = '';
-    for (let i = 0; i < chunks.length; i++) {
-        if (isBig(chunk + ' ' + chunks[i])) {
-            mergedChunks.push(chunk);
-            chunk = '';
+    const breakers = [breakByParagraphs, breakBySentences, breakByWords];
+
+    for (let i = 0; i < breakers.length; i++) {
+      const tokens = breakers[i](str);
+      if (tokens.length > 1) {
+        let chunks = createChunks(tokens);
+        chunks = combineChunks(chunks);
+        const brokenChunks = chunks.flatMap(breakText);
+        if (brokenChunks.every(chunk => encode(chunk).length <= chunkSize)) {
+          return brokenChunks;
         }
-        chunk += chunks[i];
-    }
-    if (chunk.length > 0) {
-        mergedChunks.push(chunk);
+      }
     }
-    return mergedChunks;
+
+    return createChunks([...str]); // Split by characters
+  };
+
+  return breakText(text);
 }
 
 
 const semanticTruncate = (text, maxLength) => {
-    if (text.length > maxLength) {
-        text = getSemanticChunks({ text, maxChunkLength: maxLength })[0].slice(0, maxLength - 3).trim() + "...";
-    }
+  if (text.length <= maxLength) {
     return text;
-}
+  }
 
+  const truncatedText = text.slice(0, maxLength - 3).trim();
+  const lastSpaceIndex = truncatedText.lastIndexOf(" ");
 
+  return (lastSpaceIndex !== -1)
+    ? truncatedText.slice(0, lastSpaceIndex) + "..."
+    : truncatedText + "...";
+};
 
 module.exports = {
-    getSemanticChunks, semanticTruncate, mergeChunks,
-    getLastNChar, getLastNToken, getFirstNToken, estimateCharPerToken
+    getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken
 }

package/graphql/pathwayResolver.js

@@ -163,7 +163,7 @@ class PathwayResolver {
         }
 
         // chunk the text and return the chunks with newline separators
-        return getSemanticChunks({ text, maxChunkToken: chunkTokenLength });
+        return getSemanticChunks(text, chunkTokenLength);
     }
 
     truncate(str, n) {

package/graphql/plugins/modelPlugin.js

@@ -2,6 +2,7 @@
 const handlebars = require('handlebars');
 const { request } = require("../../lib/request");
 const { encode } = require("gpt-3-encoder");
+const { getFirstNToken } = require("../chunker");
 
 const DEFAULT_MAX_TOKENS = 4096;
 const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
@@ -38,37 +39,77 @@ class ModelPlugin {
         this.shouldCache = config.get('enableCache') && (pathway.enableCache || pathway.temperature == 0);
     }
 
-    // Function to remove non-system messages until token length is less than target
-    removeMessagesUntilTarget = (messages, targetTokenLength) => {
-        let chatML = this.messagesToChatML(messages);
-        let tokenLength = encode(chatML).length;
+    truncateMessagesToTargetLength = (messages, targetTokenLength) => {
+        // Calculate the token length of each message
+        const tokenLengths = messages.map((message) => ({
+            message,
+            tokenLength: encode(this.messagesToChatML([message], false)).length,
+        }));
     
-        while (tokenLength > targetTokenLength) {
-            for (let i = 0; i < messages.length; i++) {
-                if (messages[i].role !== 'system') {
-                    messages.splice(i, 1);
-                    chatML = this.messagesToChatML(messages);
-                    tokenLength = encode(chatML).length;
-                    break;
-                }
+        // Calculate the total token length of all messages
+        let totalTokenLength = tokenLengths.reduce(
+            (sum, { tokenLength }) => sum + tokenLength,
+            0
+        );
+    
+        // If we're already under the target token length, just bail
+        if (totalTokenLength <= targetTokenLength) return messages;
+    
+        // Remove and/or truncate messages until the target token length is reached
+        let index = 0;
+        while (totalTokenLength > targetTokenLength) {
+            const message = tokenLengths[index].message;
+
+            // Skip system messages
+            if (message.role === 'system') {
+                index++;
+                continue;
             }
-            if (messages.every(message => message.role === 'system')) {
-                break; // All remaining messages are 'system', stop removing messages
+
+            const currentTokenLength = tokenLengths[index].tokenLength;
+            
+            if (totalTokenLength - currentTokenLength >= targetTokenLength) {
+                // Remove the message entirely if doing so won't go below the target token length
+                totalTokenLength -= currentTokenLength;
+                tokenLengths.splice(index, 1);
+            } else {
+                // Truncate the message to fit the remaining target token length
+                const emptyContentLength = encode(this.messagesToChatML([{ ...message, content: '' }], false)).length;
+                const otherMessageTokens = totalTokenLength - currentTokenLength;
+                const tokensToKeep = targetTokenLength - (otherMessageTokens + emptyContentLength);
+
+                const truncatedContent = getFirstNToken(message.content, tokensToKeep);
+                const truncatedMessage = { ...message, content: truncatedContent };
+            
+                tokenLengths[index] = {
+                    message: truncatedMessage,
+                    tokenLength: encode(this.messagesToChatML([ truncatedMessage ], false)).length
+                }
+
+                // calculate the length again to keep us honest
+                totalTokenLength = tokenLengths.reduce(
+                    (sum, { tokenLength }) => sum + tokenLength,
+                    0
+                );
             }
         }
-        return messages;
-    }
-
+    
+        // Return the modified messages array
+        return tokenLengths.map(({ message }) => message);
+    };
+    
     //convert a messages array to a simple chatML format
-    messagesToChatML = (messages) => {
+    messagesToChatML(messages, addAssistant = true) {
         let output = "";
         if (messages && messages.length) {
             for (let message of messages) {
-                output += (message.role && message.content) ? `<|im_start|>${message.role}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
+                output += (message.role && (message.content || message.content === '')) ? `<|im_start|>${message.role}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
             }
             // you always want the assistant to respond next so add a
             // directive for that
-            output += "<|im_start|>assistant\n";
+            if (addAssistant) {
+                output += "<|im_start|>assistant\n";
+            }
         }
         return output;
     }

package/graphql/plugins/openAiChatPlugin.js

@@ -19,7 +19,7 @@ class OpenAIChatPlugin extends ModelPlugin {
         // Check if the token length exceeds the model's max token length
         if (tokenLength > modelMaxTokenLength) {
             // Remove older messages until the token length is within the model's limit
-            requestMessages = this.removeMessagesUntilTarget(requestMessages, modelMaxTokenLength);
+            requestMessages = this.truncateMessagesToTargetLength(requestMessages, modelMaxTokenLength);
         }
     
         const requestParameters = {

package/graphql/plugins/openAiCompletionPlugin.js

@@ -16,7 +16,7 @@ class OpenAICompletionPlugin extends ModelPlugin {
         let requestParameters = {};
     
         if (modelPromptMessages) {
-            const requestMessages = this.removeMessagesUntilTarget(modelPromptMessages, modelMaxTokenLength - 1);
+            const requestMessages = this.truncateMessagesToTargetLength(modelPromptMessages, modelMaxTokenLength - 1);
             modelPromptMessagesML = this.messagesToChatML(requestMessages);
             tokenLength = encode(modelPromptMessagesML).length;
         

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "0.0.9",
+  "version": "0.0.10",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",
@@ -21,7 +21,7 @@
   "main": "index.js",
   "scripts": {
     "start": "node start.js",
-    "test": "jest --setupFiles dotenv/config"
+    "test": "ava"
   },
   "author": "",
   "license": "MIT",
@@ -53,10 +53,18 @@
     "ytdl-core": "^4.11.2"
   },
   "devDependencies": {
-    "dotenv": "^16.0.3",
-    "jest": "^29.3.1"
+    "ava": "^5.2.0",
+    "dotenv": "^16.0.3"
   },
   "publishConfig": {
     "access": "private"
+  },
+  "ava": {
+    "files": [
+      "tests/**/*.test.js"
+    ],
+    "require": [
+      "dotenv/config"
+    ]
   }
 }

package/pathways/summary.js

@@ -8,7 +8,7 @@ const { PathwayResolver } = require('../graphql/pathwayResolver');
 
 module.exports = {
     // The main prompt function that takes the input text and asks to generate a summary.
-    prompt: `{{{text}}}\n\nWrite a summary of the above text:\n\n`,
+    prompt: `{{{text}}}\n\nWrite a summary of the above text. If the text is in a language other than english, make sure the summary is written in the same language:\n\n`,
 
     // Define input parameters for the prompt, such as the target length of the summary.
     inputParameters: {
@@ -26,7 +26,7 @@ module.exports = {
             return await pathwayResolver.resolve(args);
         }
 
-        const errorMargin = 0.2;
+        const errorMargin = 0.1;
         const lowTargetLength = originalTargetLength * (1 - errorMargin);
         const targetWords = Math.round(originalTargetLength / 6.6);
 
@@ -39,17 +39,32 @@ module.exports = {
         let summary = '';
         let pathwayResolver = new PathwayResolver({ config, pathway, args, requestState });
 
-        
         // Modify the prompt to be words-based instead of characters-based.
-        pathwayResolver.pathwayPrompt = `{{{text}}}\n\nWrite a summary of the above text in exactly ${targetWords} words:\n\n`
+        pathwayResolver.pathwayPrompt = `Write a summary of all of the text below. If the text is in a language other than english, make sure the summary is written in the same language. Your summary should be ${targetWords} words in length.\n\nText:\n\n{{{text}}}\n\nSummary:\n\n`
 
         let i = 0;
-        // Reprompt if summary is too long or too short.
-        while (((summary.length > originalTargetLength) || (summary.length < lowTargetLength)) && i < MAX_ITERATIONS) {
+        // Make sure it's long enough to start
+        while ((summary.length < lowTargetLength) && i < MAX_ITERATIONS) {
             summary = await pathwayResolver.resolve(args);
             i++;
         }
 
+        // If it's too long, it could be because the input text was chunked
+        // and now we have all the chunks together. We can summarize that
+        // to get a comprehensive summary.
+        if (summary.length > originalTargetLength) {
+            pathwayResolver.pathwayPrompt = `Write a summary of all of the text below. If the text is in a language other than english, make sure the summary is written in the same language. Your summary should be ${targetWords} words in length.\n\nText:\n\n${summary}\n\nSummary:\n\n`
+            summary = await pathwayResolver.resolve(args);
+            i++;
+
+            // Now make sure it's not too long
+            while ((summary.length > originalTargetLength) && i < MAX_ITERATIONS) {
+                pathwayResolver.pathwayPrompt = `${summary}\n\nIs that less than ${targetWords} words long? If not, try again using a length of no more than ${targetWords} words.\n\n`;
+                summary = await pathwayResolver.resolve(args);
+                i++;
+            }
+        }
+
         // If the summary is still too long, truncate it.
         if (summary.length > originalTargetLength) {
             return semanticTruncate(summary, originalTargetLength);

package/tests/chunkfunction.test.js

@@ -0,0 +1,126 @@
+const test = require('ava');
+
+const { getSemanticChunks } = require('../graphql/chunker');
+const { encode } = require('gpt-3-encoder')
+
+const testText = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. In id erat sem. Phasellus ac dapibus purus, in fermentum nunc. Mauris quis rutrum magna. Quisque rutrum, augue vel blandit posuere, augue magna convallis turpis, nec elementum augue mauris sit amet nunc. Aenean sit amet leo est. Nunc ante ex, blandit et felis ut, iaculis lacinia est. Phasellus dictum orci id libero ullamcorper tempor.
+
+Vivamus id pharetra odio.   Sed consectetur leo sed tortor dictum venenatis.Donec gravida libero non accumsan suscipit.Donec lectus turpis, ullamcorper eu pulvinar iaculis, ornare ut risus.Phasellus aliquam, turpis quis viverra condimentum, risus est pretium    metus, in porta ipsum tortor vitae elit.Pellentesque id finibus erat.  In suscipit, sapien non posuere dignissim, augue nisl ultrices tortor, sit amet eleifend nibh elit at risus.
+
+Donec diam ligula, sagittis ut nisl tincidunt, porta sodales magna.Vestibulum ut dui arcu.Fusce at dolor ex.Aliquam eu justo non libero volutpat pulvinar at id urna.Donec nec purus sed elit bibendum faucibus.Pellentesque habitant morbi                                     tristique senectus et netus et malesuada fames ac turpis egestas.Vivamus iaculis mattis velit, ut lacinia massa lacinia quis.Phasellus porttitor gravida ex, id aliquet eros rhoncus quis. Ut fringilla, lectus a vehicula luctus, diam odio convallis dolor, sodales pharetra nulla ex dictum justo.Ut faucibus, augue quis dictum iaculis, diam leo    maximus sapien, sit amet vulputate eros quam sed sem.Cras malesuada, sapien sit amet iaculis euismod, nunc odio lacinia est, dictum iaculis ante nisi in est.Fusce vehicula lorem tellus.Nullam a tempus nisi    .
+
+Sed ut lectus nec ligula blandit tempus.Donec faucibus turpis id urna vehicula imperdiet.Duis tempor vitae orci interdum dignissim.Phasellus sed efficitur sem.Nullam accumsan, turpis vitae consectetur ullamcorper, lectus purus tincidunt nisi, in pulvinar leo tortor at sem.Donec at feugiat dui, nec rhoncus nibh.Nam faucibus ultrices nisl at lobortis.Morbi congue, nisl vel fermentum tristique, dui ipsum rhoncus massa, non varius nibh massa in turpis.Vestibulum vulputate, felis quis lacinia porta, nulla ex volutpat lorem, non rhoncus neque erat quis arcu.Morbi massa nisl, hendrerit eget tortor condimentum, lobortis dapibus sem.Aliquam ut dapibus elit.Sed porta dignissim ante.Nullam interdum ligula et massa vehicula, vel gravida diam laoreet.Vivamus et enim eget turpis pellentesque laoreet.Vivamus pellentesque neque et mauris imperdiet pulvinar.
+
+Aliquam eget ligula congue, tincidunt magna eu, rutrum urna.Sed consequat orci est, vel laoreet magna tincidunt sit amet.Curabitur eget condimentum odio, vitae condimentum elit.Duis viverra lobortis magna.Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.Sed facilisis mi eu scelerisque pharetra.Cras et massa odio.Praesent quis nulla vitae mi blandit egestas ac vitae libero.Cras ultricies ex non consequat scelerisque.Nulla et est ac sem placerat convallis ac vitae massa.Phasellus lobortis mauris vel est vehicula lobortis.Curabitur ipsum ipsum, ullamcorper eget placerat sit amet, dapibus iaculis dui.Phasellus facilisis rutrum metus nec euismod.
+
+Nam viverra est ac orci rhoncus, mollis mattis mi lobortis.Maecenas lectus ex, pulvinar vel mauris vel, egestas ornare massa.Nam placerat, tellus vel ullamcorper ullamcorper, enim felis egestas tellus, eu dictum augue tortor vel libero.Integer vel nunc felis.Nulla vehicula et enim non luctus.Vestibulum non odio magna.Donec vitae ipsum et nisl vestibulum maximus eu at augue.Morbi ac tristique quam.Suspendisse vestibulum nec dui et consectetur.Aliquam a dapibus dolor, sit amet fringilla eros.Nam id lorem nulla.
+
+Proin vulputate risus purus, id tincidunt magna eleifend vel.Pellentesque et commodo leo, sit amet molestie nunc.Nunc purus lectus, interdum ut mauris ac, varius pretium magna.Etiam sollicitudin eros at pretium molestie.Cras fermentum sagittis elit at egestas.Fusce auctor lacinia nisl ac ullamcorper.Interdum et malesuada fames ac ante ipsum primis in faucibus.Fusce commodo pretium urna vel consequat.In finibus tellus vitae magna pharetra, porttitor egestas libero cursus.Donec eget tincidunt dolor, ac tristique diam.Etiam interdum dictum ex suscipit tempus.In hac habitasse platea dictumst.Nulla ornare libero a leo mollis, sed gravida leo finibus.Nunc ornare, dolor ac convallis varius, quam ipsum ultricies dui, non vehicula est eros eget ipsum.Mauris vel rhoncus ligula, non porta metus.
+
+Ut non felis pretium leo viverra tincidunt.Vivamus et ligula commodo dolor faucibus gravida.Quisque eu dolor ac metus pretium pharetra.Integer mattis efficitur libero, sed condimentum nulla ultricies eu.Donec turpis orci, fermentum vitae imperdiet nec, luctus quis lectus.Nunc viverra ornare libero.Vestibulum elementum tempus tortor id semper.
+
+Aliquam in dapibus risus.Praesent vitae condimentum elit, sodales pellentesque diam.Curabitur luctus pellentesque nunc, ut eleifend urna dictum ac.Aenean rhoncus lacinia quam a suscipit.Proin purus metus, egestas a pretium eu, tempus ut ante.Sed tellus turpis, hendrerit consequat porta id, porttitor non dolor.Proin volutpat massa a dui dictum facilisis a vel eros.Fusce eu efficitur odio.Aliquam interdum metus id ex dapibus dapibus.Nullam porttitor non sapien nec rhoncus.Cras iaculis fringilla cursus.Praesent at leo orci.Sed eget vulputate eros, eget auctor sapien.Nulla auctor, lectus ut tincidunt rhoncus, ante lorem volutpat eros, ac tincidunt enim ipsum at ex.Fusce dolor arcu, pretium eget elementum vel, semper at ipsum.
+
+Integer rhoncus fringilla felis ac tincidunt.Phasellus eu ultricies tellus.Sed pharetra, eros sed dignissim mattis, mi lectus blandit elit, vitae euismod ipsum sapien a eros.Aliquam lobortis tellus venenatis, sagittis lorem non, eleifend odio.Duis ultrices urna vel commodo varius.Sed ultricies mauris ut velit dignissim, eu lobortis ex tempor.Praesent vitae iaculis nisl.Vestibulum id convallis tellus.Vivamus eu consectetur erat.Curabitur interdum est non nibh malesuada ullamcorper.Phasellus mollis arcu a pharetra lacinia.Praesent sit amet sem non dui iaculis tincidunt.Aliquam vitae libero viverra metus feugiat volutpat ut eget sem.Nam facilisis pulvinar urna, ut venenatis ligula accumsan finibus.Maecenas nec aliquam nulla.Maecenas mattis magna erat.
+
+Nunc a nulla sed ante sollicitudin ultrices a a ante.Sed feugiat scelerisque leo, eget venenatis orci cursus eget.Ut pretium leo et nunc sodales, in luctus erat faucibus.Interdum et malesuada fames ac ante ipsum primis in faucibus.Mauris facilisis lorem quis turpis commodo, id vulputate metus molestie.Fusce id neque vestibulum, pretium elit in, ultrices justo.Praesent turpis dui, ullamcorper in vulputate ut, posuere at sapien.Etiam laoreet ultrices felis, id venenatis purus.Sed nec mauris pharetra, rhoncus sem non, interdum justo.Nulla sed tincidunt nisi.Suspendisse luctus viverra volutpat.Duis arcu nulla, euismod eu scelerisque in, vulputate eget quam.
+
+Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Fusce at dignissim quam.Suspendisse eget metus nec sem accumsan sagittis.Suspendisse non mollis lacus.Donec ac consectetur ante.Nullam luctus, nibh ac imperdiet porta, sem felis bibendum nibh, ut sollicitudin libero nulla a sapien.Sed tristique odio enim, eget tempor enim cursus vel.Morbi tristique lobortis tortor, nec placerat lorem suscipit ac.Nullam sed sodales diam, sed tincidunt est.Quisque semper velit sed risus dictum pretium.Proin condimentum, nisi a vulputate tristique, tellus erat scelerisque nisi, tincidunt viverra est neque non magna.Quisque nibh augue, interdum non justo et, varius rutrum erat.Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.
+
+Vestibulum et lorem auctor, vestibulum nisl id, elementum metus.Pellentesque quis mi a augue consectetur cursus.Morbi sodales risus et faucibus dictum.Ut in lobortis nisl, et euismod nisl.Donec ornare tellus placerat, blandit justo quis, pharetra nisl.Nulla scelerisque magna at nisi suscipit commodo.Fusce pellentesque in elit et consequat.Phasellus vehicula accumsan enim, vitae pellentesque nulla.Nullam id arcu vitae nunc consectetur mattis.Fusce ac sapien vel mi congue fringilla.Nulla mattis consectetur fringilla.Morbi orci elit, tempor in rhoncus eget, fringilla eget erat.
+
+Phasellus nec lorem lectus.Donec in cursus elit.In dictum elementum odio a scelerisque.Phasellus ac sapien eget velit accumsan elementum.Mauris odio eros, vulputate eu aliquet a, bibendum in quam.Integer euismod libero ac massa imperdiet, ullamcorper cursus risus auctor.Nam rutrum erat eget tortor suscipit semper sit amet nec mauris.Pellentesque nec semper neque.Nunc fringilla nisl erat, a sollicitudin tortor accumsan finibus.
+
+Integer vulputate ex dui, vitae scelerisque purus viverra vel.Cras ultricies purus in nibh dapibus, non hendrerit nulla aliquam.Fusce vitae gravida urna.Mauris eleifend rutrum ex, at fermentum enim fringilla quis.Suspendisse dignissim est eget tempus condimentum.Fusce scelerisque, felis et malesuada dictum, mauris dolor cursus ex, eget pulvinar sem nulla id diam.Ut volutpat tincidunt efficitur.Nunc vel risus fringilla, lacinia urna vitae, aliquet nulla.Nunc sed pulvinar dolor, eu fermentum velit.Curabitur a pretium quam, ut consectetur neque.Nunc ultricies, ex sed mattis efficitur, nulla nunc convallis odio, sit amet pellentesque orci tortor ut sapien.Vivamus felis orci, ultricies eget lacinia at, blandit vitae quam.In lacinia dui nec tincidunt maximus.Donec feugiat consectetur bibendum.Aenean eget vestibulum lacus.
+
+Suspendisse vel molestie magna, et viverra justo.Aenean nec mi felis.Nam lacinia purus et congue facilisis.Pellentesque eget odio sed sem tincidunt imperdiet.Proin finibus ex nec placerat aliquet.Phasellus quis sapien nunc.Mauris eu augue aliquam sem suscipit vehicula a luctus augue.Phasellus ac scelerisque nibh.Nullam eleifend eleifend sapien eget convallis.
+
+Nunc vitae metus risus.Ut iaculis dolor accumsan bibendum posuere.Morbi vitae odio sed velit dictum consequat.Aliquam vel erat vitae lacus luctus cursus vel ut risus.Aliquam a nunc eu lorem consequat finibus.Sed non enim vestibulum, ornare dui id, dignissim turpis.Etiam fermentum rutrum porttitor.Maecenas id nisl sodales, ornare turpis placerat, tincidunt dui.
+
+Nulla aliquam purus at leo fringilla euismod.Praesent condimentum augue nibh, sed scelerisque mauris bibendum vitae.Vivamus maximus enim non massa commodo gravida.Cras iaculis elit ac est dapibus convallis.Quisque in tortor tincidunt, placerat turpis pulvinar, rhoncus orci.In vel risus et lacus lacinia volutpat.Maecenas facilisis fermentum dictum.Lorem ipsum dolor sit amet, consectetur adipiscing elit.Praesent aliquam pretium pellentesque.In eleifend leo eros, in lobortis eros elementum maximus.Fusce in orci ut massa vehicula mollis vitae non nibh.Sed ac porttitor urna.Nulla ac venenatis sapien, eget vulputate metus.
+
+Mauris hendrerit lacus quam, vel mollis ligula porttitor ac.Nulla ornare libero at faucibus dictum.Donec tincidunt viverra sapien nec tincidunt.Donec leo sapien, rutrum quis dui a, auctor sodales nisi.Fusce condimentum eros sit amet ligula viverra, eget ullamcorper erat dapibus.Suspendisse dignissim ligula sed luctus aliquet.Aenean consectetur enim non nibh semper volutpat.
+
+Mauris diam dolor, maximus et ultrices sed, semper sed felis.Morbi ac eros tellus.Maecenas eget ex vitae quam lacinia eleifend non nec leo.Donec condimentum consectetur nunc, quis luctus elit commodo eu.Nunc tincidunt condimentum neque, sed porta ligula porttitor et.Suspendisse scelerisque id massa sit amet placerat.Sed eleifend aliquet facilisis.Donec ac purus nec metus vestibulum euismod.Maecenas sollicitudin consequat ornare.Suspendisse pharetra vehicula eros nec malesuada.`;
+
+test('should return identical text that chunker was passed, given large chunk size (1812)', async t => {
+    const maxChunkToken = 1812;
+    const chunks = getSemanticChunks(testText, maxChunkToken);
+    t.true(chunks.length > 0); //check chunking
+    t.true(chunks.every(chunk => encode(chunk).length <= maxChunkToken)); //check chunk size
+    const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
+    t.is(recomposedText, testText); //check recomposition
+});
+
+test('should return identical text that chunker was passed, given medium chunk size (500)', async t => {
+    const maxChunkToken = 500;
+    const chunks = getSemanticChunks(testText, maxChunkToken);
+    t.true(chunks.length > 1); //check chunking
+    t.true(chunks.every(chunk => encode(chunk).length <= maxChunkToken)); //check chunk size
+    const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
+    t.is(recomposedText, testText); //check recomposition
+});
+
+test('should return identical text that chunker was passed, given tiny chunk size (1)', async t => {
+    const maxChunkToken = 1;
+    const chunks = getSemanticChunks(testText, maxChunkToken);
+    t.true(chunks.length > 1); //check chunking
+    t.true(chunks.every(chunk => encode(chunk).length <= maxChunkToken)); //check chunk size
+    const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
+    t.is(recomposedText, testText); //check recomposition
+});
+
+/*
+it('should return identical text that chunker was passed, given tiny chunk size (1)', () => {
+    const maxChunkToken = 1;
+    const chunks = getSemanticChunks(testText, maxChunkToken);
+    expect(chunks.length).toBeGreaterThan(1); //check chunking
+    expect(chunks.every(chunk => encode(chunk).length <= maxChunkToken)).toBe(true); //check chunk size
+    const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
+    expect(recomposedText).toBe(testText); //check recomposition
+});
+
+it('should return identical text that chunker was passed, given huge chunk size (32000)', () => {
+    const maxChunkToken = 32000;
+    const chunks = getSemanticChunks(testText, maxChunkToken);
+    expect(chunks.length).toBe(1); //check chunking
+    expect(chunks.every(chunk => encode(chunk).length <= maxChunkToken)).toBe(true); //check chunk size
+    const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
+    expect(recomposedText).toBe(testText); //check recomposition
+});
+
+const testTextNoSpaces = `Loremipsumdolorsitamet,consecteturadipiscingelit.Inideratsem.Phasellusacdapibuspurus,infermentumnunc.Maurisquisrutrummagna.Quisquerutrum,auguevelblanditposuere,auguemagnacon vallisturpis,necelementumauguemaurissitametnunc.Aeneansitametleoest.Nuncanteex,blanditetfelisut,iaculislaciniaest.Phasellusdictumorciidliberoullamcorpertempor.Vivamusidpharetraodioq.Sedconsecteturleosedtortordictumvenenatis.Donecgravidaliberononaccumsansuscipit.Doneclectusturpis,ullamcorpereupulvinariaculis,ornareutrisus.Phasellusaliquam,turpisquisviverracondimentum,risusestpretiummetus,inportaips umtortorvita elit.Pellentesqueidfinibuserat.Insuscipit,sapiennonposueredignissim,auguenisl ultricestortor,sitameteleifendnibhelitatrisus.`;
+
+it('should return identical text that chunker was passed, given no spaces and small chunks(5)', () => {
+    const maxChunkToken = 5;
+    const chunks = getSemanticChunks(testTextNoSpaces, maxChunkToken);
+    expect(chunks.length).toBeGreaterThan(0); //check chunking
+    expect(chunks.every(chunk => encode(chunk).length <= maxChunkToken)).toBe(true); //check chunk size
+    const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
+    expect(recomposedText).toBe(testTextNoSpaces); //check recomposition
+});
+
+const testTextShortWeirdSpaces=`Lorem ipsum dolor sit amet, consectetur adipiscing elit. In id erat sem. Phasellus ac dapibus purus, in fermentum nunc.............................. Mauris quis rutrum magna. Quisque rutrum, augue vel blandit posuere, augue magna convallis turpis, nec elementum augue mauris sit amet nunc. Aenean sit a;lksjdf 098098- -23 eln ;lkn l;kn09 oij[0u ,,,,,,,,,,,,,,,,,,,,, amet leo est. Nunc ante ex, blandit et felis ut, iaculis lacinia est. Phasellus dictum orci id libero ullamcorper tempor.
+
+
+
+
+    Vivamus id pharetra odio.   Sed consectetur leo sed tortor dictum venenatis.Donec gravida libero non accumsan suscipit.Donec lectus turpis, ullamcorper eu pulvinar iaculis, ornare ut risus.Phasellus aliquam, turpis quis viverra condimentum, risus est pretium    metus, in porta ipsum tortor vitae elit.Pellentesque id finibus erat.  In suscipit, sapien non posuere dignissim, augue nisl ultrices tortor, sit amet eleifend nibh elit at risus.`;
+
+it('should return identical text that chunker was passed, given weird spaces and tiny chunks(1)', () => {
+    const maxChunkToken = 1;
+    const chunks = getSemanticChunks(testTextShortWeirdSpaces, maxChunkToken);
+    expect(chunks.length).toBeGreaterThan(0); //check chunking
+    expect(chunks.every(chunk => encode(chunk).length <= maxChunkToken)).toBe(true); //check chunk size
+    const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
+    expect(recomposedText).toBe(testTextShortWeirdSpaces); //check recomposition
+});
+
+it('should return identical text that chunker was passed, given weird spaces and small chunks(10)', () => {
+    const maxChunkToken = 1;
+    const chunks = getSemanticChunks(testTextShortWeirdSpaces, maxChunkToken);
+    expect(chunks.length).toBeGreaterThan(0); //check chunking
+    expect(chunks.every(chunk => encode(chunk).length <= maxChunkToken)).toBe(true); //check chunk size
+    const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
+    expect(recomposedText).toBe(testTextShortWeirdSpaces); //check recomposition
+});*/

package/tests/chunking.test.js

@@ -1,15 +1,14 @@
+const test = require('ava');
 const { getTestServer } = require('./main.test');
 
-jest.setTimeout(1800000);
-
 const testServer = getTestServer();
 
-//stop server after all tests
-afterAll(async () => {
+test.after.always(async () => {
     await testServer.stop();
 });
 
-it('chunking test of translate endpoint with huge text', async () => {
+test('chunking test of translate endpoint with huge text', async t => {
+    t.timeout(180000);
     const response = await testServer.executeOperation({
         query: 'query translate($text: String!) { translate(text: $text) { result } }',
         variables: {
@@ -54,12 +53,12 @@ Mauris hendrerit lacus quam, vel mollis ligula porttitor ac.Nulla ornare libero
 Mauris diam dolor, maximus et ultrices sed, semper sed felis.Morbi ac eros tellus.Maecenas eget ex vitae quam lacinia eleifend non nec leo.Donec condimentum consectetur nunc, quis luctus elit commodo eu.Nunc tincidunt condimentum neque, sed porta ligula porttitor et.Suspendisse scelerisque id massa sit amet placerat.Sed eleifend aliquet facilisis.Donec ac purus nec metus vestibulum euismod.Maecenas sollicitudin consequat ornare.Suspendisse pharetra vehicula eros nec malesuada.` },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.translate.result.length).toBeGreaterThan(1000); //check return length huge
+    t.is(response.errors, undefined);
+    t.true(response.data?.translate.result.length > 1000);
 });
 
-
-it('chunking test of translate endpoint with single long text sentence', async () => {
+test('chunking test of translate endpoint with single long text sentence', async t => {
+    t.timeout(180000);
     const response = await testServer.executeOperation({
         query: 'query translate($text: String!) { translate(text: $text) { result } }',
         variables: {
@@ -67,11 +66,12 @@ it('chunking test of translate endpoint with single long text sentence', async (
         }
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.translate.result.length).toBeGreaterThan(200); //check return length huge
+    t.is(response.errors, undefined);
+    t.true(response.data?.translate.result.length > 200);
 });
 
-it('chunking test of translate endpoint with two long text sentence', async () => {
+test('chunking test of translate endpoint with two long text sentence', async t => {
+    t.timeout(180000);
     const response = await testServer.executeOperation({
         query: 'query translate($text: String!) { translate(text: $text) { result } }',
         variables: {
@@ -79,11 +79,12 @@ it('chunking test of translate endpoint with two long text sentence', async () =
         }
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.translate.result.length).toBeGreaterThan(500); //check return length huge
+    t.is(response.errors, undefined);
+    t.true(response.data?.translate.result.length > 500);
 });
 
-it('chunking test...', async () => {
+test('chunking test...', async t => {
+    t.timeout(180000);
     const response = await testServer.executeOperation({
         query: 'query translate($text: String!) { translate(text: $text) { result } }',
         variables: {
@@ -144,6 +145,6 @@ it('chunking test...', async () => {
         }
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.translate.result.length).toBeGreaterThan(500); //check return length huge
-});
+    t.is(response.errors, undefined);
+    t.true(response.data?.translate.result.length > 500);
+});

package/tests/main.test.js

@@ -1,9 +1,9 @@
+const test = require('ava');
+
 const { ApolloServer } = require('apollo-server');
 const { config } = require('../config');
 const { typeDefs, resolvers } = require('../index')();
 
-jest.setTimeout(60000);
-
 const getTestServer = () => {
     return new ApolloServer({
         typeDefs,
@@ -15,82 +15,82 @@ const getTestServer = () => {
 const testServer = getTestServer();
 
 //stop server after all tests
-afterAll(async () => {
+test.after.always('cleanup', async () => {
     await testServer.stop();
 });
 
-it('validates bias endpoint', async () => {
+test('validates bias endpoint', async (t) => {
     const response = await testServer.executeOperation({
         query: 'query bias($text: String!) { bias(text: $text) { result } }',
         variables: { text: 'hello there my dear world!' },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.bias?.result).toMatch(/(yes|no|bias)/i)
+    t.is(response.errors, undefined);
+    t.regex(response.data?.bias?.result, /(yes|no|bias)/i);
 });
 
-it('validates completion endpoint', async () => {
+test('validates completion endpoint', async (t) => {
     const response = await testServer.executeOperation({
         query: 'query complete($text: String!) { complete(text: $text) { result } }',
         variables: { text: 'hello there my dear world!' },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.complete?.result.length).toBeGreaterThan(0);
+    t.is(response.errors, undefined);
+    t.true(response.data?.complete?.result.length > 0);
 });
 
-it('validates entities endpoint with given num of count return', async () => {
+test('validates entities endpoint with given num of count return', async (t) => {
     const response = await testServer.executeOperation({
         query: 'query entities($text: String!, $count: Int) { entities(text: $text, count: $count){ result { name, definition } } }',
         variables: { text: 'hello there my dear world!', count: 3 },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.entities.result.length).toBe(3);
+    t.is(response.errors, undefined);
+    t.is(response.data?.entities.result.length, 3);
     response.data?.result?.entities.forEach((entity) => {
-        expect(entity.name).toBeDefined();
-        expect(entity.definition).toBeDefined();
+        t.truthy(entity.name);
+        t.truthy(entity.definition);
     });
 });
 
-it('validates paraphrase endpoint', async () => {
+test('validates paraphrase endpoint', async (t) => {
     const response = await testServer.executeOperation({
         query: 'query paraphrase($text: String!) { paraphrase(text: $text) { result } }',
         variables: { text: 'hello there my dear world!' },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.paraphrase?.result).toBeDefined();
+    t.is(response.errors, undefined);
+    t.truthy(response.data?.paraphrase?.result);
 });
 
-it('validates sentiment endpoint', async () => {
+test('validates sentiment endpoint', async (t) => {
     const response = await testServer.executeOperation({
         query: 'query sentiment($text: String!) { sentiment(text: $text) { result } }',
         variables: { text: 'hello there my dear world!' },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.sentiment.result).toBeDefined();
+    t.is(response.errors, undefined);
+    t.truthy(response.data?.sentiment.result);
 });
 
-it('validates edit endpoint', async () => {
+test('validates edit endpoint', async (t) => {
     const response = await testServer.executeOperation({
         query: 'query edit($text: String!) { edit(text: $text) { result } }',
         variables: { text: 'helo there my dear worldd!' },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.edit.result).toMatch(/hello.*world/i);
+    t.is(response.errors, undefined);
+    t.regex(response.data?.edit.result, /hello.*world/i);
 });
 
-it('validates summary endpoint', async () => {
+test('validates summary endpoint', async (t) => {
     const response = await testServer.executeOperation({
         query: 'query summary($text: String!) { summary(text: $text) { result } }',
         variables: { text: 'hello there my dear world!' },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.summary.result).toBeDefined();
+    t.is(response.errors, undefined);
+    t.truthy(response.data?.summary.result);
 });
 
 module.exports = {

package/tests/translate.test.js

@@ -1,15 +1,14 @@
+const test = require('ava');
 const { getTestServer } = require('./main.test');
 
-jest.setTimeout(1800000);
-
 const testServer = getTestServer();
 
-//stop server after all tests
-afterAll(async () => {
+// stop server after all tests
+test.after.always(async () => {
     await testServer.stop();
 });
 
-it('test translate endpoint with huge arabic text english translation and check return non-arabic/english', async () => {
+test('test translate endpoint with huge arabic text english translation and check return non-arabic/english', async t => {
     const response = await testServer.executeOperation({
         query: 'query translate($text: String!, $to:String) { translate(text: $text, to:$to) { result } }',
         variables: {
@@ -115,9 +114,9 @@ it('test translate endpoint with huge arabic text english translation and check
 ` },
     });
 
-    expect(response.errors).toBeUndefined();
-    expect(response.data?.translate.result.length).toBeGreaterThan(1000); //check return length huge
-    //check return only contains non-Arabic characters
-    expect(response.data?.translate.result).not.toMatch(/[ء-ي]/);
+    t.falsy(response.errors);
+    t.true(response.data?.translate.result.length > 1000); // check return length huge
+    // check return only contains non-Arabic characters
+    t.notRegex(response.data?.translate.result, /[ء-ي]/);
 });