@aj-archipelago/cortex 1.3.32 → 1.3.34

package/tests/encodeCache.test.js

@@ -2,7 +2,21 @@ import test from 'ava';
 import { faker } from '@faker-js/faker';
 import { performance } from 'perf_hooks';
 import { encode, decode } from '../lib/encodeCache.js';
-import { encode as gpt3Encode, decode as gpt3Decode } from 'gpt-3-encoder';
+import { encoding_for_model } from '@dqbd/tiktoken';
+
+// Create reference encoder with same model as used in encodeCache
+const encoder = encoding_for_model("gpt-4o");
+
+// Helper to create a stable representation of token arrays for comparisons
+const tokenArrayToString = arr => Array.from(arr).toString();
+
+// Helper to normalize decoded content to string
+const normalizeDecoded = decoded => {
+    if (decoded instanceof Uint8Array) {
+        return new TextDecoder().decode(decoded);
+    }
+    return decoded;
+};
 
 // Test the accuracy of the cached encoding and decoding
 test('cached encode and decode are reversible', t => {
@@ -12,23 +26,29 @@ test('cached encode and decode are reversible', t => {
     t.is(decoded, original);
 })
 
-// Test whether the cached encoding and decoding is identical to the gpt3-encoder
+// Test whether the cached encoding and decoding is identical to tiktoken
 test('cached encode and decode are identical to noncached', t => {
     const original = faker.lorem.paragraph(50);
     const encoded = encode(original);
-    const gpt3Encoded = gpt3Encode(original);
-    t.deepEqual(encoded, gpt3Encoded);
+    const tiktokenEncoded = encoder.encode(original);
+    
+    // Compare arrays by converting to strings
+    t.is(tokenArrayToString(encoded), tokenArrayToString(tiktokenEncoded));
 
     const decoded = decode(encoded);
-    const gpt3Decoded = gpt3Decode(encoded);
-    t.is(decoded, gpt3Decoded);
+    const tiktokenDecoded = encoder.decode(tiktokenEncoded);
+    
+    // Normalize tiktoken decoded output to string for comparison
+    const normalizedTiktokenDecoded = normalizeDecoded(tiktokenDecoded);
+    
+    t.is(decoded, normalizedTiktokenDecoded);
 })
 
 // Test whether decoding adds the encoded value to the encode cache
 // the only way to tell is if the encode is faster after the cached decode
 test('decode operation adds to encode cache', t => {
     const original = faker.lorem.paragraph(50);
-    const encodedOriginal = gpt3Encode(original);
+    const encodedOriginal = encoder.encode(original);
 
     const startEncode = performance.now();
     const encoded = encode(original);
@@ -36,39 +56,51 @@ test('decode operation adds to encode cache', t => {
     const encodeTime = endEncode - startEncode;
     console.log("pre-decode encode time", encodeTime);
 
-    t.deepEqual(encoded, encodedOriginal);
+    // Compare arrays using our helper
+    t.is(tokenArrayToString(encoded), tokenArrayToString(encodedOriginal));
 
     const original2 = faker.lorem.paragraph(50);
-    const encodedOriginal2 = gpt3Encode(original2);
+    const encodedOriginal2 = encoder.encode(original2);
+    
+    // Decode should add to cache
     const decodedOriginal2 = decode(encodedOriginal2);
+    
     const startEncode2 = performance.now();
     const encoded2 = encode(original2);
     const endEncode2 = performance.now();
     const encodeTime2 = endEncode2 - startEncode2;
     console.log("post-decode encode time", encodeTime2);
 
-    t.deepEqual(encoded2, encodedOriginal2);
+    t.is(tokenArrayToString(encoded2), tokenArrayToString(encodedOriginal2));
+    
+    // Allow some buffer for timing variations
     t.true(encodeTime2 <= encodeTime);
 })
 
-
 // Test encode and decode caching
 test('caching', t => {
     const original = faker.lorem.paragraph(50);
+    
+    // First encode should be uncached
     const startEncode1 = performance.now();
     const encoded1 = encode(original);
     const endEncode1 = performance.now();
     const encodeTime1 = endEncode1 - startEncode1;
 
     const original2 = faker.lorem.paragraph(50);
-    const encodedOriginal2 = gpt3Encode(original2);
+    const encodedOriginal2 = encoder.encode(original2);
+    
+    // First decode should be uncached
     const startDecode1 = performance.now();
     const decoded1 = decode(encodedOriginal2);
     const endDecode1 = performance.now();
     const decodeTime1 = endDecode1 - startDecode1;
 
-    t.deepEqual(encoded1, gpt3Encode(original));
-    t.is(decoded1, original2);
+    t.is(tokenArrayToString(encoded1), tokenArrayToString(encoder.encode(original)));
+    
+    // Compare with normalized tiktoken output
+    const normalizedOriginal2 = normalizeDecoded(encoder.decode(encodedOriginal2));
+    t.is(decoded1, normalizedOriginal2);
 
     console.log('uncached encode time', encodeTime1);
     console.log('uncached decode time', decodeTime1);
@@ -87,6 +119,7 @@ test('caching', t => {
     console.log('cached encode time', encodeTime2);
     console.log('cached decode time', decodeTime2);
 
+    // Allow some buffer for timing variations
     t.true(encodeTime2 <= encodeTime1);
     t.true(decodeTime2 <= decodeTime1);
 });

package/tests/modelPlugin.test.js

@@ -71,6 +71,27 @@ test('getPromptTokenRatio', (t) => {
     t.is(modelPlugin.getPromptTokenRatio(), DEFAULT_PROMPT_TOKEN_RATIO, 'getPromptTokenRatio should return default prompt token ratio');
 });
 
+test('getModelMaxPromptTokens', (t) => {
+    const { modelPlugin } = t.context;
+    
+    // Default case - should use token ratio
+    t.is(
+        modelPlugin.getModelMaxPromptTokens(), 
+        Math.floor(DEFAULT_MAX_TOKENS * DEFAULT_PROMPT_TOKEN_RATIO), 
+        'Should return maxTokenLength * tokenRatio when maxReturnTokens is not defined'
+    );
+    
+    // When maxReturnTokens is defined
+    const returnTokens = 256;
+    modelPlugin.promptParameters.maxReturnTokens = returnTokens;
+    
+    t.is(
+        modelPlugin.getModelMaxPromptTokens(), 
+        DEFAULT_MAX_TOKENS - returnTokens, 
+        'Should return maxTokenLength - maxReturnTokens when maxReturnTokens is defined'
+    );
+});
+
 test('default parseResponse', (t) => {
     const { modelPlugin } = t.context;
     const multipleChoicesResponse = {

package/tests/multimodal_conversion.test.js

@@ -200,7 +200,7 @@ test('Unsupported mime type conversion', async (t) => {
     const pdfMessage = [
         { role: 'user', content: [
             { type: 'text', text: 'Can you analyze this PDF?' },
-            { type: 'image_url', image_url: { url: 'https://unec.edu.az/application/uploads/2014/12/pdf-sample.pdf' } }
+            { type: 'image_url', image_url: { url: 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf' } }
         ]}
     ];
 

package/tests/subscription.test.js

@@ -162,13 +162,19 @@ function validateProgressMessage(t, progress, requestId = null) {
   }
 
   // Validate info field if present and not an error
-  if (progress.info && !progress.info.startsWith('ERROR:')) {
+  if (progress.info) {
     t.true(typeof progress.info === 'string', 'Info field should be a string');
     t.notThrows(() => {
       const parsedInfo = JSON.parse(progress.info);
       t.true(typeof parsedInfo === 'object', 'Info should be valid JSON object');
     }, 'Info should be valid JSON');
   }
+
+  // Validate error field if present
+  if (progress.error) {
+    t.true(typeof progress.error === 'string', 'Error field should be a string');
+    t.notThrows(() => JSON.parse(progress.error), 'Error should be valid JSON');
+  }
 }
 
 test.serial('Request progress messages have string data and info fields', async (t) => {