@aj-archipelago/cortex 1.3.67 → 1.4.0

package/server/typeDef.js

@@ -1,30 +1,113 @@
+// Check if a value is a JSON Schema object for parameter typing
+const isJsonSchemaObject = (value) => {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) return false;
+  // Basic JSON Schema indicators
+  return (
+    typeof value.type === 'string' ||
+    value.$ref !== undefined ||
+    value.oneOf !== undefined ||
+    value.anyOf !== undefined ||
+    value.allOf !== undefined ||
+    value.enum !== undefined ||
+    value.properties !== undefined ||
+    value.items !== undefined
+  );
+};
+
+// Extract the default value from a JSON Schema object or return the value as-is
+const extractValueFromTypeSpec = (value) => {
+  if (isJsonSchemaObject(value)) {
+    return value.hasOwnProperty('default') ? value.default : undefined;
+  }
+  return value;
+};
+
+// Process parameters to convert any type specification objects to their actual values
+const processPathwayParameters = (params) => {
+  if (!params || typeof params !== 'object') {
+    return params;
+  }
+  
+  const processed = {};
+  for (const [key, value] of Object.entries(params)) {
+    processed[key] = extractValueFromTypeSpec(value);
+  }
+  return processed;
+};
+
 const getGraphQlType = (value) => {
+  // The value might be an object with JSON Schema type specification
+  if (isJsonSchemaObject(value)) {
+    const schema = value;
+    // Map JSON Schema to GraphQL
+    if (schema.type === 'boolean') {
+      return { type: 'Boolean', defaultValue: schema.default === undefined ? undefined : schema.default };
+    }
+    if (schema.type === 'string') {
+      return { type: 'String', defaultValue: schema.default === undefined ? undefined : `"${schema.default}"` };
+    }
+    if (schema.type === 'integer') {
+      return { type: 'Int', defaultValue: schema.default };
+    }
+    if (schema.type === 'number') {
+      const def = schema.default;
+      return { type: 'Float', defaultValue: def };
+    }
+    if (schema.type === 'array') {
+      // Support arrays of primitive types; fall back to JSON string for complex types
+      const items = schema.items || {};
+      const def = schema.default;
+      const defaultArray = Array.isArray(def) ? JSON.stringify(def) : '[]';
+      if (items.type === 'string') {
+        return { type: '[String]', defaultValue: defaultArray };
+      }
+      if (items.type === 'integer') {
+        return { type: '[Int]', defaultValue: defaultArray };
+      }
+      if (items.type === 'number') {
+        return { type: '[Float]', defaultValue: defaultArray };
+      }
+      if (items.type === 'boolean') {
+        return { type: '[Boolean]', defaultValue: defaultArray };
+      }
+      // Unknown item type: pass as serialized JSON string argument
+      return { type: 'String', defaultValue: def === undefined ? '"[]"' : `"${JSON.stringify(def).replace(/"/g, '\\"')}"` };
+    }
+    if (schema.type === 'object' || schema.properties) {
+      // Until explicit input types are defined, accept as stringified JSON
+      const def = schema.default;
+      return { type: 'String', defaultValue: def === undefined ? '"{}"' : `"${JSON.stringify(def).replace(/"/g, '\\"')}"` };
+    }
+  }
+  
+  // Otherwise, autodetect the type
   switch (typeof value) {
     case 'boolean':
-      return {type: 'Boolean'};
+      return {type: 'Boolean', defaultValue: value};
     case 'string':
-      return {type: 'String'};
+      return {type: 'String', defaultValue: `"${value}"`};
     case 'number':
-      return {type: 'Int'};
+      // Check if it's an integer or float
+      return Number.isInteger(value) ? {type: 'Int', defaultValue: value} : {type: 'Float', defaultValue: value};
     case 'object':
       if (Array.isArray(value)) {
         if (value.length > 0 && typeof(value[0]) === 'string') {
-          return {type: '[String]'};
+          return {type: '[String]', defaultValue: JSON.stringify(value)};
         }
         else {
-          // New case for MultiMessage type
+          // Check if it's MultiMessage (content is array) or Message (content is string)
           if (Array.isArray(value[0]?.content)) {
-            return {type: '[MultiMessage]'};
+            return {type: '[MultiMessage]', defaultValue: `"${JSON.stringify(value).replace(/"/g, '\\"')}"`};
           }
           else {
-            return {type: '[Message]'};
+            return {type: '[Message]', defaultValue: `"${JSON.stringify(value).replace(/"/g, '\\"')}"`};
           }
         }
       } else {
-        return {type: `[${value.objName}]`};
+        return {type: `[${value.objName}]`, defaultValue: JSON.stringify(value)};
       }
     default:
-      return {type: 'String'};
+      return {type: 'String', defaultValue: `"${value}"`};
   }
 };
 
@@ -80,7 +163,7 @@ const getPathwayTypeDefAndExtendQuery = (pathway) => {
     };
   });
 
-  const gqlDefinition = `${type}\n\n${responseType}\n\nextend type Query {${name}(${paramsStr}): ${objName}}`;
+  const gqlDefinition = `${type}\n\n${responseType}\n\nextend type Query {${name}${paramsStr ? `(${paramsStr})` : ''}: ${objName}}`;
 
   return {
     gqlDefinition,
@@ -100,4 +183,7 @@ export {
   getMessageTypeDefs,
   getPathwayTypeDef,
   userPathwayInputParameters,
+  isJsonSchemaObject,
+  extractValueFromTypeSpec,
+  processPathwayParameters,
 };

package/tests/integration/apptekTranslatePlugin.integration.test.js

@@ -155,7 +155,7 @@ test.serial('AppTek Plugin: Force failure and test GPT-4 Omni fallback', async (
 });
 
 // Test AppTek failure with default fallback (translate_groq)
-test.skip('AppTek Plugin: Force failure and test default fallback', async (t) => {
+test('AppTek Plugin: Force failure and test default fallback', async (t) => {
     // Set a longer timeout for this test since Groq might be slower
     t.timeout(180000); // 3 minutes
     

package/tests/unit/core/pathwayManager.test.js

@@ -193,8 +193,7 @@ test('_createPromptObject handles empty system prompt', t => {
 
     t.true(result instanceof Prompt);
     t.is(result.name, 'test_prompt');
-    t.is(result.messages[0].content, '');
-    t.is(result.messages[1].content, '{{text}}\n\nTest prompt text');
+    t.is(result.messages[0].content, '{{text}}\n\nTest prompt text');
 });
 
 test('_createPromptObject handles null system prompt', t => {
@@ -206,8 +205,7 @@ test('_createPromptObject handles null system prompt', t => {
 
     t.true(result instanceof Prompt);
     t.is(result.name, 'test_prompt');
-    t.is(result.messages[0].content, '');
-    t.is(result.messages[1].content, '{{text}}\n\nTest prompt text');
+    t.is(result.messages[0].content, '{{text}}\n\nTest prompt text');
 });
 
 test('putPathway requires userId and secret', async t => {

package/tests/unit/plugins/gemini25ImagePlugin.test.js

@@ -0,0 +1,294 @@
+import test from 'ava';
+import Gemini25ImagePlugin from '../../../server/plugins/gemini25ImagePlugin.js';
+import { PathwayResolver } from '../../../server/pathwayResolver.js';
+import { config } from '../../../config.js';
+import { requestState } from '../../../server/requestState.js';
+
+// Mock logger to prevent issues in tests
+const mockLogger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {}
+};
+
+// Mock the logger module globally
+global.logger = mockLogger;
+
+function createResolverWithPlugin(pluginClass, modelName = 'test-model') {
+  const pathway = {
+    name: 'test-pathway',
+    model: modelName,
+    prompt: 'test prompt',
+    toolCallback: () => {} // Mock tool callback
+  };
+  
+  const model = {
+    name: modelName,
+    type: 'GEMINI-2.5-IMAGE'
+  };
+  
+  const resolver = new PathwayResolver({ 
+    config,
+    pathway,
+    args: {},
+    endpoints: { [modelName]: model }
+  });
+  
+  resolver.modelExecutor.plugin = new pluginClass(pathway, model);
+  return resolver;
+}
+
+test('Gemini25ImagePlugin - filters undefined tool calls from buffer', async t => {
+  const resolver = createResolverWithPlugin(Gemini25ImagePlugin);
+  const plugin = resolver.modelExecutor.plugin;
+  
+  // Simulate buffer with undefined elements
+  plugin.toolCallsBuffer = [
+    undefined,
+    {
+      id: 'call_1_1234567890',
+      type: 'function',
+      function: {
+        name: 'test_tool',
+        arguments: '{"param": "value"}'
+      }
+    }
+  ];
+  
+  // Mock the tool callback
+  let capturedToolCalls = null;
+  plugin.pathwayToolCallback = (args, message, resolver) => {
+    capturedToolCalls = message.tool_calls;
+  };
+  
+  // Mock requestProgress and pathwayResolver
+  const requestProgress = { progress: 0, started: true };
+  const pathwayResolver = { args: {} };
+  
+  // Mock requestState
+  requestState[plugin.requestId] = { pathwayResolver };
+  
+  // Simulate a tool_calls finish reason
+  const eventData = {
+    candidates: [{
+      finishReason: 'STOP'
+    }]
+  };
+  
+  // Set hadToolCalls to true to trigger tool_calls finish reason
+  plugin.hadToolCalls = true;
+  
+  // Process the stream event
+  plugin.processStreamEvent({ data: JSON.stringify(eventData) }, requestProgress);
+  
+  // Verify that undefined elements were filtered out
+  t.truthy(capturedToolCalls, 'Tool callback should have been called');
+  t.is(capturedToolCalls.length, 1, 'Should have filtered out undefined elements');
+  t.is(capturedToolCalls[0].function.name, 'test_tool', 'Valid tool call should be preserved');
+  
+  // Clean up
+  delete requestState[plugin.requestId];
+});
+
+test('Gemini25ImagePlugin - handles empty buffer gracefully', async t => {
+  const resolver = createResolverWithPlugin(Gemini25ImagePlugin);
+  const plugin = resolver.modelExecutor.plugin;
+  
+  // Empty buffer
+  plugin.toolCallsBuffer = [];
+  
+  // Mock the tool callback
+  let callbackCalled = false;
+  plugin.pathwayToolCallback = () => {
+    callbackCalled = true;
+  };
+  
+  // Mock requestProgress and pathwayResolver
+  const requestProgress = { progress: 0, started: true };
+  const pathwayResolver = { args: {} };
+  
+  // Mock requestState
+  requestState[plugin.requestId] = { pathwayResolver };
+  
+  // Simulate a tool_calls finish reason
+  const eventData = {
+    candidates: [{
+      finishReason: 'STOP'
+    }]
+  };
+  
+  // Set hadToolCalls to true
+  plugin.hadToolCalls = true;
+  
+  // Process the stream event
+  plugin.processStreamEvent({ data: JSON.stringify(eventData) }, requestProgress);
+  
+  // Verify that callback was not called with empty buffer
+  t.falsy(callbackCalled, 'Tool callback should not be called with empty buffer');
+  
+  // Clean up
+  delete requestState[plugin.requestId];
+});
+
+test('Gemini25ImagePlugin - handles image artifacts in streaming', async t => {
+  const resolver = createResolverWithPlugin(Gemini25ImagePlugin);
+  const plugin = resolver.modelExecutor.plugin;
+  
+  // Mock requestProgress
+  const requestProgress = { progress: 0, started: true };
+  
+  // Simulate event with image artifact
+  const eventData = {
+    candidates: [{
+      content: {
+        parts: [{
+          inlineData: {
+            data: 'base64imagedata',
+            mimeType: 'image/png'
+          }
+        }]
+      }
+    }]
+  };
+  
+  // Process the stream event
+  plugin.processStreamEvent({ data: JSON.stringify(eventData) }, requestProgress);
+  
+  // Verify that image artifacts were captured
+  t.truthy(requestProgress.artifacts, 'Artifacts should be created');
+  t.is(requestProgress.artifacts.length, 1, 'Should have one image artifact');
+  t.is(requestProgress.artifacts[0].type, 'image', 'Artifact should be of type image');
+  t.is(requestProgress.artifacts[0].data, 'base64imagedata', 'Image data should be preserved');
+  t.is(requestProgress.artifacts[0].mimeType, 'image/png', 'MIME type should be preserved');
+});
+
+test('Gemini25ImagePlugin - handles multiple image artifacts', async t => {
+  const resolver = createResolverWithPlugin(Gemini25ImagePlugin);
+  const plugin = resolver.modelExecutor.plugin;
+  
+  // Mock requestProgress
+  const requestProgress = { progress: 0, started: true };
+  
+  // Simulate event with multiple image artifacts
+  const eventData = {
+    candidates: [{
+      content: {
+        parts: [
+          {
+            inlineData: {
+              data: 'base64image1',
+              mimeType: 'image/png'
+            }
+          },
+          {
+            inlineData: {
+              data: 'base64image2',
+              mimeType: 'image/jpeg'
+            }
+          }
+        ]
+      }
+    }]
+  };
+  
+  // Process the stream event
+  plugin.processStreamEvent({ data: JSON.stringify(eventData) }, requestProgress);
+  
+  // Verify that multiple image artifacts were captured
+  t.truthy(requestProgress.artifacts, 'Artifacts should be created');
+  t.is(requestProgress.artifacts.length, 2, 'Should have two image artifacts');
+  t.is(requestProgress.artifacts[0].type, 'image', 'First artifact should be of type image');
+  t.is(requestProgress.artifacts[0].data, 'base64image1', 'First image data should be preserved');
+  t.is(requestProgress.artifacts[0].mimeType, 'image/png', 'First MIME type should be preserved');
+  t.is(requestProgress.artifacts[1].type, 'image', 'Second artifact should be of type image');
+  t.is(requestProgress.artifacts[1].data, 'base64image2', 'Second image data should be preserved');
+  t.is(requestProgress.artifacts[1].mimeType, 'image/jpeg', 'Second MIME type should be preserved');
+});
+
+test('Gemini25ImagePlugin - handles mixed content with text and images', async t => {
+  const resolver = createResolverWithPlugin(Gemini25ImagePlugin);
+  const plugin = resolver.modelExecutor.plugin;
+  
+  // Mock requestProgress
+  const requestProgress = { progress: 0, started: true };
+  
+  // Simulate event with mixed content (text + image)
+  const eventData = {
+    candidates: [{
+      content: {
+        parts: [
+          {
+            text: 'Here is an image:'
+          },
+          {
+            inlineData: {
+              data: 'base64imagedata',
+              mimeType: 'image/png'
+            }
+          }
+        ]
+      }
+    }]
+  };
+  
+  // Process the stream event
+  plugin.processStreamEvent({ data: JSON.stringify(eventData) }, requestProgress);
+  
+  // Verify that both text content and image artifacts were handled
+  t.truthy(requestProgress.artifacts, 'Artifacts should be created');
+  t.is(requestProgress.artifacts.length, 1, 'Should have one image artifact');
+  t.is(requestProgress.artifacts[0].type, 'image', 'Artifact should be of type image');
+  t.is(requestProgress.artifacts[0].data, 'base64imagedata', 'Image data should be preserved');
+  
+  // Clean up
+  delete requestState[plugin.requestId];
+});
+
+test('Gemini25ImagePlugin - handles response_modalities parameter', async t => {
+  const resolver = createResolverWithPlugin(Gemini25ImagePlugin);
+  const plugin = resolver.modelExecutor.plugin;
+  
+  // Test with response_modalities parameter
+  const parameters = {
+    response_modalities: '["TEXT", "IMAGE"]'
+  };
+  
+  const prompt = {
+    prompt: 'test prompt'
+  };
+  
+  const cortexRequest = {
+    pathway: {}
+  };
+  
+  const requestParams = plugin.getRequestParameters('test text', parameters, prompt, cortexRequest);
+  
+  // Verify that response_modalities was added to generationConfig
+  t.truthy(requestParams.generationConfig.response_modalities, 'response_modalities should be set');
+  t.deepEqual(requestParams.generationConfig.response_modalities, ['TEXT', 'IMAGE'], 'response_modalities should be parsed correctly');
+});
+
+test('Gemini25ImagePlugin - handles response_modalities from pathway', async t => {
+  const resolver = createResolverWithPlugin(Gemini25ImagePlugin);
+  const plugin = resolver.modelExecutor.plugin;
+  
+  // Test with response_modalities from pathway
+  const parameters = {};
+  
+  const prompt = {
+    prompt: 'test prompt'
+  };
+  
+  const cortexRequest = {
+    pathway: {
+      response_modalities: ['TEXT', 'IMAGE']
+    }
+  };
+  
+  const requestParams = plugin.getRequestParameters('test text', parameters, prompt, cortexRequest);
+  
+  // Verify that response_modalities was added to generationConfig
+  t.truthy(requestParams.generationConfig.response_modalities, 'response_modalities should be set');
+  t.deepEqual(requestParams.generationConfig.response_modalities, ['TEXT', 'IMAGE'], 'response_modalities should be set correctly');
+});