@aj-archipelago/cortex 1.3.67 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/config.js +27 -0
  2. package/helper-apps/cortex-doc-to-pdf/DocToPdfFunction/__init__.py +3 -0
  3. package/helper-apps/cortex-doc-to-pdf/DocToPdfFunction/function.json +20 -0
  4. package/helper-apps/cortex-doc-to-pdf/Dockerfile +46 -0
  5. package/helper-apps/cortex-doc-to-pdf/README.md +408 -0
  6. package/helper-apps/cortex-doc-to-pdf/converter.py +157 -0
  7. package/helper-apps/cortex-doc-to-pdf/docker-compose.yml +23 -0
  8. package/helper-apps/cortex-doc-to-pdf/document_converter.py +181 -0
  9. package/helper-apps/cortex-doc-to-pdf/examples/README.md +252 -0
  10. package/helper-apps/cortex-doc-to-pdf/examples/nodejs-client.js +266 -0
  11. package/helper-apps/cortex-doc-to-pdf/examples/package-lock.json +297 -0
  12. package/helper-apps/cortex-doc-to-pdf/examples/package.json +23 -0
  13. package/helper-apps/cortex-doc-to-pdf/function_app.py +85 -0
  14. package/helper-apps/cortex-doc-to-pdf/host.json +16 -0
  15. package/helper-apps/cortex-doc-to-pdf/request_handlers.py +193 -0
  16. package/helper-apps/cortex-doc-to-pdf/requirements.txt +3 -0
  17. package/helper-apps/cortex-doc-to-pdf/tests/run_tests.sh +26 -0
  18. package/helper-apps/cortex-doc-to-pdf/tests/test_conversion.py +320 -0
  19. package/helper-apps/cortex-doc-to-pdf/tests/test_streaming.py +419 -0
  20. package/helper-apps/cortex-file-handler/package-lock.json +1 -0
  21. package/helper-apps/cortex-file-handler/package.json +1 -0
  22. package/helper-apps/cortex-file-handler/src/services/ConversionService.js +81 -8
  23. package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +54 -7
  24. package/helper-apps/cortex-file-handler/tests/getOperations.test.js +19 -7
  25. package/lib/encodeCache.js +5 -0
  26. package/lib/keyValueStorageClient.js +5 -0
  27. package/lib/logger.js +1 -1
  28. package/lib/pathwayTools.js +8 -1
  29. package/lib/redisSubscription.js +6 -0
  30. package/lib/requestExecutor.js +4 -0
  31. package/lib/util.js +88 -0
  32. package/package.json +1 -1
  33. package/pathways/basePathway.js +3 -3
  34. package/pathways/bing_afagent.js +1 -0
  35. package/pathways/gemini_15_vision.js +1 -1
  36. package/pathways/google_cse.js +2 -2
  37. package/pathways/image_gemini_25.js +85 -0
  38. package/pathways/image_prompt_optimizer_gemini_25.js +149 -0
  39. package/pathways/image_qwen.js +28 -0
  40. package/pathways/image_seedream4.js +26 -0
  41. package/pathways/rag.js +1 -1
  42. package/pathways/rag_jarvis.js +1 -1
  43. package/pathways/system/entity/sys_entity_continue.js +1 -1
  44. package/pathways/system/entity/sys_generator_results.js +1 -1
  45. package/pathways/system/entity/tools/sys_tool_google_search.js +15 -2
  46. package/pathways/system/entity/tools/sys_tool_grok_x_search.js +3 -3
  47. package/pathways/system/entity/tools/sys_tool_image.js +28 -23
  48. package/pathways/system/entity/tools/sys_tool_image_gemini.js +135 -0
  49. package/server/graphql.js +9 -2
  50. package/server/modelExecutor.js +4 -0
  51. package/server/pathwayResolver.js +19 -18
  52. package/server/plugins/claude3VertexPlugin.js +13 -8
  53. package/server/plugins/gemini15ChatPlugin.js +15 -10
  54. package/server/plugins/gemini15VisionPlugin.js +2 -23
  55. package/server/plugins/gemini25ImagePlugin.js +155 -0
  56. package/server/plugins/modelPlugin.js +3 -2
  57. package/server/plugins/openAiChatPlugin.js +6 -6
  58. package/server/plugins/replicateApiPlugin.js +268 -12
  59. package/server/plugins/veoVideoPlugin.js +15 -1
  60. package/server/rest.js +2 -0
  61. package/server/typeDef.js +96 -10
  62. package/tests/integration/apptekTranslatePlugin.integration.test.js +1 -1
  63. package/tests/unit/core/pathwayManager.test.js +2 -4
  64. package/tests/unit/plugins/gemini25ImagePlugin.test.js +294 -0
@@ -0,0 +1,155 @@
1
+ import Gemini15VisionPlugin from './gemini15VisionPlugin.js';
2
+ import logger from '../../lib/logger.js';
3
+ import CortexResponse from '../../lib/cortexResponse.js';
4
+
5
+ class Gemini25ImagePlugin extends Gemini15VisionPlugin {
6
+
7
+ constructor(pathway, model) {
8
+ super(pathway, model);
9
+ }
10
+
11
+ // Override getRequestParameters to add Gemini 2.5 specific response_modalities support
12
+ getRequestParameters(text, parameters, prompt, cortexRequest) {
13
+ const baseParameters = super.getRequestParameters(text, parameters, prompt, cortexRequest);
14
+
15
+ // Add Gemini 2.5 specific response_modalities support
16
+ let responseModalities = parameters?.response_modalities;
17
+ if (typeof responseModalities === 'string') {
18
+ try {
19
+ responseModalities = JSON.parse(responseModalities);
20
+ } catch (e) {
21
+ responseModalities = null;
22
+ }
23
+ }
24
+
25
+ // Also check pathway parameters
26
+ if (!responseModalities && cortexRequest?.pathway?.response_modalities) {
27
+ responseModalities = cortexRequest.pathway.response_modalities;
28
+ }
29
+
30
+ // Set up generation_config with response_modalities if specified
31
+ if (responseModalities && Array.isArray(responseModalities)) {
32
+ baseParameters.generationConfig.response_modalities = responseModalities;
33
+ }
34
+
35
+ return baseParameters;
36
+ }
37
+
38
+
39
+ // Override parseResponse to add Gemini 2.5 image artifact support
40
+ parseResponse(data) {
41
+ // First, let the parent Gemini15VisionPlugin handle the response
42
+ const baseResponse = super.parseResponse(data);
43
+
44
+ // If the parent already created a CortexResponse (for tool calls, safety blocks, etc.),
45
+ // we need to add image artifacts to it
46
+ if (baseResponse && typeof baseResponse === 'object' && baseResponse.constructor && baseResponse.constructor.name === 'CortexResponse') {
47
+ // Check for image artifacts in the raw data
48
+ if (data?.candidates?.[0]?.content?.parts) {
49
+ const parts = data.candidates[0].content.parts;
50
+ let imageContent = [];
51
+
52
+ for (const part of parts) {
53
+ if (part.inlineData) {
54
+ // Handle generated image content
55
+ imageContent.push({
56
+ type: "image",
57
+ data: part.inlineData.data,
58
+ mimeType: part.inlineData.mimeType
59
+ });
60
+ }
61
+ }
62
+
63
+ // If we have image artifacts, add them to the existing CortexResponse
64
+ if (imageContent.length > 0) {
65
+ baseResponse.artifacts = imageContent;
66
+ }
67
+ }
68
+
69
+ return baseResponse;
70
+ }
71
+
72
+ // If the parent returned a string or other non-CortexResponse, check for image artifacts
73
+ if (data?.candidates?.[0]?.content?.parts) {
74
+ const parts = data.candidates[0].content.parts;
75
+ let imageContent = [];
76
+ let textContent = '';
77
+
78
+ for (const part of parts) {
79
+ if (part.inlineData) {
80
+ // Handle generated image content
81
+ imageContent.push({
82
+ type: "image",
83
+ data: part.inlineData.data,
84
+ mimeType: part.inlineData.mimeType
85
+ });
86
+ } else if (part.text) {
87
+ textContent += part.text;
88
+ }
89
+ }
90
+
91
+ // If we have image artifacts, create a new CortexResponse object
92
+ if (imageContent.length > 0) {
93
+ return new CortexResponse({
94
+ output_text: textContent || baseResponse || '',
95
+ artifacts: imageContent,
96
+ finishReason: data?.candidates?.[0]?.finishReason || 'stop',
97
+ usage: data?.usage || null
98
+ });
99
+ }
100
+ }
101
+
102
+ return baseResponse;
103
+ }
104
+
105
+
106
+ // Override processStreamEvent to add Gemini 2.5 image artifact streaming
107
+ processStreamEvent(event, requestProgress) {
108
+ const baseProgress = super.processStreamEvent(event, requestProgress);
109
+
110
+ // Add image artifact streaming for Gemini 2.5
111
+ const eventData = JSON.parse(event.data);
112
+ if (eventData.candidates?.[0]?.content?.parts) {
113
+ const parts = eventData.candidates[0].content.parts;
114
+
115
+ for (const part of parts) {
116
+ if (part.inlineData) {
117
+ // Handle generated image content in streaming
118
+ // For now, we'll accumulate images and send them at the end
119
+ // This could be enhanced to stream image data if needed
120
+ if (!requestProgress.artifacts) {
121
+ requestProgress.artifacts = [];
122
+ }
123
+ const inlineData = part.inlineData;
124
+ requestProgress.artifacts.push({
125
+ type: "image",
126
+ data: inlineData.data,
127
+ mimeType: inlineData.mimeType
128
+ });
129
+ }
130
+ }
131
+ }
132
+
133
+ return baseProgress;
134
+ }
135
+
136
+ // Override logRequestData to properly handle CortexResponse objects
137
+ logRequestData(data, responseData, prompt) {
138
+ // Check if responseData is a CortexResponse object
139
+ if (responseData && typeof responseData === 'object' && responseData.constructor && responseData.constructor.name === 'CortexResponse') {
140
+ const { length, units } = this.getLength(responseData.output_text || '');
141
+ logger.info(`[response received containing ${length} ${units}]`);
142
+ if (responseData.artifacts && responseData.artifacts.length > 0) {
143
+ logger.info(`[response contains ${responseData.artifacts.length} image artifact(s)]`);
144
+ }
145
+ logger.verbose(`${this.shortenContent(responseData.output_text || '')}`);
146
+ return;
147
+ }
148
+
149
+ // Fall back to parent implementation for non-CortexResponse objects
150
+ super.logRequestData(data, responseData, prompt);
151
+ }
152
+
153
+ }
154
+
155
+ export default Gemini25ImagePlugin;
@@ -6,6 +6,7 @@ import { getFirstNToken } from '../chunker.js';
6
6
  import logger, { obscureUrlParams } from '../../lib/logger.js';
7
7
  import { config } from '../../config.js';
8
8
  import axios from 'axios';
9
+ import { extractValueFromTypeSpec } from '../typeDef.js';
9
10
 
10
11
  const DEFAULT_MAX_TOKENS = 4096;
11
12
  const DEFAULT_MAX_RETURN_TOKENS = 256;
@@ -28,11 +29,11 @@ class ModelPlugin {
28
29
 
29
30
  // Make all of the parameters defined on the pathway itself available to the prompt
30
31
  for (const [k, v] of Object.entries(pathway)) {
31
- this.promptParameters[k] = v?.default ?? v;
32
+ this.promptParameters[k] = extractValueFromTypeSpec(v?.default ?? v);
32
33
  }
33
34
  if (pathway.inputParameters) {
34
35
  for (const [k, v] of Object.entries(pathway.inputParameters)) {
35
- this.promptParameters[k] = v?.default ?? v;
36
+ this.promptParameters[k] = extractValueFromTypeSpec(v?.default ?? v);
36
37
  }
37
38
  }
38
39
 
@@ -48,12 +48,12 @@ class OpenAIChatPlugin extends ModelPlugin {
48
48
  const { modelPromptText, modelPromptMessages, tokenLength, modelPrompt } = this.getCompiledPrompt(text, parameters, prompt);
49
49
  let { stream, tools, functions } = parameters;
50
50
 
51
- if (typeof tools === 'string') {
52
- tools = JSON.parse(tools);
53
- }
54
-
55
- if (typeof functions === 'string') {
56
- functions = JSON.parse(functions);
51
+ try {
52
+ tools = (tools && typeof tools === 'string' && tools !== '' ? JSON.parse(tools) : tools);
53
+ functions = (functions && typeof functions === 'string' && functions !== '' ? JSON.parse(functions) : functions);
54
+ } catch (e) {
55
+ tools = [];
56
+ functions = [];
57
57
  }
58
58
 
59
59
  // Define the model's max token length
@@ -1,5 +1,6 @@
1
1
  // replicateApiPlugin.js
2
2
  import ModelPlugin from "./modelPlugin.js";
3
+ import CortexResponse from "../../lib/cortexResponse.js";
3
4
  import logger from "../../lib/logger.js";
4
5
  import axios from "axios";
5
6
 
@@ -92,6 +93,122 @@ class ReplicateApiPlugin extends ModelPlugin {
92
93
  };
93
94
  break;
94
95
  }
96
+ case "replicate-qwen-image": {
97
+ const aspectRatio = combinedParameters.aspect_ratio ?? combinedParameters.aspectRatio ?? "16:9";
98
+ const imageSize = combinedParameters.image_size ?? combinedParameters.imageSize ?? "optimize_for_quality";
99
+ const outputFormat = combinedParameters.output_format ?? combinedParameters.outputFormat ?? "webp";
100
+ const outputQuality = combinedParameters.output_quality ?? combinedParameters.outputQuality ?? 80;
101
+ const loraScale = combinedParameters.lora_scale ?? combinedParameters.loraScale ?? 1;
102
+ const enhancePrompt = combinedParameters.enhance_prompt ?? combinedParameters.enhancePrompt ?? false;
103
+ const negativePrompt = combinedParameters.negative_prompt ?? combinedParameters.negativePrompt ?? " ";
104
+ const numInferenceSteps = combinedParameters.num_inference_steps ?? combinedParameters.steps ?? 50;
105
+ const goFast = combinedParameters.go_fast ?? combinedParameters.goFast ?? true;
106
+ const guidance = combinedParameters.guidance ?? 4;
107
+ const strength = combinedParameters.strength ?? 0.9;
108
+ const numOutputs = combinedParameters.num_outputs ?? combinedParameters.numberResults;
109
+ const disableSafetyChecker = combinedParameters.disable_safety_checker ?? combinedParameters.disableSafetyChecker ?? false;
110
+
111
+ requestParameters = {
112
+ input: {
113
+ prompt: modelPromptText,
114
+ go_fast: goFast,
115
+ guidance,
116
+ strength,
117
+ image_size: imageSize,
118
+ lora_scale: loraScale,
119
+ aspect_ratio: aspectRatio,
120
+ output_format: outputFormat,
121
+ enhance_prompt: enhancePrompt,
122
+ output_quality: outputQuality,
123
+ negative_prompt: negativePrompt,
124
+ num_inference_steps: numInferenceSteps,
125
+ disable_safety_checker: disableSafetyChecker,
126
+ ...(numOutputs ? { num_outputs: numOutputs } : {}),
127
+ ...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) ? { seed: combinedParameters.seed } : {}),
128
+ ...(combinedParameters.image ? { image: combinedParameters.image } : {}),
129
+ ...(combinedParameters.input_image ? { input_image: combinedParameters.input_image } : {}),
130
+ },
131
+ };
132
+ break;
133
+ }
134
+ case "replicate-qwen-image-edit-plus": {
135
+ const aspectRatio = combinedParameters.aspect_ratio ?? combinedParameters.aspectRatio ?? "match_input_image";
136
+ const outputFormat = combinedParameters.output_format ?? combinedParameters.outputFormat ?? "webp";
137
+ const outputQuality = combinedParameters.output_quality ?? combinedParameters.outputQuality ?? 95;
138
+ const goFast = combinedParameters.go_fast ?? combinedParameters.goFast ?? true;
139
+ const disableSafetyChecker = combinedParameters.disable_safety_checker ?? combinedParameters.disableSafetyChecker ?? false;
140
+
141
+ const collectImages = (candidate, accumulator) => {
142
+ if (!candidate) return;
143
+ if (Array.isArray(candidate)) {
144
+ candidate.forEach((item) => collectImages(item, accumulator));
145
+ return;
146
+ }
147
+ accumulator.push(candidate);
148
+ };
149
+
150
+ const imageCandidates = [];
151
+ collectImages(combinedParameters.image, imageCandidates);
152
+ collectImages(combinedParameters.images, imageCandidates);
153
+ collectImages(combinedParameters.input_image, imageCandidates);
154
+ collectImages(combinedParameters.input_images, imageCandidates);
155
+ collectImages(combinedParameters.input_image_1, imageCandidates);
156
+ collectImages(combinedParameters.input_image_2, imageCandidates);
157
+ collectImages(combinedParameters.input_image_3, imageCandidates);
158
+ collectImages(combinedParameters.image_1, imageCandidates);
159
+ collectImages(combinedParameters.image_2, imageCandidates);
160
+
161
+ const normalizeImageEntry = (entry) => {
162
+ if (!entry) return null;
163
+ if (typeof entry === "string") {
164
+ return entry; // Return the URL string directly
165
+ }
166
+ if (typeof entry === "object") {
167
+ if (Array.isArray(entry)) {
168
+ return null;
169
+ }
170
+ if (entry.value) {
171
+ return entry.value; // Return the value as a string
172
+ }
173
+ if (entry.url) {
174
+ return entry.url; // Return the URL as a string
175
+ }
176
+ if (entry.path) {
177
+ return entry.path; // Return the path as a string
178
+ }
179
+ }
180
+ return null;
181
+ };
182
+
183
+ const normalizedImages = imageCandidates
184
+ .map((candidate) => normalizeImageEntry(candidate))
185
+ .filter((candidate) => candidate && typeof candidate === 'string');
186
+
187
+ const omitUndefined = (obj) =>
188
+ Object.fromEntries(
189
+ Object.entries(obj).filter(([, value]) => value !== undefined && value !== null),
190
+ );
191
+
192
+ const basePayload = omitUndefined({
193
+ prompt: modelPromptText,
194
+ go_fast: goFast,
195
+ aspect_ratio: aspectRatio,
196
+ output_format: outputFormat,
197
+ output_quality: outputQuality,
198
+ disable_safety_checker: disableSafetyChecker,
199
+ });
200
+
201
+ // For qwen-image-edit-plus, always include the image array if we have images
202
+ const inputPayload = {
203
+ ...basePayload,
204
+ ...(normalizedImages.length > 0 ? { image: normalizedImages } : {})
205
+ };
206
+
207
+ requestParameters = {
208
+ input: inputPayload,
209
+ };
210
+ break;
211
+ }
95
212
  case "replicate-flux-kontext-pro":
96
213
  case "replicate-flux-kontext-max": {
97
214
  const validRatios = [
@@ -110,7 +227,7 @@ class ReplicateApiPlugin extends ModelPlugin {
110
227
  input_image: combinedParameters.input_image,
111
228
  aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
112
229
  safety_tolerance: safetyTolerance,
113
- ...(combinedParameters.seed && Number.isInteger(combinedParameters.seed && combinedParameters.seed > 0) ? { seed: combinedParameters.seed } : {}),
230
+ ...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) && combinedParameters.seed > 0 ? { seed: combinedParameters.seed } : {}),
114
231
  },
115
232
  };
116
233
  break;
@@ -133,7 +250,7 @@ class ReplicateApiPlugin extends ModelPlugin {
133
250
  input_image_2: combinedParameters.input_image_2,
134
251
  aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
135
252
  safety_tolerance: safetyTolerance,
136
- ...(combinedParameters.seed && Number.isInteger(combinedParameters.seed && combinedParameters.seed > 0) ? { seed: combinedParameters.seed } : {}),
253
+ ...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) && combinedParameters.seed > 0 ? { seed: combinedParameters.seed } : {}),
137
254
  },
138
255
  };
139
256
  break;
@@ -148,7 +265,7 @@ class ReplicateApiPlugin extends ModelPlugin {
148
265
  prompt: modelPromptText,
149
266
  resolution: validResolutions.includes(combinedParameters.resolution) ? combinedParameters.resolution : "1080p",
150
267
  aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "16:9",
151
- ...(combinedParameters.seed && Number.isInteger(combinedParameters.seed && combinedParameters.seed > 0) ? { seed: combinedParameters.seed } : {}),
268
+ ...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) && combinedParameters.seed > 0 ? { seed: combinedParameters.seed } : {}),
152
269
  fps: validFps.includes(combinedParameters.fps) ? combinedParameters.fps : 24,
153
270
  camera_fixed: combinedParameters.camera_fixed || false,
154
271
  duration: combinedParameters.duration || 5,
@@ -157,6 +274,86 @@ class ReplicateApiPlugin extends ModelPlugin {
157
274
  };
158
275
  break;
159
276
  }
277
+ case "replicate-seedream-4": {
278
+ const validSizes = ["1K", "2K", "4K", "custom"];
279
+ const validRatios = ["1:1", "4:3", "3:4", "16:9", "9:16", "match_input_image"];
280
+ const validSequentialModes = ["disabled", "auto"];
281
+
282
+ // Collect input images from multiple parameter sources (same pattern as qwen-image-edit-plus)
283
+ const collectImages = (candidate, accumulator) => {
284
+ if (!candidate) return;
285
+ if (Array.isArray(candidate)) {
286
+ candidate.forEach((item) => collectImages(item, accumulator));
287
+ return;
288
+ }
289
+ accumulator.push(candidate);
290
+ };
291
+
292
+ const imageCandidates = [];
293
+ collectImages(combinedParameters.image, imageCandidates);
294
+ collectImages(combinedParameters.images, imageCandidates);
295
+ collectImages(combinedParameters.input_image, imageCandidates);
296
+ collectImages(combinedParameters.input_images, imageCandidates);
297
+ collectImages(combinedParameters.input_image_1, imageCandidates);
298
+ collectImages(combinedParameters.input_image_2, imageCandidates);
299
+ collectImages(combinedParameters.input_image_3, imageCandidates);
300
+ collectImages(combinedParameters.image_1, imageCandidates);
301
+ collectImages(combinedParameters.image_2, imageCandidates);
302
+ collectImages(combinedParameters.imageInput, imageCandidates);
303
+
304
+ const normalizeImageEntry = (entry) => {
305
+ if (!entry) return null;
306
+ if (typeof entry === "string") {
307
+ return entry; // Return the URL string directly
308
+ }
309
+ if (typeof entry === "object") {
310
+ if (Array.isArray(entry)) {
311
+ return null;
312
+ }
313
+ if (entry.value) {
314
+ return entry.value; // Return the value as a string
315
+ }
316
+ if (entry.url) {
317
+ return entry.url; // Return the URL as a string
318
+ }
319
+ if (entry.path) {
320
+ return entry.path; // Return the path as a string
321
+ }
322
+ }
323
+ return null;
324
+ };
325
+
326
+ const normalizedImages = imageCandidates
327
+ .map((candidate) => normalizeImageEntry(candidate))
328
+ .filter((candidate) => candidate && typeof candidate === 'string');
329
+
330
+ const omitUndefined = (obj) =>
331
+ Object.fromEntries(
332
+ Object.entries(obj).filter(([, value]) => value !== undefined && value !== null),
333
+ );
334
+
335
+ const basePayload = omitUndefined({
336
+ prompt: modelPromptText,
337
+ size: validSizes.includes(combinedParameters.size) ? combinedParameters.size : "2K",
338
+ width: combinedParameters.width || 2048,
339
+ height: combinedParameters.height || 2048,
340
+ max_images: combinedParameters.maxImages || combinedParameters.numberResults || 1,
341
+ aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "4:3",
342
+ sequential_image_generation: validSequentialModes.includes(combinedParameters.sequentialImageGeneration) ? combinedParameters.sequentialImageGeneration : "disabled",
343
+ ...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) && combinedParameters.seed > 0 ? { seed: combinedParameters.seed } : {}),
344
+ });
345
+
346
+ // For seedream-4, include the image_input array if we have images
347
+ const inputPayload = {
348
+ ...basePayload,
349
+ ...(normalizedImages.length > 0 ? { image_input: normalizedImages } : {})
350
+ };
351
+
352
+ requestParameters = {
353
+ input: inputPayload,
354
+ };
355
+ break;
356
+ }
160
357
  }
161
358
 
162
359
  return requestParameters;
@@ -174,12 +371,14 @@ class ReplicateApiPlugin extends ModelPlugin {
174
371
  cortexRequest.params = requestParameters.params;
175
372
 
176
373
  // Make initial request to start prediction
177
- const stringifiedResponse = await this.executeRequest(cortexRequest);
178
- const parsedResponse = JSON.parse(stringifiedResponse);
374
+ const response = await this.executeRequest(cortexRequest);
375
+
376
+ // Parse the response to get the actual Replicate data
377
+ const parsedResponse = JSON.parse(response.output_text);
179
378
 
180
- // If we got a completed response, return it
379
+ // If we got a completed response, return it as CortexResponse
181
380
  if (parsedResponse?.status === "succeeded") {
182
- return stringifiedResponse;
381
+ return this.createCortexResponse(response);
183
382
  }
184
383
 
185
384
  logger.info("Replicate API returned a non-completed response.");
@@ -211,7 +410,9 @@ class ReplicateApiPlugin extends ModelPlugin {
211
410
 
212
411
  if (status === "succeeded") {
213
412
  logger.info("Replicate API returned a completed response after polling");
214
- return JSON.stringify(pollResponse.data);
413
+ // Parse the polled response to extract artifacts
414
+ const parsedResponse = this.parseResponse(pollResponse.data);
415
+ return this.createCortexResponse(parsedResponse);
215
416
  } else if (status === "failed" || status === "canceled") {
216
417
  throw new Error(`Prediction ${status}: ${pollResponse.data?.error || "Unknown error"}`);
217
418
  }
@@ -227,12 +428,67 @@ class ReplicateApiPlugin extends ModelPlugin {
227
428
  throw new Error(`Prediction ${predictionId} timed out after ${maxAttempts * pollInterval / 1000} seconds`);
228
429
  }
229
430
 
230
- // Stringify the response from the Replicate API
431
+ // Parse the response from the Replicate API and extract image artifacts
231
432
  parseResponse(data) {
232
- if (data.data) {
233
- return JSON.stringify(data.data);
433
+ const responseData = data.data || data;
434
+ const stringifiedResponse = JSON.stringify(responseData);
435
+
436
+ // Extract image URLs from Replicate response for artifacts
437
+ const imageArtifacts = [];
438
+ if (responseData?.output && Array.isArray(responseData.output)) {
439
+ for (const outputItem of responseData.output) {
440
+ if (typeof outputItem === 'string' && outputItem.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
441
+ // This is an image URL from Replicate
442
+ imageArtifacts.push({
443
+ type: "image",
444
+ url: outputItem,
445
+ mimeType: this.getMimeTypeFromUrl(outputItem)
446
+ });
447
+ }
448
+ }
449
+ }
450
+
451
+ return {
452
+ output_text: stringifiedResponse,
453
+ artifacts: imageArtifacts
454
+ };
455
+ }
456
+
457
+ // Create a CortexResponse from parsed response data
458
+ createCortexResponse(parsedResponse) {
459
+ if (typeof parsedResponse === 'string') {
460
+ // Handle string response (backward compatibility)
461
+ return new CortexResponse({
462
+ output_text: parsedResponse,
463
+ artifacts: []
464
+ });
465
+ } else if (parsedResponse && typeof parsedResponse === 'object') {
466
+ // Handle object response with artifacts
467
+ return new CortexResponse({
468
+ output_text: parsedResponse.output_text,
469
+ artifacts: parsedResponse.artifacts || []
470
+ });
471
+ } else {
472
+ throw new Error('Unexpected response format');
473
+ }
474
+ }
475
+
476
+ // Helper method to determine MIME type from URL extension
477
+ getMimeTypeFromUrl(url) {
478
+ const extension = url.split('.').pop().toLowerCase();
479
+ switch (extension) {
480
+ case 'jpg':
481
+ case 'jpeg':
482
+ return 'image/jpeg';
483
+ case 'png':
484
+ return 'image/png';
485
+ case 'gif':
486
+ return 'image/gif';
487
+ case 'webp':
488
+ return 'image/webp';
489
+ default:
490
+ return 'image/jpeg'; // Default fallback
234
491
  }
235
- return JSON.stringify(data);
236
492
  }
237
493
 
238
494
  // Override the logging function to display the request and response
@@ -19,7 +19,8 @@ class VeoVideoPlugin extends ModelPlugin {
19
19
  // Available Veo models
20
20
  const availableModels = {
21
21
  'veo-2.0-generate': 'GA',
22
- 'veo-3.0-generate': 'Preview'
22
+ 'veo-3.0-generate': 'Preview',
23
+ 'veo-3.0-fast-generate': 'Preview'
23
24
  };
24
25
 
25
26
  // Get the model ID from the pathway or use default
@@ -51,6 +52,7 @@ class VeoVideoPlugin extends ModelPlugin {
51
52
  ...(combinedParameters.enhancePrompt !== undefined && { enhancePrompt: combinedParameters.enhancePrompt }),
52
53
  // generateAudio is required for 3.0 and not supported by 2.0
53
54
  ...(model === 'veo-3.0-generate' && { generateAudio: combinedParameters.generateAudio !== undefined ? combinedParameters.generateAudio : true }),
55
+ ...(model === 'veo-3.0-fast-generate' && { generateAudio: combinedParameters.generateAudio !== undefined ? combinedParameters.generateAudio : true }),
54
56
  ...(combinedParameters.negativePrompt && { negativePrompt: combinedParameters.negativePrompt }),
55
57
  ...(combinedParameters.personGeneration && { personGeneration: combinedParameters.personGeneration }),
56
58
  ...(combinedParameters.sampleCount && { sampleCount: combinedParameters.sampleCount }),
@@ -69,6 +71,9 @@ class VeoVideoPlugin extends ModelPlugin {
69
71
  if (model === 'veo-3.0-generate' && parameters.durationSeconds !== 8) {
70
72
  throw new Error(`Veo 3.0 only supports durationSeconds: 8, got: ${parameters.durationSeconds}`);
71
73
  }
74
+ if (model === 'veo-3.0-fast-generate' && parameters.durationSeconds !== 8) {
75
+ throw new Error(`Veo 3.0 only supports durationSeconds: 8, got: ${parameters.durationSeconds}`);
76
+ }
72
77
  if (model === 'veo-2.0-generate' && (parameters.durationSeconds < 5 || parameters.durationSeconds > 8)) {
73
78
  throw new Error(`Veo 2.0 supports durationSeconds between 5-8, got: ${parameters.durationSeconds}`);
74
79
  }
@@ -82,6 +87,12 @@ class VeoVideoPlugin extends ModelPlugin {
82
87
  if (parameters.video) {
83
88
  throw new Error('video parameter is not supported in Veo 3.0');
84
89
  }
90
+ if (model === 'veo-3.0-fast-generate' && parameters.lastFrame) {
91
+ throw new Error('lastFrame parameter is not supported in Veo 3.0');
92
+ }
93
+ if (model === 'veo-3.0-fast-generate' && parameters.video) {
94
+ throw new Error('video parameter is not supported in Veo 3.0');
95
+ }
85
96
  }
86
97
 
87
98
  // generateAudio constraints
@@ -91,6 +102,9 @@ class VeoVideoPlugin extends ModelPlugin {
91
102
  if (model === 'veo-3.0-generate' && parameters.generateAudio === undefined) {
92
103
  logger.warn('generateAudio is required for Veo 3.0, defaulting to true');
93
104
  }
105
+ if (model === 'veo-3.0-fast-generate' && parameters.generateAudio === undefined) {
106
+ logger.warn('generateAudio is required for Veo 3.0, defaulting to true');
107
+ }
94
108
  }
95
109
 
96
110
  // Execute the request to the Veo API
package/server/rest.js CHANGED
@@ -139,6 +139,8 @@ const processRestRequest = async (server, req, pathway, name, parameterMap = {})
139
139
  return Boolean(value);
140
140
  } else if (type === 'Int') {
141
141
  return parseInt(value, 10);
142
+ } else if (type === 'Float') {
143
+ return parseFloat(value);
142
144
  } else if (type === '[MultiMessage]' && Array.isArray(value)) {
143
145
  return value.map(msg => ({
144
146
  ...msg,