@aj-archipelago/cortex 1.3.67 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.js +27 -0
- package/helper-apps/cortex-doc-to-pdf/DocToPdfFunction/__init__.py +3 -0
- package/helper-apps/cortex-doc-to-pdf/DocToPdfFunction/function.json +20 -0
- package/helper-apps/cortex-doc-to-pdf/Dockerfile +46 -0
- package/helper-apps/cortex-doc-to-pdf/README.md +408 -0
- package/helper-apps/cortex-doc-to-pdf/converter.py +157 -0
- package/helper-apps/cortex-doc-to-pdf/docker-compose.yml +23 -0
- package/helper-apps/cortex-doc-to-pdf/document_converter.py +181 -0
- package/helper-apps/cortex-doc-to-pdf/examples/README.md +252 -0
- package/helper-apps/cortex-doc-to-pdf/examples/nodejs-client.js +266 -0
- package/helper-apps/cortex-doc-to-pdf/examples/package-lock.json +297 -0
- package/helper-apps/cortex-doc-to-pdf/examples/package.json +23 -0
- package/helper-apps/cortex-doc-to-pdf/function_app.py +85 -0
- package/helper-apps/cortex-doc-to-pdf/host.json +16 -0
- package/helper-apps/cortex-doc-to-pdf/request_handlers.py +193 -0
- package/helper-apps/cortex-doc-to-pdf/requirements.txt +3 -0
- package/helper-apps/cortex-doc-to-pdf/tests/run_tests.sh +26 -0
- package/helper-apps/cortex-doc-to-pdf/tests/test_conversion.py +320 -0
- package/helper-apps/cortex-doc-to-pdf/tests/test_streaming.py +419 -0
- package/helper-apps/cortex-file-handler/package-lock.json +1 -0
- package/helper-apps/cortex-file-handler/package.json +1 -0
- package/helper-apps/cortex-file-handler/src/services/ConversionService.js +81 -8
- package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +54 -7
- package/helper-apps/cortex-file-handler/tests/getOperations.test.js +19 -7
- package/lib/encodeCache.js +5 -0
- package/lib/keyValueStorageClient.js +5 -0
- package/lib/logger.js +1 -1
- package/lib/pathwayTools.js +8 -1
- package/lib/redisSubscription.js +6 -0
- package/lib/requestExecutor.js +4 -0
- package/lib/util.js +88 -0
- package/package.json +1 -1
- package/pathways/basePathway.js +3 -3
- package/pathways/bing_afagent.js +1 -0
- package/pathways/gemini_15_vision.js +1 -1
- package/pathways/google_cse.js +2 -2
- package/pathways/image_gemini_25.js +85 -0
- package/pathways/image_prompt_optimizer_gemini_25.js +149 -0
- package/pathways/image_qwen.js +28 -0
- package/pathways/image_seedream4.js +26 -0
- package/pathways/rag.js +1 -1
- package/pathways/rag_jarvis.js +1 -1
- package/pathways/system/entity/sys_entity_continue.js +1 -1
- package/pathways/system/entity/sys_generator_results.js +1 -1
- package/pathways/system/entity/tools/sys_tool_google_search.js +15 -2
- package/pathways/system/entity/tools/sys_tool_grok_x_search.js +3 -3
- package/pathways/system/entity/tools/sys_tool_image.js +28 -23
- package/pathways/system/entity/tools/sys_tool_image_gemini.js +135 -0
- package/server/graphql.js +9 -2
- package/server/modelExecutor.js +4 -0
- package/server/pathwayResolver.js +19 -18
- package/server/plugins/claude3VertexPlugin.js +13 -8
- package/server/plugins/gemini15ChatPlugin.js +15 -10
- package/server/plugins/gemini15VisionPlugin.js +2 -23
- package/server/plugins/gemini25ImagePlugin.js +155 -0
- package/server/plugins/modelPlugin.js +3 -2
- package/server/plugins/openAiChatPlugin.js +6 -6
- package/server/plugins/replicateApiPlugin.js +268 -12
- package/server/plugins/veoVideoPlugin.js +15 -1
- package/server/rest.js +2 -0
- package/server/typeDef.js +96 -10
- package/tests/integration/apptekTranslatePlugin.integration.test.js +1 -1
- package/tests/unit/core/pathwayManager.test.js +2 -4
- package/tests/unit/plugins/gemini25ImagePlugin.test.js +294 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import Gemini15VisionPlugin from './gemini15VisionPlugin.js';
|
|
2
|
+
import logger from '../../lib/logger.js';
|
|
3
|
+
import CortexResponse from '../../lib/cortexResponse.js';
|
|
4
|
+
|
|
5
|
+
class Gemini25ImagePlugin extends Gemini15VisionPlugin {
|
|
6
|
+
|
|
7
|
+
constructor(pathway, model) {
|
|
8
|
+
super(pathway, model);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
// Override getRequestParameters to add Gemini 2.5 specific response_modalities support
|
|
12
|
+
getRequestParameters(text, parameters, prompt, cortexRequest) {
|
|
13
|
+
const baseParameters = super.getRequestParameters(text, parameters, prompt, cortexRequest);
|
|
14
|
+
|
|
15
|
+
// Add Gemini 2.5 specific response_modalities support
|
|
16
|
+
let responseModalities = parameters?.response_modalities;
|
|
17
|
+
if (typeof responseModalities === 'string') {
|
|
18
|
+
try {
|
|
19
|
+
responseModalities = JSON.parse(responseModalities);
|
|
20
|
+
} catch (e) {
|
|
21
|
+
responseModalities = null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Also check pathway parameters
|
|
26
|
+
if (!responseModalities && cortexRequest?.pathway?.response_modalities) {
|
|
27
|
+
responseModalities = cortexRequest.pathway.response_modalities;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// Set up generation_config with response_modalities if specified
|
|
31
|
+
if (responseModalities && Array.isArray(responseModalities)) {
|
|
32
|
+
baseParameters.generationConfig.response_modalities = responseModalities;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
return baseParameters;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
// Override parseResponse to add Gemini 2.5 image artifact support
|
|
40
|
+
parseResponse(data) {
|
|
41
|
+
// First, let the parent Gemini15VisionPlugin handle the response
|
|
42
|
+
const baseResponse = super.parseResponse(data);
|
|
43
|
+
|
|
44
|
+
// If the parent already created a CortexResponse (for tool calls, safety blocks, etc.),
|
|
45
|
+
// we need to add image artifacts to it
|
|
46
|
+
if (baseResponse && typeof baseResponse === 'object' && baseResponse.constructor && baseResponse.constructor.name === 'CortexResponse') {
|
|
47
|
+
// Check for image artifacts in the raw data
|
|
48
|
+
if (data?.candidates?.[0]?.content?.parts) {
|
|
49
|
+
const parts = data.candidates[0].content.parts;
|
|
50
|
+
let imageContent = [];
|
|
51
|
+
|
|
52
|
+
for (const part of parts) {
|
|
53
|
+
if (part.inlineData) {
|
|
54
|
+
// Handle generated image content
|
|
55
|
+
imageContent.push({
|
|
56
|
+
type: "image",
|
|
57
|
+
data: part.inlineData.data,
|
|
58
|
+
mimeType: part.inlineData.mimeType
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// If we have image artifacts, add them to the existing CortexResponse
|
|
64
|
+
if (imageContent.length > 0) {
|
|
65
|
+
baseResponse.artifacts = imageContent;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
return baseResponse;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// If the parent returned a string or other non-CortexResponse, check for image artifacts
|
|
73
|
+
if (data?.candidates?.[0]?.content?.parts) {
|
|
74
|
+
const parts = data.candidates[0].content.parts;
|
|
75
|
+
let imageContent = [];
|
|
76
|
+
let textContent = '';
|
|
77
|
+
|
|
78
|
+
for (const part of parts) {
|
|
79
|
+
if (part.inlineData) {
|
|
80
|
+
// Handle generated image content
|
|
81
|
+
imageContent.push({
|
|
82
|
+
type: "image",
|
|
83
|
+
data: part.inlineData.data,
|
|
84
|
+
mimeType: part.inlineData.mimeType
|
|
85
|
+
});
|
|
86
|
+
} else if (part.text) {
|
|
87
|
+
textContent += part.text;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// If we have image artifacts, create a new CortexResponse object
|
|
92
|
+
if (imageContent.length > 0) {
|
|
93
|
+
return new CortexResponse({
|
|
94
|
+
output_text: textContent || baseResponse || '',
|
|
95
|
+
artifacts: imageContent,
|
|
96
|
+
finishReason: data?.candidates?.[0]?.finishReason || 'stop',
|
|
97
|
+
usage: data?.usage || null
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return baseResponse;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
// Override processStreamEvent to add Gemini 2.5 image artifact streaming
|
|
107
|
+
processStreamEvent(event, requestProgress) {
|
|
108
|
+
const baseProgress = super.processStreamEvent(event, requestProgress);
|
|
109
|
+
|
|
110
|
+
// Add image artifact streaming for Gemini 2.5
|
|
111
|
+
const eventData = JSON.parse(event.data);
|
|
112
|
+
if (eventData.candidates?.[0]?.content?.parts) {
|
|
113
|
+
const parts = eventData.candidates[0].content.parts;
|
|
114
|
+
|
|
115
|
+
for (const part of parts) {
|
|
116
|
+
if (part.inlineData) {
|
|
117
|
+
// Handle generated image content in streaming
|
|
118
|
+
// For now, we'll accumulate images and send them at the end
|
|
119
|
+
// This could be enhanced to stream image data if needed
|
|
120
|
+
if (!requestProgress.artifacts) {
|
|
121
|
+
requestProgress.artifacts = [];
|
|
122
|
+
}
|
|
123
|
+
const inlineData = part.inlineData;
|
|
124
|
+
requestProgress.artifacts.push({
|
|
125
|
+
type: "image",
|
|
126
|
+
data: inlineData.data,
|
|
127
|
+
mimeType: inlineData.mimeType
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return baseProgress;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Override logRequestData to properly handle CortexResponse objects
|
|
137
|
+
logRequestData(data, responseData, prompt) {
|
|
138
|
+
// Check if responseData is a CortexResponse object
|
|
139
|
+
if (responseData && typeof responseData === 'object' && responseData.constructor && responseData.constructor.name === 'CortexResponse') {
|
|
140
|
+
const { length, units } = this.getLength(responseData.output_text || '');
|
|
141
|
+
logger.info(`[response received containing ${length} ${units}]`);
|
|
142
|
+
if (responseData.artifacts && responseData.artifacts.length > 0) {
|
|
143
|
+
logger.info(`[response contains ${responseData.artifacts.length} image artifact(s)]`);
|
|
144
|
+
}
|
|
145
|
+
logger.verbose(`${this.shortenContent(responseData.output_text || '')}`);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Fall back to parent implementation for non-CortexResponse objects
|
|
150
|
+
super.logRequestData(data, responseData, prompt);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export default Gemini25ImagePlugin;
|
|
@@ -6,6 +6,7 @@ import { getFirstNToken } from '../chunker.js';
|
|
|
6
6
|
import logger, { obscureUrlParams } from '../../lib/logger.js';
|
|
7
7
|
import { config } from '../../config.js';
|
|
8
8
|
import axios from 'axios';
|
|
9
|
+
import { extractValueFromTypeSpec } from '../typeDef.js';
|
|
9
10
|
|
|
10
11
|
const DEFAULT_MAX_TOKENS = 4096;
|
|
11
12
|
const DEFAULT_MAX_RETURN_TOKENS = 256;
|
|
@@ -28,11 +29,11 @@ class ModelPlugin {
|
|
|
28
29
|
|
|
29
30
|
// Make all of the parameters defined on the pathway itself available to the prompt
|
|
30
31
|
for (const [k, v] of Object.entries(pathway)) {
|
|
31
|
-
this.promptParameters[k] = v?.default ?? v;
|
|
32
|
+
this.promptParameters[k] = extractValueFromTypeSpec(v?.default ?? v);
|
|
32
33
|
}
|
|
33
34
|
if (pathway.inputParameters) {
|
|
34
35
|
for (const [k, v] of Object.entries(pathway.inputParameters)) {
|
|
35
|
-
this.promptParameters[k] = v?.default ?? v;
|
|
36
|
+
this.promptParameters[k] = extractValueFromTypeSpec(v?.default ?? v);
|
|
36
37
|
}
|
|
37
38
|
}
|
|
38
39
|
|
|
@@ -48,12 +48,12 @@ class OpenAIChatPlugin extends ModelPlugin {
|
|
|
48
48
|
const { modelPromptText, modelPromptMessages, tokenLength, modelPrompt } = this.getCompiledPrompt(text, parameters, prompt);
|
|
49
49
|
let { stream, tools, functions } = parameters;
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
tools = JSON.parse(tools);
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
functions =
|
|
51
|
+
try {
|
|
52
|
+
tools = (tools && typeof tools === 'string' && tools !== '' ? JSON.parse(tools) : tools);
|
|
53
|
+
functions = (functions && typeof functions === 'string' && functions !== '' ? JSON.parse(functions) : functions);
|
|
54
|
+
} catch (e) {
|
|
55
|
+
tools = [];
|
|
56
|
+
functions = [];
|
|
57
57
|
}
|
|
58
58
|
|
|
59
59
|
// Define the model's max token length
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// replicateApiPlugin.js
|
|
2
2
|
import ModelPlugin from "./modelPlugin.js";
|
|
3
|
+
import CortexResponse from "../../lib/cortexResponse.js";
|
|
3
4
|
import logger from "../../lib/logger.js";
|
|
4
5
|
import axios from "axios";
|
|
5
6
|
|
|
@@ -92,6 +93,122 @@ class ReplicateApiPlugin extends ModelPlugin {
|
|
|
92
93
|
};
|
|
93
94
|
break;
|
|
94
95
|
}
|
|
96
|
+
case "replicate-qwen-image": {
|
|
97
|
+
const aspectRatio = combinedParameters.aspect_ratio ?? combinedParameters.aspectRatio ?? "16:9";
|
|
98
|
+
const imageSize = combinedParameters.image_size ?? combinedParameters.imageSize ?? "optimize_for_quality";
|
|
99
|
+
const outputFormat = combinedParameters.output_format ?? combinedParameters.outputFormat ?? "webp";
|
|
100
|
+
const outputQuality = combinedParameters.output_quality ?? combinedParameters.outputQuality ?? 80;
|
|
101
|
+
const loraScale = combinedParameters.lora_scale ?? combinedParameters.loraScale ?? 1;
|
|
102
|
+
const enhancePrompt = combinedParameters.enhance_prompt ?? combinedParameters.enhancePrompt ?? false;
|
|
103
|
+
const negativePrompt = combinedParameters.negative_prompt ?? combinedParameters.negativePrompt ?? " ";
|
|
104
|
+
const numInferenceSteps = combinedParameters.num_inference_steps ?? combinedParameters.steps ?? 50;
|
|
105
|
+
const goFast = combinedParameters.go_fast ?? combinedParameters.goFast ?? true;
|
|
106
|
+
const guidance = combinedParameters.guidance ?? 4;
|
|
107
|
+
const strength = combinedParameters.strength ?? 0.9;
|
|
108
|
+
const numOutputs = combinedParameters.num_outputs ?? combinedParameters.numberResults;
|
|
109
|
+
const disableSafetyChecker = combinedParameters.disable_safety_checker ?? combinedParameters.disableSafetyChecker ?? false;
|
|
110
|
+
|
|
111
|
+
requestParameters = {
|
|
112
|
+
input: {
|
|
113
|
+
prompt: modelPromptText,
|
|
114
|
+
go_fast: goFast,
|
|
115
|
+
guidance,
|
|
116
|
+
strength,
|
|
117
|
+
image_size: imageSize,
|
|
118
|
+
lora_scale: loraScale,
|
|
119
|
+
aspect_ratio: aspectRatio,
|
|
120
|
+
output_format: outputFormat,
|
|
121
|
+
enhance_prompt: enhancePrompt,
|
|
122
|
+
output_quality: outputQuality,
|
|
123
|
+
negative_prompt: negativePrompt,
|
|
124
|
+
num_inference_steps: numInferenceSteps,
|
|
125
|
+
disable_safety_checker: disableSafetyChecker,
|
|
126
|
+
...(numOutputs ? { num_outputs: numOutputs } : {}),
|
|
127
|
+
...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) ? { seed: combinedParameters.seed } : {}),
|
|
128
|
+
...(combinedParameters.image ? { image: combinedParameters.image } : {}),
|
|
129
|
+
...(combinedParameters.input_image ? { input_image: combinedParameters.input_image } : {}),
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
case "replicate-qwen-image-edit-plus": {
|
|
135
|
+
const aspectRatio = combinedParameters.aspect_ratio ?? combinedParameters.aspectRatio ?? "match_input_image";
|
|
136
|
+
const outputFormat = combinedParameters.output_format ?? combinedParameters.outputFormat ?? "webp";
|
|
137
|
+
const outputQuality = combinedParameters.output_quality ?? combinedParameters.outputQuality ?? 95;
|
|
138
|
+
const goFast = combinedParameters.go_fast ?? combinedParameters.goFast ?? true;
|
|
139
|
+
const disableSafetyChecker = combinedParameters.disable_safety_checker ?? combinedParameters.disableSafetyChecker ?? false;
|
|
140
|
+
|
|
141
|
+
const collectImages = (candidate, accumulator) => {
|
|
142
|
+
if (!candidate) return;
|
|
143
|
+
if (Array.isArray(candidate)) {
|
|
144
|
+
candidate.forEach((item) => collectImages(item, accumulator));
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
accumulator.push(candidate);
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
const imageCandidates = [];
|
|
151
|
+
collectImages(combinedParameters.image, imageCandidates);
|
|
152
|
+
collectImages(combinedParameters.images, imageCandidates);
|
|
153
|
+
collectImages(combinedParameters.input_image, imageCandidates);
|
|
154
|
+
collectImages(combinedParameters.input_images, imageCandidates);
|
|
155
|
+
collectImages(combinedParameters.input_image_1, imageCandidates);
|
|
156
|
+
collectImages(combinedParameters.input_image_2, imageCandidates);
|
|
157
|
+
collectImages(combinedParameters.input_image_3, imageCandidates);
|
|
158
|
+
collectImages(combinedParameters.image_1, imageCandidates);
|
|
159
|
+
collectImages(combinedParameters.image_2, imageCandidates);
|
|
160
|
+
|
|
161
|
+
const normalizeImageEntry = (entry) => {
|
|
162
|
+
if (!entry) return null;
|
|
163
|
+
if (typeof entry === "string") {
|
|
164
|
+
return entry; // Return the URL string directly
|
|
165
|
+
}
|
|
166
|
+
if (typeof entry === "object") {
|
|
167
|
+
if (Array.isArray(entry)) {
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
if (entry.value) {
|
|
171
|
+
return entry.value; // Return the value as a string
|
|
172
|
+
}
|
|
173
|
+
if (entry.url) {
|
|
174
|
+
return entry.url; // Return the URL as a string
|
|
175
|
+
}
|
|
176
|
+
if (entry.path) {
|
|
177
|
+
return entry.path; // Return the path as a string
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return null;
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
const normalizedImages = imageCandidates
|
|
184
|
+
.map((candidate) => normalizeImageEntry(candidate))
|
|
185
|
+
.filter((candidate) => candidate && typeof candidate === 'string');
|
|
186
|
+
|
|
187
|
+
const omitUndefined = (obj) =>
|
|
188
|
+
Object.fromEntries(
|
|
189
|
+
Object.entries(obj).filter(([, value]) => value !== undefined && value !== null),
|
|
190
|
+
);
|
|
191
|
+
|
|
192
|
+
const basePayload = omitUndefined({
|
|
193
|
+
prompt: modelPromptText,
|
|
194
|
+
go_fast: goFast,
|
|
195
|
+
aspect_ratio: aspectRatio,
|
|
196
|
+
output_format: outputFormat,
|
|
197
|
+
output_quality: outputQuality,
|
|
198
|
+
disable_safety_checker: disableSafetyChecker,
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
// For qwen-image-edit-plus, always include the image array if we have images
|
|
202
|
+
const inputPayload = {
|
|
203
|
+
...basePayload,
|
|
204
|
+
...(normalizedImages.length > 0 ? { image: normalizedImages } : {})
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
requestParameters = {
|
|
208
|
+
input: inputPayload,
|
|
209
|
+
};
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
95
212
|
case "replicate-flux-kontext-pro":
|
|
96
213
|
case "replicate-flux-kontext-max": {
|
|
97
214
|
const validRatios = [
|
|
@@ -110,7 +227,7 @@ class ReplicateApiPlugin extends ModelPlugin {
|
|
|
110
227
|
input_image: combinedParameters.input_image,
|
|
111
228
|
aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
|
|
112
229
|
safety_tolerance: safetyTolerance,
|
|
113
|
-
...(combinedParameters.seed && Number.isInteger(combinedParameters.seed && combinedParameters.seed > 0
|
|
230
|
+
...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) && combinedParameters.seed > 0 ? { seed: combinedParameters.seed } : {}),
|
|
114
231
|
},
|
|
115
232
|
};
|
|
116
233
|
break;
|
|
@@ -133,7 +250,7 @@ class ReplicateApiPlugin extends ModelPlugin {
|
|
|
133
250
|
input_image_2: combinedParameters.input_image_2,
|
|
134
251
|
aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
|
|
135
252
|
safety_tolerance: safetyTolerance,
|
|
136
|
-
...(combinedParameters.seed && Number.isInteger(combinedParameters.seed && combinedParameters.seed > 0
|
|
253
|
+
...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) && combinedParameters.seed > 0 ? { seed: combinedParameters.seed } : {}),
|
|
137
254
|
},
|
|
138
255
|
};
|
|
139
256
|
break;
|
|
@@ -148,7 +265,7 @@ class ReplicateApiPlugin extends ModelPlugin {
|
|
|
148
265
|
prompt: modelPromptText,
|
|
149
266
|
resolution: validResolutions.includes(combinedParameters.resolution) ? combinedParameters.resolution : "1080p",
|
|
150
267
|
aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "16:9",
|
|
151
|
-
...(combinedParameters.seed && Number.isInteger(combinedParameters.seed && combinedParameters.seed > 0
|
|
268
|
+
...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) && combinedParameters.seed > 0 ? { seed: combinedParameters.seed } : {}),
|
|
152
269
|
fps: validFps.includes(combinedParameters.fps) ? combinedParameters.fps : 24,
|
|
153
270
|
camera_fixed: combinedParameters.camera_fixed || false,
|
|
154
271
|
duration: combinedParameters.duration || 5,
|
|
@@ -157,6 +274,86 @@ class ReplicateApiPlugin extends ModelPlugin {
|
|
|
157
274
|
};
|
|
158
275
|
break;
|
|
159
276
|
}
|
|
277
|
+
case "replicate-seedream-4": {
|
|
278
|
+
const validSizes = ["1K", "2K", "4K", "custom"];
|
|
279
|
+
const validRatios = ["1:1", "4:3", "3:4", "16:9", "9:16", "match_input_image"];
|
|
280
|
+
const validSequentialModes = ["disabled", "auto"];
|
|
281
|
+
|
|
282
|
+
// Collect input images from multiple parameter sources (same pattern as qwen-image-edit-plus)
|
|
283
|
+
const collectImages = (candidate, accumulator) => {
|
|
284
|
+
if (!candidate) return;
|
|
285
|
+
if (Array.isArray(candidate)) {
|
|
286
|
+
candidate.forEach((item) => collectImages(item, accumulator));
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
accumulator.push(candidate);
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
const imageCandidates = [];
|
|
293
|
+
collectImages(combinedParameters.image, imageCandidates);
|
|
294
|
+
collectImages(combinedParameters.images, imageCandidates);
|
|
295
|
+
collectImages(combinedParameters.input_image, imageCandidates);
|
|
296
|
+
collectImages(combinedParameters.input_images, imageCandidates);
|
|
297
|
+
collectImages(combinedParameters.input_image_1, imageCandidates);
|
|
298
|
+
collectImages(combinedParameters.input_image_2, imageCandidates);
|
|
299
|
+
collectImages(combinedParameters.input_image_3, imageCandidates);
|
|
300
|
+
collectImages(combinedParameters.image_1, imageCandidates);
|
|
301
|
+
collectImages(combinedParameters.image_2, imageCandidates);
|
|
302
|
+
collectImages(combinedParameters.imageInput, imageCandidates);
|
|
303
|
+
|
|
304
|
+
const normalizeImageEntry = (entry) => {
|
|
305
|
+
if (!entry) return null;
|
|
306
|
+
if (typeof entry === "string") {
|
|
307
|
+
return entry; // Return the URL string directly
|
|
308
|
+
}
|
|
309
|
+
if (typeof entry === "object") {
|
|
310
|
+
if (Array.isArray(entry)) {
|
|
311
|
+
return null;
|
|
312
|
+
}
|
|
313
|
+
if (entry.value) {
|
|
314
|
+
return entry.value; // Return the value as a string
|
|
315
|
+
}
|
|
316
|
+
if (entry.url) {
|
|
317
|
+
return entry.url; // Return the URL as a string
|
|
318
|
+
}
|
|
319
|
+
if (entry.path) {
|
|
320
|
+
return entry.path; // Return the path as a string
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
return null;
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
const normalizedImages = imageCandidates
|
|
327
|
+
.map((candidate) => normalizeImageEntry(candidate))
|
|
328
|
+
.filter((candidate) => candidate && typeof candidate === 'string');
|
|
329
|
+
|
|
330
|
+
const omitUndefined = (obj) =>
|
|
331
|
+
Object.fromEntries(
|
|
332
|
+
Object.entries(obj).filter(([, value]) => value !== undefined && value !== null),
|
|
333
|
+
);
|
|
334
|
+
|
|
335
|
+
const basePayload = omitUndefined({
|
|
336
|
+
prompt: modelPromptText,
|
|
337
|
+
size: validSizes.includes(combinedParameters.size) ? combinedParameters.size : "2K",
|
|
338
|
+
width: combinedParameters.width || 2048,
|
|
339
|
+
height: combinedParameters.height || 2048,
|
|
340
|
+
max_images: combinedParameters.maxImages || combinedParameters.numberResults || 1,
|
|
341
|
+
aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "4:3",
|
|
342
|
+
sequential_image_generation: validSequentialModes.includes(combinedParameters.sequentialImageGeneration) ? combinedParameters.sequentialImageGeneration : "disabled",
|
|
343
|
+
...(combinedParameters.seed && Number.isInteger(combinedParameters.seed) && combinedParameters.seed > 0 ? { seed: combinedParameters.seed } : {}),
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
// For seedream-4, include the image_input array if we have images
|
|
347
|
+
const inputPayload = {
|
|
348
|
+
...basePayload,
|
|
349
|
+
...(normalizedImages.length > 0 ? { image_input: normalizedImages } : {})
|
|
350
|
+
};
|
|
351
|
+
|
|
352
|
+
requestParameters = {
|
|
353
|
+
input: inputPayload,
|
|
354
|
+
};
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
160
357
|
}
|
|
161
358
|
|
|
162
359
|
return requestParameters;
|
|
@@ -174,12 +371,14 @@ class ReplicateApiPlugin extends ModelPlugin {
|
|
|
174
371
|
cortexRequest.params = requestParameters.params;
|
|
175
372
|
|
|
176
373
|
// Make initial request to start prediction
|
|
177
|
-
const
|
|
178
|
-
|
|
374
|
+
const response = await this.executeRequest(cortexRequest);
|
|
375
|
+
|
|
376
|
+
// Parse the response to get the actual Replicate data
|
|
377
|
+
const parsedResponse = JSON.parse(response.output_text);
|
|
179
378
|
|
|
180
|
-
// If we got a completed response, return it
|
|
379
|
+
// If we got a completed response, return it as CortexResponse
|
|
181
380
|
if (parsedResponse?.status === "succeeded") {
|
|
182
|
-
return
|
|
381
|
+
return this.createCortexResponse(response);
|
|
183
382
|
}
|
|
184
383
|
|
|
185
384
|
logger.info("Replicate API returned a non-completed response.");
|
|
@@ -211,7 +410,9 @@ class ReplicateApiPlugin extends ModelPlugin {
|
|
|
211
410
|
|
|
212
411
|
if (status === "succeeded") {
|
|
213
412
|
logger.info("Replicate API returned a completed response after polling");
|
|
214
|
-
|
|
413
|
+
// Parse the polled response to extract artifacts
|
|
414
|
+
const parsedResponse = this.parseResponse(pollResponse.data);
|
|
415
|
+
return this.createCortexResponse(parsedResponse);
|
|
215
416
|
} else if (status === "failed" || status === "canceled") {
|
|
216
417
|
throw new Error(`Prediction ${status}: ${pollResponse.data?.error || "Unknown error"}`);
|
|
217
418
|
}
|
|
@@ -227,12 +428,67 @@ class ReplicateApiPlugin extends ModelPlugin {
|
|
|
227
428
|
throw new Error(`Prediction ${predictionId} timed out after ${maxAttempts * pollInterval / 1000} seconds`);
|
|
228
429
|
}
|
|
229
430
|
|
|
230
|
-
//
|
|
431
|
+
// Parse the response from the Replicate API and extract image artifacts
|
|
231
432
|
parseResponse(data) {
|
|
232
|
-
|
|
233
|
-
|
|
433
|
+
const responseData = data.data || data;
|
|
434
|
+
const stringifiedResponse = JSON.stringify(responseData);
|
|
435
|
+
|
|
436
|
+
// Extract image URLs from Replicate response for artifacts
|
|
437
|
+
const imageArtifacts = [];
|
|
438
|
+
if (responseData?.output && Array.isArray(responseData.output)) {
|
|
439
|
+
for (const outputItem of responseData.output) {
|
|
440
|
+
if (typeof outputItem === 'string' && outputItem.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
|
|
441
|
+
// This is an image URL from Replicate
|
|
442
|
+
imageArtifacts.push({
|
|
443
|
+
type: "image",
|
|
444
|
+
url: outputItem,
|
|
445
|
+
mimeType: this.getMimeTypeFromUrl(outputItem)
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
return {
|
|
452
|
+
output_text: stringifiedResponse,
|
|
453
|
+
artifacts: imageArtifacts
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Create a CortexResponse from parsed response data
|
|
458
|
+
createCortexResponse(parsedResponse) {
|
|
459
|
+
if (typeof parsedResponse === 'string') {
|
|
460
|
+
// Handle string response (backward compatibility)
|
|
461
|
+
return new CortexResponse({
|
|
462
|
+
output_text: parsedResponse,
|
|
463
|
+
artifacts: []
|
|
464
|
+
});
|
|
465
|
+
} else if (parsedResponse && typeof parsedResponse === 'object') {
|
|
466
|
+
// Handle object response with artifacts
|
|
467
|
+
return new CortexResponse({
|
|
468
|
+
output_text: parsedResponse.output_text,
|
|
469
|
+
artifacts: parsedResponse.artifacts || []
|
|
470
|
+
});
|
|
471
|
+
} else {
|
|
472
|
+
throw new Error('Unexpected response format');
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
// Helper method to determine MIME type from URL extension
|
|
477
|
+
getMimeTypeFromUrl(url) {
|
|
478
|
+
const extension = url.split('.').pop().toLowerCase();
|
|
479
|
+
switch (extension) {
|
|
480
|
+
case 'jpg':
|
|
481
|
+
case 'jpeg':
|
|
482
|
+
return 'image/jpeg';
|
|
483
|
+
case 'png':
|
|
484
|
+
return 'image/png';
|
|
485
|
+
case 'gif':
|
|
486
|
+
return 'image/gif';
|
|
487
|
+
case 'webp':
|
|
488
|
+
return 'image/webp';
|
|
489
|
+
default:
|
|
490
|
+
return 'image/jpeg'; // Default fallback
|
|
234
491
|
}
|
|
235
|
-
return JSON.stringify(data);
|
|
236
492
|
}
|
|
237
493
|
|
|
238
494
|
// Override the logging function to display the request and response
|
|
@@ -19,7 +19,8 @@ class VeoVideoPlugin extends ModelPlugin {
|
|
|
19
19
|
// Available Veo models
|
|
20
20
|
const availableModels = {
|
|
21
21
|
'veo-2.0-generate': 'GA',
|
|
22
|
-
'veo-3.0-generate': 'Preview'
|
|
22
|
+
'veo-3.0-generate': 'Preview',
|
|
23
|
+
'veo-3.0-fast-generate': 'Preview'
|
|
23
24
|
};
|
|
24
25
|
|
|
25
26
|
// Get the model ID from the pathway or use default
|
|
@@ -51,6 +52,7 @@ class VeoVideoPlugin extends ModelPlugin {
|
|
|
51
52
|
...(combinedParameters.enhancePrompt !== undefined && { enhancePrompt: combinedParameters.enhancePrompt }),
|
|
52
53
|
// generateAudio is required for 3.0 and not supported by 2.0
|
|
53
54
|
...(model === 'veo-3.0-generate' && { generateAudio: combinedParameters.generateAudio !== undefined ? combinedParameters.generateAudio : true }),
|
|
55
|
+
...(model === 'veo-3.0-fast-generate' && { generateAudio: combinedParameters.generateAudio !== undefined ? combinedParameters.generateAudio : true }),
|
|
54
56
|
...(combinedParameters.negativePrompt && { negativePrompt: combinedParameters.negativePrompt }),
|
|
55
57
|
...(combinedParameters.personGeneration && { personGeneration: combinedParameters.personGeneration }),
|
|
56
58
|
...(combinedParameters.sampleCount && { sampleCount: combinedParameters.sampleCount }),
|
|
@@ -69,6 +71,9 @@ class VeoVideoPlugin extends ModelPlugin {
|
|
|
69
71
|
if (model === 'veo-3.0-generate' && parameters.durationSeconds !== 8) {
|
|
70
72
|
throw new Error(`Veo 3.0 only supports durationSeconds: 8, got: ${parameters.durationSeconds}`);
|
|
71
73
|
}
|
|
74
|
+
if (model === 'veo-3.0-fast-generate' && parameters.durationSeconds !== 8) {
|
|
75
|
+
throw new Error(`Veo 3.0 only supports durationSeconds: 8, got: ${parameters.durationSeconds}`);
|
|
76
|
+
}
|
|
72
77
|
if (model === 'veo-2.0-generate' && (parameters.durationSeconds < 5 || parameters.durationSeconds > 8)) {
|
|
73
78
|
throw new Error(`Veo 2.0 supports durationSeconds between 5-8, got: ${parameters.durationSeconds}`);
|
|
74
79
|
}
|
|
@@ -82,6 +87,12 @@ class VeoVideoPlugin extends ModelPlugin {
|
|
|
82
87
|
if (parameters.video) {
|
|
83
88
|
throw new Error('video parameter is not supported in Veo 3.0');
|
|
84
89
|
}
|
|
90
|
+
if (model === 'veo-3.0-fast-generate' && parameters.lastFrame) {
|
|
91
|
+
throw new Error('lastFrame parameter is not supported in Veo 3.0');
|
|
92
|
+
}
|
|
93
|
+
if (model === 'veo-3.0-fast-generate' && parameters.video) {
|
|
94
|
+
throw new Error('video parameter is not supported in Veo 3.0');
|
|
95
|
+
}
|
|
85
96
|
}
|
|
86
97
|
|
|
87
98
|
// generateAudio constraints
|
|
@@ -91,6 +102,9 @@ class VeoVideoPlugin extends ModelPlugin {
|
|
|
91
102
|
if (model === 'veo-3.0-generate' && parameters.generateAudio === undefined) {
|
|
92
103
|
logger.warn('generateAudio is required for Veo 3.0, defaulting to true');
|
|
93
104
|
}
|
|
105
|
+
if (model === 'veo-3.0-fast-generate' && parameters.generateAudio === undefined) {
|
|
106
|
+
logger.warn('generateAudio is required for Veo 3.0, defaulting to true');
|
|
107
|
+
}
|
|
94
108
|
}
|
|
95
109
|
|
|
96
110
|
// Execute the request to the Veo API
|
package/server/rest.js
CHANGED
|
@@ -139,6 +139,8 @@ const processRestRequest = async (server, req, pathway, name, parameterMap = {})
|
|
|
139
139
|
return Boolean(value);
|
|
140
140
|
} else if (type === 'Int') {
|
|
141
141
|
return parseInt(value, 10);
|
|
142
|
+
} else if (type === 'Float') {
|
|
143
|
+
return parseFloat(value);
|
|
142
144
|
} else if (type === '[MultiMessage]' && Array.isArray(value)) {
|
|
143
145
|
return value.map(msg => ({
|
|
144
146
|
...msg,
|