@aj-archipelago/cortex 1.3.67 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.js +27 -0
- package/helper-apps/cortex-doc-to-pdf/DocToPdfFunction/__init__.py +3 -0
- package/helper-apps/cortex-doc-to-pdf/DocToPdfFunction/function.json +20 -0
- package/helper-apps/cortex-doc-to-pdf/Dockerfile +46 -0
- package/helper-apps/cortex-doc-to-pdf/README.md +408 -0
- package/helper-apps/cortex-doc-to-pdf/converter.py +157 -0
- package/helper-apps/cortex-doc-to-pdf/docker-compose.yml +23 -0
- package/helper-apps/cortex-doc-to-pdf/document_converter.py +181 -0
- package/helper-apps/cortex-doc-to-pdf/examples/README.md +252 -0
- package/helper-apps/cortex-doc-to-pdf/examples/nodejs-client.js +266 -0
- package/helper-apps/cortex-doc-to-pdf/examples/package-lock.json +297 -0
- package/helper-apps/cortex-doc-to-pdf/examples/package.json +23 -0
- package/helper-apps/cortex-doc-to-pdf/function_app.py +85 -0
- package/helper-apps/cortex-doc-to-pdf/host.json +16 -0
- package/helper-apps/cortex-doc-to-pdf/request_handlers.py +193 -0
- package/helper-apps/cortex-doc-to-pdf/requirements.txt +3 -0
- package/helper-apps/cortex-doc-to-pdf/tests/run_tests.sh +26 -0
- package/helper-apps/cortex-doc-to-pdf/tests/test_conversion.py +320 -0
- package/helper-apps/cortex-doc-to-pdf/tests/test_streaming.py +419 -0
- package/helper-apps/cortex-file-handler/package-lock.json +1 -0
- package/helper-apps/cortex-file-handler/package.json +1 -0
- package/helper-apps/cortex-file-handler/src/services/ConversionService.js +81 -8
- package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +54 -7
- package/helper-apps/cortex-file-handler/tests/getOperations.test.js +19 -7
- package/lib/encodeCache.js +5 -0
- package/lib/keyValueStorageClient.js +5 -0
- package/lib/logger.js +1 -1
- package/lib/pathwayTools.js +8 -1
- package/lib/redisSubscription.js +6 -0
- package/lib/requestExecutor.js +4 -0
- package/lib/util.js +88 -0
- package/package.json +1 -1
- package/pathways/basePathway.js +3 -3
- package/pathways/bing_afagent.js +1 -0
- package/pathways/gemini_15_vision.js +1 -1
- package/pathways/google_cse.js +2 -2
- package/pathways/image_gemini_25.js +85 -0
- package/pathways/image_prompt_optimizer_gemini_25.js +149 -0
- package/pathways/image_qwen.js +28 -0
- package/pathways/image_seedream4.js +26 -0
- package/pathways/rag.js +1 -1
- package/pathways/rag_jarvis.js +1 -1
- package/pathways/system/entity/sys_entity_continue.js +1 -1
- package/pathways/system/entity/sys_generator_results.js +1 -1
- package/pathways/system/entity/tools/sys_tool_google_search.js +15 -2
- package/pathways/system/entity/tools/sys_tool_grok_x_search.js +3 -3
- package/pathways/system/entity/tools/sys_tool_image.js +28 -23
- package/pathways/system/entity/tools/sys_tool_image_gemini.js +135 -0
- package/server/graphql.js +9 -2
- package/server/modelExecutor.js +4 -0
- package/server/pathwayResolver.js +19 -18
- package/server/plugins/claude3VertexPlugin.js +13 -8
- package/server/plugins/gemini15ChatPlugin.js +15 -10
- package/server/plugins/gemini15VisionPlugin.js +2 -23
- package/server/plugins/gemini25ImagePlugin.js +155 -0
- package/server/plugins/modelPlugin.js +3 -2
- package/server/plugins/openAiChatPlugin.js +6 -6
- package/server/plugins/replicateApiPlugin.js +268 -12
- package/server/plugins/veoVideoPlugin.js +15 -1
- package/server/rest.js +2 -0
- package/server/typeDef.js +96 -10
- package/tests/integration/apptekTranslatePlugin.integration.test.js +1 -1
- package/tests/unit/core/pathwayManager.test.js +2 -4
- package/tests/unit/plugins/gemini25ImagePlugin.test.js +294 -0
|
@@ -232,20 +232,32 @@ test.serial("should fetch remote file", async (t) => {
|
|
|
232
232
|
const requestId = uuidv4();
|
|
233
233
|
const remoteUrl = "https://example.com/test.txt";
|
|
234
234
|
|
|
235
|
+
// Mock external HEAD and GET to avoid network dependency
|
|
236
|
+
const body = "hello from example";
|
|
237
|
+
const scope = nock("https://example.com")
|
|
238
|
+
.head("/test.txt")
|
|
239
|
+
.reply(200, "", {
|
|
240
|
+
"Content-Type": "text/plain",
|
|
241
|
+
"Content-Length": body.length.toString(),
|
|
242
|
+
})
|
|
243
|
+
.get("/test.txt")
|
|
244
|
+
.reply(200, body, {
|
|
245
|
+
"Content-Type": "text/plain",
|
|
246
|
+
"Content-Length": body.length.toString(),
|
|
247
|
+
});
|
|
248
|
+
|
|
235
249
|
const response = await axios.get(baseUrl, {
|
|
236
250
|
params: {
|
|
237
251
|
fetch: remoteUrl,
|
|
238
252
|
requestId,
|
|
239
253
|
},
|
|
240
254
|
validateStatus: (status) => true,
|
|
255
|
+
timeout: 10000,
|
|
241
256
|
});
|
|
242
257
|
|
|
243
|
-
t.is(response.status,
|
|
244
|
-
t.
|
|
245
|
-
|
|
246
|
-
"Invalid or inaccessible URL",
|
|
247
|
-
"Should return correct error message",
|
|
248
|
-
);
|
|
258
|
+
t.is(response.status, 200, "Should fetch and store remote file");
|
|
259
|
+
t.truthy(response.data.url, "Should return file URL");
|
|
260
|
+
t.true(scope.isDone(), "All external requests should be mocked and used");
|
|
249
261
|
});
|
|
250
262
|
|
|
251
263
|
// Test: Redis caching behavior for remote files
|
|
@@ -296,7 +308,7 @@ test.serial("should cache remote files in Redis", async (t) => {
|
|
|
296
308
|
|
|
297
309
|
t.is(secondResponse.status, 200, "Should return cached file from Redis");
|
|
298
310
|
t.truthy(secondResponse.data.url, "Should return cached file URL");
|
|
299
|
-
|
|
311
|
+
|
|
300
312
|
// Cleanup
|
|
301
313
|
await cleanupHashAndFile(hash, secondResponse.data.url, baseUrl);
|
|
302
314
|
} finally {
|
package/lib/encodeCache.js
CHANGED
|
@@ -9,6 +9,11 @@ class EncodeCache {
|
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
encode(value) {
|
|
12
|
+
// Handle CortexResponse objects by extracting the text content
|
|
13
|
+
if (value && typeof value === 'object' && value.constructor && value.constructor.name === 'CortexResponse') {
|
|
14
|
+
value = value.output_text || '';
|
|
15
|
+
}
|
|
16
|
+
|
|
12
17
|
if (this.encodeCache.get(value) !== -1) {
|
|
13
18
|
return this.encodeCache.get(value);
|
|
14
19
|
}
|
|
@@ -30,6 +30,11 @@ const keyValueStorageClient = new Keyv(storageConnectionString, {
|
|
|
30
30
|
namespace: `${cortexId}-cortex-context`
|
|
31
31
|
});
|
|
32
32
|
|
|
33
|
+
// Handle Redis connection errors to prevent crashes
|
|
34
|
+
keyValueStorageClient.on('error', (error) => {
|
|
35
|
+
logger.error(`Keyv Redis connection error: ${error}`);
|
|
36
|
+
});
|
|
37
|
+
|
|
33
38
|
// Set values to keyv
|
|
34
39
|
async function setv(key, value) {
|
|
35
40
|
return keyValueStorageClient && (await keyValueStorageClient.set(key, value));
|
package/lib/logger.js
CHANGED
|
@@ -37,9 +37,9 @@ const getTransport = () => {
|
|
|
37
37
|
case 'production':
|
|
38
38
|
return new winston.transports.Console({ level: 'info', format: winston.format.combine(suppressNonErrorFormat(), prodFormat) });
|
|
39
39
|
case 'development':
|
|
40
|
-
case 'test':
|
|
41
40
|
return new winston.transports.Console({ level: 'verbose', format: winston.format.combine(suppressNonErrorFormat(), debugFormat) });
|
|
42
41
|
case 'debug':
|
|
42
|
+
case 'test':
|
|
43
43
|
return new winston.transports.Console({ level: 'debug', format: winston.format.combine(suppressNonErrorFormat(), debugFormat) });
|
|
44
44
|
default:
|
|
45
45
|
// Default to development settings if NODE_ENV is not set or unknown
|
package/lib/pathwayTools.js
CHANGED
|
@@ -5,6 +5,7 @@ import { publishRequestProgress } from "../lib/redisSubscription.js";
|
|
|
5
5
|
import { getSemanticChunks } from "../server/chunker.js";
|
|
6
6
|
import logger from '../lib/logger.js';
|
|
7
7
|
import { requestState } from '../server/requestState.js';
|
|
8
|
+
import { processPathwayParameters } from '../server/typeDef.js';
|
|
8
9
|
|
|
9
10
|
// callPathway - call a pathway from another pathway
|
|
10
11
|
const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
|
|
@@ -17,12 +18,18 @@ const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
|
|
|
17
18
|
throw new Error(`Pathway ${pathwayName} not found`);
|
|
18
19
|
}
|
|
19
20
|
|
|
21
|
+
// Merge pathway default parameters with input args, similar to GraphQL typeDef behavior
|
|
22
|
+
const mergedParams = { ...pathway.defaultInputParameters, ...pathway.inputParameters, ...args };
|
|
23
|
+
|
|
24
|
+
// Process the merged parameters to convert type specification objects to actual values
|
|
25
|
+
const processedArgs = processPathwayParameters(mergedParams);
|
|
26
|
+
|
|
20
27
|
const parent = {};
|
|
21
28
|
let rootRequestId = pathwayResolver?.rootRequestId || pathwayResolver?.requestId;
|
|
22
29
|
|
|
23
30
|
const contextValue = { config, pathway, requestState };
|
|
24
31
|
|
|
25
|
-
let data = await pathway.rootResolver(parent, {...
|
|
32
|
+
let data = await pathway.rootResolver(parent, {...processedArgs, rootRequestId}, contextValue );
|
|
26
33
|
|
|
27
34
|
if (pathwayResolver && contextValue.pathwayResolver) {
|
|
28
35
|
pathwayResolver.mergeResolver(contextValue.pathwayResolver);
|
package/lib/redisSubscription.js
CHANGED
|
@@ -24,6 +24,12 @@ if (connectionString) {
|
|
|
24
24
|
logger.info(`Using Redis publish for channel(s) ${requestProgressChannel}, ${requestProgressSubscriptionsChannel}`);
|
|
25
25
|
try {
|
|
26
26
|
publisherClient = connectionString && new Redis(connectionString);
|
|
27
|
+
// Handle Redis publisher client errors to prevent crashes
|
|
28
|
+
if (publisherClient) {
|
|
29
|
+
publisherClient.on('error', (error) => {
|
|
30
|
+
logger.error(`Redis publisherClient error: ${error}`);
|
|
31
|
+
});
|
|
32
|
+
}
|
|
27
33
|
} catch (error) {
|
|
28
34
|
logger.error(`Redis connection error: ${error}`);
|
|
29
35
|
}
|
package/lib/requestExecutor.js
CHANGED
|
@@ -20,6 +20,10 @@ let client;
|
|
|
20
20
|
if (connectionString) {
|
|
21
21
|
try {
|
|
22
22
|
client = new Redis(connectionString);
|
|
23
|
+
// Handle Redis connection errors to prevent crashes
|
|
24
|
+
client.on('error', (error) => {
|
|
25
|
+
logger.error(`Redis client connection error: ${error}`);
|
|
26
|
+
});
|
|
23
27
|
} catch (error) {
|
|
24
28
|
logger.error(`Redis connection error: ${error}`);
|
|
25
29
|
}
|
package/lib/util.js
CHANGED
|
@@ -10,6 +10,8 @@ import { promisify } from 'util';
|
|
|
10
10
|
import { axios } from './requestExecutor.js';
|
|
11
11
|
import { config } from '../config.js';
|
|
12
12
|
import fs from 'fs';
|
|
13
|
+
import path from 'path';
|
|
14
|
+
import FormData from 'form-data';
|
|
13
15
|
|
|
14
16
|
const pipeline = promisify(stream.pipeline);
|
|
15
17
|
const MEDIA_API_URL = config.get('whisperMediaApiUrl');
|
|
@@ -412,6 +414,91 @@ function getAvailableFiles(chatHistory) {
|
|
|
412
414
|
return availableFiles;
|
|
413
415
|
}
|
|
414
416
|
|
|
417
|
+
// Helper function to upload base64 image data to cloud storage
|
|
418
|
+
const uploadImageToCloud = async (base64Data, mimeType, pathwayResolver = null) => {
|
|
419
|
+
let tempFilePath = null;
|
|
420
|
+
let tempDir = null;
|
|
421
|
+
|
|
422
|
+
try {
|
|
423
|
+
// Convert base64 to buffer
|
|
424
|
+
const imageBuffer = Buffer.from(base64Data, 'base64');
|
|
425
|
+
|
|
426
|
+
// Determine file extension from mime type
|
|
427
|
+
const extension = mimeType.split('/')[1] || 'png';
|
|
428
|
+
const filename = `generated_image_${Date.now()}.${extension}`;
|
|
429
|
+
|
|
430
|
+
// Create temporary file
|
|
431
|
+
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'image-upload-'));
|
|
432
|
+
tempFilePath = path.join(tempDir, filename);
|
|
433
|
+
|
|
434
|
+
// Write buffer to temp file
|
|
435
|
+
fs.writeFileSync(tempFilePath, imageBuffer);
|
|
436
|
+
|
|
437
|
+
// Upload to file handler service
|
|
438
|
+
const fileHandlerUrl = MEDIA_API_URL;
|
|
439
|
+
if (!fileHandlerUrl) {
|
|
440
|
+
throw new Error('WHISPER_MEDIA_API_URL is not set');
|
|
441
|
+
}
|
|
442
|
+
const requestId = uuidv4();
|
|
443
|
+
|
|
444
|
+
// Create form data for upload
|
|
445
|
+
const formData = new FormData();
|
|
446
|
+
formData.append('file', fs.createReadStream(tempFilePath), {
|
|
447
|
+
filename: filename,
|
|
448
|
+
contentType: mimeType
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
// Upload file
|
|
452
|
+
const uploadResponse = await axios.post(`${fileHandlerUrl}?requestId=${requestId}`, formData, {
|
|
453
|
+
headers: {
|
|
454
|
+
...formData.getHeaders()
|
|
455
|
+
},
|
|
456
|
+
timeout: 30000
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
if (uploadResponse.data && uploadResponse.data.url) {
|
|
460
|
+
return uploadResponse.data.url;
|
|
461
|
+
} else {
|
|
462
|
+
throw new Error('No URL returned from file handler');
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
} catch (error) {
|
|
466
|
+
const errorMessage = `Failed to upload image: ${error.message}`;
|
|
467
|
+
if (pathwayResolver && pathwayResolver.logError) {
|
|
468
|
+
pathwayResolver.logError(errorMessage);
|
|
469
|
+
} else {
|
|
470
|
+
logger.error(errorMessage);
|
|
471
|
+
}
|
|
472
|
+
throw error;
|
|
473
|
+
} finally {
|
|
474
|
+
// Clean up temp files
|
|
475
|
+
if (tempFilePath && fs.existsSync(tempFilePath)) {
|
|
476
|
+
try {
|
|
477
|
+
fs.unlinkSync(tempFilePath);
|
|
478
|
+
} catch (cleanupError) {
|
|
479
|
+
const warningMessage = `Failed to clean up temp file: ${cleanupError.message}`;
|
|
480
|
+
if (pathwayResolver && pathwayResolver.logWarning) {
|
|
481
|
+
pathwayResolver.logWarning(warningMessage);
|
|
482
|
+
} else {
|
|
483
|
+
logger.warn(warningMessage);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
if (tempDir && fs.existsSync(tempDir)) {
|
|
488
|
+
try {
|
|
489
|
+
fs.rmdirSync(tempDir);
|
|
490
|
+
} catch (cleanupError) {
|
|
491
|
+
const warningMessage = `Failed to clean up temp directory: ${cleanupError.message}`;
|
|
492
|
+
if (pathwayResolver && pathwayResolver.logWarning) {
|
|
493
|
+
pathwayResolver.logWarning(warningMessage);
|
|
494
|
+
} else {
|
|
495
|
+
logger.warn(warningMessage);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
};
|
|
501
|
+
|
|
415
502
|
/**
|
|
416
503
|
* Convert file hashes to content format suitable for LLM processing
|
|
417
504
|
* @param {Array<string>} fileHashes - Array of file hashes to resolve
|
|
@@ -476,6 +563,7 @@ export {
|
|
|
476
563
|
chatArgsHasType,
|
|
477
564
|
deleteTempPath,
|
|
478
565
|
downloadFile,
|
|
566
|
+
uploadImageToCloud,
|
|
479
567
|
convertSrtToText,
|
|
480
568
|
alignSubtitles,
|
|
481
569
|
getMediaChunks,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aj-archipelago/cortex",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
|
|
5
5
|
"private": false,
|
|
6
6
|
"repository": {
|
package/pathways/basePathway.js
CHANGED
|
@@ -3,11 +3,11 @@ import { typeDef } from '../server/typeDef.js';
|
|
|
3
3
|
|
|
4
4
|
// all default definitions of a single pathway
|
|
5
5
|
export default {
|
|
6
|
-
prompt:
|
|
6
|
+
prompt: '{{text}}',
|
|
7
7
|
defaultInputParameters: {
|
|
8
|
-
text:
|
|
8
|
+
text: '',
|
|
9
9
|
async: false, // switch to enable async mode
|
|
10
|
-
contextId:
|
|
10
|
+
contextId: '', // used to identify the context of the request,
|
|
11
11
|
stream: false, // switch to enable stream mode
|
|
12
12
|
},
|
|
13
13
|
inputParameters: {},
|
package/pathways/bing_afagent.js
CHANGED
package/pathways/google_cse.js
CHANGED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { Prompt } from '../server/prompt.js';
|
|
2
|
+
import { callPathway } from '../lib/pathwayTools.js';
|
|
3
|
+
import logger from '../lib/logger.js';
|
|
4
|
+
|
|
5
|
+
export default {
|
|
6
|
+
prompt: [],
|
|
7
|
+
executePathway: async ({args, runAllPrompts, resolver}) => {
|
|
8
|
+
let finalPrompt = args.text || '';
|
|
9
|
+
|
|
10
|
+
const { optimizePrompt, input_image, input_image_2, input_image_3 } = { ...resolver.pathway.inputParameters, ...args };
|
|
11
|
+
|
|
12
|
+
// Check if prompt optimization is enabled
|
|
13
|
+
if (optimizePrompt && optimizePrompt !== false && finalPrompt) {
|
|
14
|
+
try {
|
|
15
|
+
// Call the prompt optimizer pathway
|
|
16
|
+
const optimizerResult = await callPathway('image_prompt_optimizer_gemini_25', {
|
|
17
|
+
userPrompt: finalPrompt,
|
|
18
|
+
hasInputImages: !!input_image || !!input_image_2 || !!input_image_3
|
|
19
|
+
}, resolver);
|
|
20
|
+
|
|
21
|
+
if (optimizerResult) {
|
|
22
|
+
finalPrompt = optimizerResult;
|
|
23
|
+
}
|
|
24
|
+
} catch (error) {
|
|
25
|
+
logger.warn(`Prompt optimization failed, proceeding with original prompt: ${error.message}`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Build the user content with text and images
|
|
30
|
+
const userContent = [{"type": "text", "text": finalPrompt}];
|
|
31
|
+
|
|
32
|
+
// Add input images if provided
|
|
33
|
+
if (input_image) {
|
|
34
|
+
userContent.push({
|
|
35
|
+
"type": "image_url",
|
|
36
|
+
"image_url": {"url": input_image}
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
if (input_image_2) {
|
|
40
|
+
userContent.push({
|
|
41
|
+
"type": "image_url",
|
|
42
|
+
"image_url": {"url": input_image_2}
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
if (input_image_3) {
|
|
46
|
+
userContent.push({
|
|
47
|
+
"type": "image_url",
|
|
48
|
+
"image_url": {"url": input_image_3}
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const userMessage = {"role": "user", "content": userContent};
|
|
53
|
+
|
|
54
|
+
const systemMessage = {"role": "system", "content": "Instructions:\nYou are Jarvis Vision 2.5, an AI entity working for a prestigious international news agency. Jarvis is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive. Your primary expertise is both image analysis and image generation/editing. You are capable of:\n\n1. Understanding and interpreting complex image data, identifying patterns and trends\n2. Generating new images based on detailed descriptions\n3. Editing existing images according to specific instructions\n4. Delivering insights and results in a clear, digestible format\n\nYou know the current date and time - it is {{now}}. When generating or editing images, ensure they are appropriate for professional news media use and follow ethical guidelines."};
|
|
55
|
+
|
|
56
|
+
const promptMessages = [systemMessage, userMessage];
|
|
57
|
+
|
|
58
|
+
resolver.pathwayPrompt = [
|
|
59
|
+
new Prompt({ messages: promptMessages }),
|
|
60
|
+
];
|
|
61
|
+
|
|
62
|
+
return await runAllPrompts({ ...args });
|
|
63
|
+
},
|
|
64
|
+
|
|
65
|
+
inputParameters: {
|
|
66
|
+
text: "",
|
|
67
|
+
input_image: "", // URL to first input image
|
|
68
|
+
input_image_2: "", // URL to second input image
|
|
69
|
+
input_image_3: "", // URL to third input image
|
|
70
|
+
contextId: ``,
|
|
71
|
+
response_modalities: ["TEXT", "IMAGE"],
|
|
72
|
+
optimizePrompt: false, // Enable prompt optimization using Google's best practices
|
|
73
|
+
},
|
|
74
|
+
max_tokens: 32000,
|
|
75
|
+
model: 'gemini-25-flash-image',
|
|
76
|
+
useInputChunking: false,
|
|
77
|
+
enableDuplicateRequests: false,
|
|
78
|
+
timeout: 600,
|
|
79
|
+
geminiSafetySettings: [
|
|
80
|
+
{category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
|
|
81
|
+
{category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
|
|
82
|
+
{category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
|
|
83
|
+
{category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}
|
|
84
|
+
],
|
|
85
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { Prompt } from '../server/prompt.js';
|
|
2
|
+
|
|
3
|
+
export default {
|
|
4
|
+
prompt: [],
|
|
5
|
+
executePathway: async ({args, runAllPrompts, resolver}) => {
|
|
6
|
+
const { userPrompt, hasInputImages } = { ...resolver.pathway.inputParameters, ...args };
|
|
7
|
+
|
|
8
|
+
// Build the system prompt dynamically based on whether input images are provided
|
|
9
|
+
let systemContent = `You are an expert prompt optimizer for Gemini 2.5 image generation. Your job is to transform basic image generation requests into highly detailed, professional prompts that follow Google's official best practices and templates for optimal image generation.
|
|
10
|
+
|
|
11
|
+
## Core Optimization Principles (from Google's Official Guide):
|
|
12
|
+
|
|
13
|
+
### 1. **Be Hyper-Specific**
|
|
14
|
+
Transform vague descriptions into detailed, specific instructions:
|
|
15
|
+
- Instead of "fantasy armor" → "ornate elven plate armor, etched with silver leaf patterns, with a high collar and pauldrons shaped like falcon wings"
|
|
16
|
+
- Instead of "a cat" → "a majestic adult tabby cat with distinctive orange and black striped fur, sitting gracefully on a sunlit wooden windowsill"
|
|
17
|
+
- If the prompt refers to something already mentioned, it might be using an image as a precise reference, so don't re-describe it in the prompt.For example, if the prompt says "this cat", don't say "the black cat sitting on a sunlit wooden window sill", just say "this cat".
|
|
18
|
+
|
|
19
|
+
### 2. **Provide Context and Intent**
|
|
20
|
+
Always explain the purpose and context of the image:
|
|
21
|
+
- Add context like "for a high-end brand logo" or "for a professional presentation"
|
|
22
|
+
- Specify the intended use: "desktop wallpaper", "magazine editorial", "social media post", "product packaging"
|
|
23
|
+
|
|
24
|
+
### 3. **Use Step-by-Step Instructions**
|
|
25
|
+
Break complex scenes into clear steps:
|
|
26
|
+
- "First, create a background of a serene, misty forest at dawn. Then, in the foreground, add a moss-covered ancient stone altar. Finally, place a single, glowing sword on top of the altar."
|
|
27
|
+
|
|
28
|
+
### 4. **Use Semantic Positive Language**
|
|
29
|
+
Instead of negative prompts, describe the desired scene positively:
|
|
30
|
+
- Instead of "no cars" → "an empty, deserted street with no signs of traffic"
|
|
31
|
+
- Instead of "not dark" → "brightly lit with warm, golden lighting"
|
|
32
|
+
|
|
33
|
+
### 5. **Control the Camera**
|
|
34
|
+
Use photographic and cinematic language to control composition:
|
|
35
|
+
- Camera angles: "wide-angle shot", "macro shot", "low-angle perspective", "close-up", "bird's eye view"
|
|
36
|
+
- Composition: "rule of thirds", "centered composition", "dynamic diagonal lines"
|
|
37
|
+
|
|
38
|
+
## Official Gemini Templates (Use These When Applicable):
|
|
39
|
+
|
|
40
|
+
### **1. Photorealistic Scenes Template**
|
|
41
|
+
"A photorealistic [shot type] of [subject], [action or expression], set in [environment]. The scene is illuminated by [lighting description], creating a [mood] atmosphere. Captured with a [camera/lens details], emphasizing [key textures and details]. The image should be in a [aspect ratio] format."
|
|
42
|
+
|
|
43
|
+
### **2. Stylized Illustrations & Stickers Template**
|
|
44
|
+
"A [style] sticker of a [subject], featuring [key characteristics] and a [color palette]. The design should have [line style] and [shading style]. The background must be transparent."
|
|
45
|
+
|
|
46
|
+
### **3. Accurate Text in Images Template**
|
|
47
|
+
"Create a [image type] for [brand/concept] with the text '[text to render]' in a [font style]. The design should be [style description], with a [color scheme]."
|
|
48
|
+
|
|
49
|
+
### **4. Product Mockups & Commercial Photography Template**
|
|
50
|
+
"A high-resolution, studio-lit product photograph of a [product description] on a [background surface/description]. The lighting is a [lighting setup, e.g., three-point softbox setup] to [lighting purpose]. The camera angle is a [angle type] to showcase [specific feature]. Ultra-realistic, with sharp focus on [key detail]. [Aspect ratio]."
|
|
51
|
+
|
|
52
|
+
### **5. Minimalist & Negative Space Design Template**
|
|
53
|
+
"A minimalist composition featuring a single [subject] positioned in the [bottom-right/top-left/etc.] of the frame. The background is a vast, empty [color] canvas, creating significant negative space. Soft, subtle lighting. [Aspect ratio]."
|
|
54
|
+
|
|
55
|
+
### **6. Sequential Art (Comic Panel / Storyboard) Template**
|
|
56
|
+
"A single comic book panel in a [art style] style. In the foreground, [character description and action]. In the background, [setting details]. The panel has a [dialogue/caption box] with the text '[Text]'. The lighting creates a [mood] mood. [Aspect ratio]."
|
|
57
|
+
|
|
58
|
+
## Image Editing Templates (When Editing Images):
|
|
59
|
+
|
|
60
|
+
### **1. Adding and Removing Elements**
|
|
61
|
+
"Using the provided image of [subject], please [add/remove/modify] [element] to/from the scene. Ensure the change is [description of how the change should integrate]."
|
|
62
|
+
|
|
63
|
+
### **2. Inpainting (Semantic Masking)**
|
|
64
|
+
"Using the provided image, change only the [specific element] to [new element/description]. Keep everything else in the image exactly the same, preserving the original style, lighting, and composition."
|
|
65
|
+
|
|
66
|
+
### **3. Style Transfer**
|
|
67
|
+
"Transform the provided photograph of [subject] into the artistic style of [artist/art style]. Preserve the original composition but render it with [description of stylistic elements]."
|
|
68
|
+
|
|
69
|
+
### **4. Advanced Composition: Combining Multiple Images**
|
|
70
|
+
"Create a new image by combining the elements from the provided images. Take the [element from image 1] and place it with/on the [element from image 2]. The final image should be a [description of the final scene]."
|
|
71
|
+
|
|
72
|
+
### **5. High-Fidelity Detail Preservation**
|
|
73
|
+
"Using the provided images, place [element from image 2] onto [element from image 1]. Ensure that the features of [element from image 1] remain completely unchanged. The added element should [description of how the element should integrate]."
|
|
74
|
+
|
|
75
|
+
## Professional Strategies:
|
|
76
|
+
|
|
77
|
+
### **Iterate and Refine**
|
|
78
|
+
Don't expect a perfect image on the first try. Use the conversational nature of the model to make small changes. Follow up with prompts like, "That's great, but can you make the lighting a bit warmer?" or "Keep everything the same, but change the character's expression to be more serious."
|
|
79
|
+
|
|
80
|
+
### **Enhance Visual Details**
|
|
81
|
+
Add comprehensive visual specifications:
|
|
82
|
+
- **Lighting**: "golden hour", "dramatic shadows", "soft lighting", "studio lighting", "natural daylight"
|
|
83
|
+
- **Mood**: "serene", "energetic", "mysterious", "tranquil", "dramatic", "playful"
|
|
84
|
+
- **Style**: "photorealistic", "artistic", "minimalist", "vintage", "modern", "cinematic"
|
|
85
|
+
- **Composition**: "rule of thirds", "centered", "dynamic", "balanced", "asymmetrical"
|
|
86
|
+
|
|
87
|
+
### **Professional Quality Standards**
|
|
88
|
+
- Specify resolution and quality: "high-resolution", "4K quality", "professional grade"
|
|
89
|
+
- Add technical details: "sharp focus", "shallow depth of field", "high contrast"
|
|
90
|
+
- Include professional context: "magazine cover quality", "advertising standard", "editorial photography"
|
|
91
|
+
|
|
92
|
+
## Example Transformations Using Templates:
|
|
93
|
+
|
|
94
|
+
**Basic Input**: "Generate an image of a cat"
|
|
95
|
+
**Template-Based Output**: "A photorealistic close-up portrait of a regal adult tabby cat with vividly detailed orange and black striped fur, sitting gracefully with its tail curled around its paws, set in a sun-drenched wooden windowsill overlooking a lush garden. The scene is illuminated by soft, natural morning light streaming through the window, creating a serene and tranquil atmosphere. Captured with a macro lens emphasizing the intricate fur patterns, expressive green eyes, and delicate whiskers. The image should be in a 16:9 aspect ratio format, perfect for desktop wallpaper or magazine editorial use."
|
|
96
|
+
|
|
97
|
+
**Basic Input**: "Make a logo for my company"
|
|
98
|
+
**Template-Based Output**: "Create a modern, minimalist logo for a technology startup called 'InnovateTech' with the text 'InnovateTech' in a clean, bold sans-serif font. The design should be sleek and professional, with a cool color scheme featuring deep blue and silver accents. The logo should feature a geometric icon representing innovation and connectivity, positioned to the left of the company name. The background must be transparent for versatile use across digital and print media."
|
|
99
|
+
|
|
100
|
+
## Input Format:
|
|
101
|
+
The user will provide a simple image generation request.
|
|
102
|
+
|
|
103
|
+
## Output Format:
|
|
104
|
+
Return ONLY the optimized prompt, ready to use with Gemini 2.5. Use the appropriate template when applicable, or create a detailed, specific prompt following the core principles. Make it professional and comprehensive.
|
|
105
|
+
|
|
106
|
+
Current date and time: {{now}}`;
|
|
107
|
+
|
|
108
|
+
// Add specific instructions for input images if they are provided
|
|
109
|
+
if (hasInputImages) {
|
|
110
|
+
systemContent += `
|
|
111
|
+
|
|
112
|
+
## IMPORTANT: Input Images Context
|
|
113
|
+
The user has provided input images that should be used as reference or source material for the image generation. When optimizing the prompt, make it explicit about using these input images:
|
|
114
|
+
|
|
115
|
+
- Reference the input images directly in your optimized prompt
|
|
116
|
+
- Use phrases like "using the provided input image(s)" or "based on the input image(s)"
|
|
117
|
+
- Specify how the input images should be incorporated (as reference, for style transfer, for composition, etc.)
|
|
118
|
+
- If the user's request is vague about how to use the input images, make educated assumptions and be explicit about them
|
|
119
|
+
- Use the Image Editing Templates above when appropriate for input image scenarios
|
|
120
|
+
|
|
121
|
+
Examples of how to reference input images:
|
|
122
|
+
- "Using the provided input image as a reference, create a photorealistic portrait..."
|
|
123
|
+
- "Transform the provided input image into the artistic style of..."
|
|
124
|
+
- "Using the input images, combine the elements to create a new composition..."
|
|
125
|
+
- "Based on the input image, modify the scene to include..."`;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const promptMessages = [
|
|
129
|
+
{"role": "system", "content": systemContent},
|
|
130
|
+
{"role": "user", "content": userPrompt},
|
|
131
|
+
];
|
|
132
|
+
|
|
133
|
+
resolver.pathwayPrompt = [
|
|
134
|
+
new Prompt({ messages: promptMessages }),
|
|
135
|
+
];
|
|
136
|
+
|
|
137
|
+
return await runAllPrompts({ ...args });
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
inputParameters: {
|
|
141
|
+
userPrompt: ``,
|
|
142
|
+
hasInputImages: false,
|
|
143
|
+
},
|
|
144
|
+
max_tokens: 4096,
|
|
145
|
+
model: 'oai-gpt41',
|
|
146
|
+
useInputChunking: false,
|
|
147
|
+
enableDuplicateRequests: false,
|
|
148
|
+
timeout: 30,
|
|
149
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export default {
|
|
2
|
+
prompt: ["{{text}}"],
|
|
3
|
+
|
|
4
|
+
enableDuplicateRequests: false,
|
|
5
|
+
inputParameters: {
|
|
6
|
+
model: "replicate-qwen-image", // Options: "replicate-qwen-image" or "replicate-qwen-image-edit-plus"
|
|
7
|
+
negativePrompt: "",
|
|
8
|
+
width: 1024,
|
|
9
|
+
height: 1024,
|
|
10
|
+
aspectRatio: "16:9", // Options: "1:1", "16:9", "9:16", "4:3", "3:4", "match_input_image" (use "match_input_image" for qwen-image-edit-plus)
|
|
11
|
+
numberResults: 1,
|
|
12
|
+
output_format: "webp",
|
|
13
|
+
output_quality: 80, // Use 95 for qwen-image-edit-plus
|
|
14
|
+
input_image: "", // URL to input image for replicate-qwen-image-edit-plus
|
|
15
|
+
input_image_2: "", // URL to second input image for replicate-qwen-image-edit-plus
|
|
16
|
+
input_image_3: "", // URL to third input image for replicate-qwen-image-edit-plus
|
|
17
|
+
|
|
18
|
+
// Qwen-specific parameters
|
|
19
|
+
go_fast: true,
|
|
20
|
+
guidance: { type: 'number', default: 4.0 }, // For replicate-qwen-image only
|
|
21
|
+
strength: 0.9,
|
|
22
|
+
image_size: "optimize_for_quality", // For replicate-qwen-image only
|
|
23
|
+
lora_scale: { type: 'number', default: 1.0 }, // For replicate-qwen-image only
|
|
24
|
+
enhance_prompt: false, // For replicate-qwen-image only
|
|
25
|
+
num_inference_steps: 50, // For replicate-qwen-image only
|
|
26
|
+
disable_safety_checker: false,
|
|
27
|
+
},
|
|
28
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export default {
|
|
2
|
+
prompt: ["{{text}}"],
|
|
3
|
+
enableDuplicateRequests: false,
|
|
4
|
+
inputParameters: {
|
|
5
|
+
model: "replicate-seedream-4",
|
|
6
|
+
size: "2K", // Options: "1K", "2K", "4K", "custom"
|
|
7
|
+
width: 2048, // Custom width (1024-4096px, only used when size="custom")
|
|
8
|
+
height: 2048, // Custom height (1024-4096px, only used when size="custom")
|
|
9
|
+
aspectRatio: "4:3", // Options: "1:1", "4:3", "3:4", "16:9", "9:16", "match_input_image"
|
|
10
|
+
maxImages: 1, // Maximum number of images to generate (1-15)
|
|
11
|
+
numberResults: 1, // Alternative parameter name for maxImages
|
|
12
|
+
imageInput: [], // Array of input images (1-10 images for image-to-image generation)
|
|
13
|
+
// Multiple input image parameters (same pattern as qwen-image-edit-plus)
|
|
14
|
+
input_image: "", // Single input image URL
|
|
15
|
+
input_image_1: "", // First input image URL
|
|
16
|
+
input_image_2: "", // Second input image URL
|
|
17
|
+
input_image_3: "", // Third input image URL
|
|
18
|
+
image: "", // Alternative single image parameter
|
|
19
|
+
image_1: "", // Alternative first image parameter
|
|
20
|
+
image_2: "", // Alternative second image parameter
|
|
21
|
+
images: [], // Alternative array of images
|
|
22
|
+
input_images: [], // Alternative array of input images
|
|
23
|
+
sequentialImageGeneration: "disabled", // Options: "disabled", "auto"
|
|
24
|
+
seed: 0, // Optional seed for reproducible results
|
|
25
|
+
},
|
|
26
|
+
};
|
package/pathways/rag.js
CHANGED
package/pathways/rag_jarvis.js
CHANGED
|
@@ -19,7 +19,7 @@ export default {
|
|
|
19
19
|
calculateEmbeddings: false,
|
|
20
20
|
language: "English",
|
|
21
21
|
chatId: ``,
|
|
22
|
-
dataSources: [
|
|
22
|
+
dataSources: { type: 'array', items: { type: 'string' }, default: [] },
|
|
23
23
|
model: 'oai-gpt4o',
|
|
24
24
|
aiStyle: "OpenAI",
|
|
25
25
|
generatorPathway: 'sys_generator_results',
|