genaicode 0.0.31 → 0.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -10
- package/bin/vertex-monkey-patch.cjs +1 -1
- package/package.json +6 -3
- package/src/ai-service/anthropic.js +6 -3
- package/src/ai-service/chat-gpt.js +5 -2
- package/src/ai-service/common.test.js +106 -0
- package/src/ai-service/dall-e.js +29 -8
- package/src/ai-service/function-calling.js +137 -10
- package/src/ai-service/vertex-ai-claude.js +5 -2
- package/src/ai-service/vertex-ai-imagen.js +78 -0
- package/src/ai-service/vertex-ai.js +9 -8
- package/src/cli/cli-options.js +14 -0
- package/src/cli/cli-options.test.js +25 -0
- package/src/cli/cli-params.js +19 -4
- package/src/cli/validate-cli-params.js +28 -1
- package/src/cli/validate-cli-params.test.js +31 -0
- package/src/files/file-utils.js +7 -0
- package/src/files/file-utils.test.js +21 -0
- package/src/files/find-files.js +4 -66
- package/src/files/read-files.js +20 -1
- package/src/files/read-files.test.js +102 -0
- package/src/files/temp-buffer.js +13 -0
- package/src/files/update-files.js +103 -16
- package/src/images/ensure-alpha.js +5 -0
- package/src/images/imgly-remove-background.js +23 -0
- package/src/images/resize-image.js +23 -0
- package/src/images/split-image.js +26 -0
- package/src/main/codegen.js +10 -3
- package/src/main/codegen.test.js +154 -6
- package/src/main/config-lib.js +33 -0
- package/src/main/config-lib.test.js +80 -0
- package/src/main/config.js +39 -0
- package/src/prompt/prompt-codegen.js +1 -1
- package/src/prompt/prompt-service.js +18 -9
- package/src/prompt/prompt-service.test.js +25 -11
- package/src/prompt/systemprompt.js +2 -2
- package/src/prompt/systemprompt.test.js +5 -2
package/README.md
CHANGED
|
@@ -1,13 +1,3 @@
|
|
|
1
|
-
<p align="center">
|
|
2
|
-
<picture>
|
|
3
|
-
<source media="(prefers-color-scheme: dark)" srcset="media/logo-dark.png">
|
|
4
|
-
<source media="(prefers-color-scheme: light)" srcset="media/logo.png">
|
|
5
|
-
<img alt="GenAIcode Logo." src="media/logo.png" width="100%" height="auto">
|
|
6
|
-
</picture>
|
|
7
|
-
</p>
|
|
8
|
-
|
|
9
|
-
<div align="center">
|
|
10
|
-
|
|
11
1
|
# Programming on steroids
|
|
12
2
|
|
|
13
3
|
<a href="https://www.npmjs.com/package/genaicode">
|
|
@@ -77,6 +67,8 @@ GenAIcode supports various command-line options to customize its behavior:
|
|
|
77
67
|
- `--temperature=<value>`: Sets the temperature parameter for the AI model (default: 0.7).
|
|
78
68
|
- `--vision`: Enables vision capabilities for processing image inputs.
|
|
79
69
|
- `--imagen`: Enables image generation capabilities using AI models.
|
|
70
|
+
- `--cheap`: Uses a cheaper, faster model for code generation, which may provide lower quality results but is more cost-effective for simpler tasks.
|
|
71
|
+
- `--content-mask=<path>`: Applies a content mask to limit the initial source code files included in the request. The value should be a prefix of the path relative to rootDir.
|
|
80
72
|
- `--help`: Displays the help message with all available options.
|
|
81
73
|
|
|
82
74
|
## Configuration (.genaicoderc)
|
|
@@ -27,7 +27,7 @@ if (content.includes(MONKEY_PATCH_TOOL_CONFIG)) {
|
|
|
27
27
|
const newContent = content.replaceAll(
|
|
28
28
|
'data: generateContentRequest,',
|
|
29
29
|
`// MONKEY PATCH TOOL_CONFIG
|
|
30
|
-
data: {...generateContentRequest,tool_config: {function_calling_config: { mode:
|
|
30
|
+
data: {...generateContentRequest,tool_config: {function_calling_config: { mode: request.toolConfig?.functionCallingConfig?.mode, allowed_function_names: request.toolConfig?.functionCallingConfig.allowedFunctionNames }}},`,
|
|
31
31
|
);
|
|
32
32
|
fs.writeFileSync(path, newContent, 'utf-8');
|
|
33
33
|
console.log('Vertex monkey patch applied');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "genaicode",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.33",
|
|
4
4
|
"author": "Grzegorz Tańczyk",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"engines": {
|
|
17
17
|
"node": ">=18"
|
|
18
18
|
},
|
|
19
|
-
"license": "
|
|
19
|
+
"license": "Apache-2.0",
|
|
20
20
|
"files": [
|
|
21
21
|
"bin",
|
|
22
22
|
"src",
|
|
@@ -46,11 +46,14 @@
|
|
|
46
46
|
"@anthropic-ai/sdk": "^0.24.3",
|
|
47
47
|
"@anthropic-ai/vertex-sdk": "^0.4.1",
|
|
48
48
|
"@google-cloud/vertexai": "^1.3.0",
|
|
49
|
+
"@google-cloud/aiplatform": "^3.25.0",
|
|
50
|
+
"@imgly/background-removal-node": "^1.4.5",
|
|
49
51
|
"diff": "^5.2.0",
|
|
50
52
|
"image-size": "^1.1.1",
|
|
51
53
|
"mime-types": "^2.1.35",
|
|
52
54
|
"openai": "^4.52.7",
|
|
53
|
-
"please-upgrade-node": "^3.2.0"
|
|
55
|
+
"please-upgrade-node": "^3.2.0",
|
|
56
|
+
"sharp": "^0.33.4"
|
|
54
57
|
},
|
|
55
58
|
"lint-staged": {
|
|
56
59
|
"*.{js,ts,css,md}": "prettier --write"
|
|
@@ -4,7 +4,7 @@ import { printTokenUsageAndCost, processFunctionCalls } from './common.js';
|
|
|
4
4
|
/**
|
|
5
5
|
* This function generates content using the Anthropic Claude model.
|
|
6
6
|
*/
|
|
7
|
-
export async function generateContent(prompt, functionDefs, requiredFunctionName, temperature) {
|
|
7
|
+
export async function generateContent(prompt, functionDefs, requiredFunctionName, temperature, cheap = false) {
|
|
8
8
|
const anthropic = new Anthropic({
|
|
9
9
|
defaultHeaders: {
|
|
10
10
|
'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15',
|
|
@@ -53,8 +53,11 @@ export async function generateContent(prompt, functionDefs, requiredFunctionName
|
|
|
53
53
|
}
|
|
54
54
|
});
|
|
55
55
|
|
|
56
|
+
const model = cheap ? 'claude-3-haiku-20240307' : 'claude-3-5-sonnet-20240620';
|
|
57
|
+
console.log(`Using Anthropic model: ${model}`);
|
|
58
|
+
|
|
56
59
|
const response = await anthropic.messages.create({
|
|
57
|
-
model:
|
|
60
|
+
model: model,
|
|
58
61
|
system: prompt.find((item) => item.type === 'systemPrompt').systemPrompt,
|
|
59
62
|
messages,
|
|
60
63
|
tools: functionDefs.map((fd) => ({
|
|
@@ -63,7 +66,7 @@ export async function generateContent(prompt, functionDefs, requiredFunctionName
|
|
|
63
66
|
input_schema: fd.parameters,
|
|
64
67
|
})),
|
|
65
68
|
tool_choice: requiredFunctionName ? { type: 'tool', name: requiredFunctionName } : { type: 'any' },
|
|
66
|
-
max_tokens: 8192,
|
|
69
|
+
max_tokens: cheap ? 4096 : 8192,
|
|
67
70
|
temperature: temperature,
|
|
68
71
|
});
|
|
69
72
|
|
|
@@ -4,7 +4,7 @@ import { printTokenUsageAndCost, processFunctionCalls } from './common.js';
|
|
|
4
4
|
/**
|
|
5
5
|
* This function generates content using the OpenAI chat model.
|
|
6
6
|
*/
|
|
7
|
-
export async function generateContent(prompt, functionDefs, requiredFunctionName, temperature) {
|
|
7
|
+
export async function generateContent(prompt, functionDefs, requiredFunctionName, temperature, cheap = false) {
|
|
8
8
|
const openai = new OpenAI();
|
|
9
9
|
|
|
10
10
|
const messages = prompt
|
|
@@ -55,8 +55,11 @@ export async function generateContent(prompt, functionDefs, requiredFunctionName
|
|
|
55
55
|
})
|
|
56
56
|
.flat();
|
|
57
57
|
|
|
58
|
+
const model = cheap ? 'gpt-4o-mini' : 'gpt-4o-2024-08-06';
|
|
59
|
+
console.log(`Using OpenAI model: ${model}`);
|
|
60
|
+
|
|
58
61
|
const response = await openai.chat.completions.create({
|
|
59
|
-
model:
|
|
62
|
+
model: model,
|
|
60
63
|
messages,
|
|
61
64
|
tools: functionDefs.map((funDef) => ({ type: 'function', function: funDef })),
|
|
62
65
|
tool_choice: requiredFunctionName ? { type: 'function', function: { name: requiredFunctionName } } : 'required',
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { printTokenUsageAndCost, processFunctionCalls } from './common.js';
|
|
3
|
+
import { functionDefs } from './function-calling.js';
|
|
4
|
+
|
|
5
|
+
// Mock cli-params.js
|
|
6
|
+
vi.mock('../cli/cli-params.js', () => ({
|
|
7
|
+
chatGpt: true,
|
|
8
|
+
anthropic: false,
|
|
9
|
+
vertexAi: false,
|
|
10
|
+
vertexAiClaude: false,
|
|
11
|
+
temperature: 0.7,
|
|
12
|
+
cheap: false,
|
|
13
|
+
geminiBlockNone: false,
|
|
14
|
+
requireExplanations: false,
|
|
15
|
+
}));
|
|
16
|
+
|
|
17
|
+
describe('printTokenUsageAndCost', () => {
|
|
18
|
+
let consoleLogSpy;
|
|
19
|
+
|
|
20
|
+
beforeEach(() => {
|
|
21
|
+
consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
afterEach(() => {
|
|
25
|
+
consoleLogSpy.mockRestore();
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('should correctly log token usage and estimated cost', () => {
|
|
29
|
+
const usage = {
|
|
30
|
+
inputTokens: 100,
|
|
31
|
+
outputTokens: 50,
|
|
32
|
+
totalTokens: 150,
|
|
33
|
+
};
|
|
34
|
+
const inputCostPerToken = 0.0001;
|
|
35
|
+
const outputCostPerToken = 0.0002;
|
|
36
|
+
|
|
37
|
+
printTokenUsageAndCost(usage, inputCostPerToken, outputCostPerToken);
|
|
38
|
+
|
|
39
|
+
expect(consoleLogSpy).toHaveBeenCalledTimes(5);
|
|
40
|
+
expect(consoleLogSpy).toHaveBeenCalledWith('Token Usage:');
|
|
41
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(' - Input tokens: ', 100);
|
|
42
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(' - Output tokens: ', 50);
|
|
43
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(' - Total tokens: ', 150);
|
|
44
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(' - Estimated cost: ', '0.020000', ' USD');
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it('should handle zero tokens correctly', () => {
|
|
48
|
+
const usage = {
|
|
49
|
+
inputTokens: 0,
|
|
50
|
+
outputTokens: 0,
|
|
51
|
+
totalTokens: 0,
|
|
52
|
+
};
|
|
53
|
+
const inputCostPerToken = 0.0001;
|
|
54
|
+
const outputCostPerToken = 0.0002;
|
|
55
|
+
|
|
56
|
+
printTokenUsageAndCost(usage, inputCostPerToken, outputCostPerToken);
|
|
57
|
+
|
|
58
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(' - Estimated cost: ', '0.000000', ' USD');
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
describe('processFunctionCalls', () => {
|
|
63
|
+
it('should process valid function calls correctly', () => {
|
|
64
|
+
const validFunctionCalls = [
|
|
65
|
+
{ name: 'explanation', args: { text: 'This is an explanation' } },
|
|
66
|
+
{ name: functionDefs[0].name, args: {} },
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
const result = processFunctionCalls(validFunctionCalls);
|
|
70
|
+
|
|
71
|
+
expect(result).toEqual(validFunctionCalls);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('should throw an error for unknown function names', () => {
|
|
75
|
+
const invalidFunctionCalls = [{ name: 'unknownFunction', args: {} }];
|
|
76
|
+
|
|
77
|
+
expect(() => processFunctionCalls(invalidFunctionCalls)).toThrow('Unknown function name: unknownFunction');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('should correctly handle explanations', () => {
|
|
81
|
+
const consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
82
|
+
|
|
83
|
+
const functionCallsWithExplanation = [
|
|
84
|
+
{ name: 'explanation', args: { text: 'This is an explanation' } },
|
|
85
|
+
{ name: functionDefs[0].name, args: {} },
|
|
86
|
+
];
|
|
87
|
+
|
|
88
|
+
processFunctionCalls(functionCallsWithExplanation);
|
|
89
|
+
|
|
90
|
+
expect(consoleLogSpy).toHaveBeenCalledWith('Explanations:', ['This is an explanation']);
|
|
91
|
+
|
|
92
|
+
consoleLogSpy.mockRestore();
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('should return all function calls including explanations', () => {
|
|
96
|
+
const functionCalls = [
|
|
97
|
+
{ name: 'explanation', args: { text: 'This is an explanation' } },
|
|
98
|
+
{ name: functionDefs[0].name, args: {} },
|
|
99
|
+
];
|
|
100
|
+
|
|
101
|
+
const result = processFunctionCalls(functionCalls);
|
|
102
|
+
|
|
103
|
+
expect(result).toEqual(functionCalls);
|
|
104
|
+
expect(result.length).toBe(2);
|
|
105
|
+
});
|
|
106
|
+
});
|
package/src/ai-service/dall-e.js
CHANGED
|
@@ -1,24 +1,45 @@
|
|
|
1
|
-
import OpenAI from 'openai';
|
|
1
|
+
import OpenAI, { toFile } from 'openai';
|
|
2
|
+
|
|
3
|
+
import { setTempBuffer } from '../files/temp-buffer.js';
|
|
4
|
+
import { resizeImageBuffer } from '../images/resize-image.js';
|
|
5
|
+
import { ensureAlpha } from '../images/ensure-alpha.js';
|
|
2
6
|
|
|
3
7
|
/**
|
|
4
8
|
* Generate an image using OpenAI's DALL-E model and save it to a file
|
|
5
9
|
* @param {string} prompt - The description of the image to generate
|
|
6
|
-
* @param {string}
|
|
10
|
+
* @param {string|undefined} contextImagePath - The image to be used as a context
|
|
11
|
+
* @param {{width: number, height: number}} size - The size of the image to generate
|
|
12
|
+
* @param {boolean} cheap - Whether to use a cheaper model
|
|
7
13
|
* @returns {Promise<string>} - The url of the image
|
|
8
14
|
*/
|
|
9
|
-
export async function generateImage(prompt, size =
|
|
15
|
+
export async function generateImage(prompt, contextImagePath, size, cheap = false) {
|
|
10
16
|
const openai = new OpenAI();
|
|
11
17
|
|
|
12
18
|
try {
|
|
13
|
-
const
|
|
14
|
-
|
|
19
|
+
const model = contextImagePath ? 'dall-e-2' : cheap ? 'dall-e-2' : 'dall-e-3';
|
|
20
|
+
console.log(`Using DALL-E model: ${model}`);
|
|
21
|
+
|
|
22
|
+
const options = {
|
|
23
|
+
model: model,
|
|
15
24
|
prompt: prompt,
|
|
16
25
|
n: 1,
|
|
17
|
-
size:
|
|
26
|
+
size: '1024x1024',
|
|
18
27
|
response_format: 'url',
|
|
19
|
-
}
|
|
28
|
+
};
|
|
29
|
+
const response = contextImagePath
|
|
30
|
+
? await openai.images.edit({ ...options, image: await toFile(await ensureAlpha(contextImagePath)) })
|
|
31
|
+
: await openai.images.generate(options);
|
|
32
|
+
|
|
33
|
+
let imageUrl = response.data[0].url;
|
|
34
|
+
|
|
35
|
+
if ((size.width !== 1024) | (size.height !== 1024)) {
|
|
36
|
+
console.log('Resizing image to desired size', size);
|
|
37
|
+
const imageResponse = await fetch(imageUrl);
|
|
38
|
+
const arrayBuffer = await imageResponse.arrayBuffer();
|
|
39
|
+
const buffer = Buffer.from(arrayBuffer);
|
|
20
40
|
|
|
21
|
-
|
|
41
|
+
imageUrl = setTempBuffer(await resizeImageBuffer(buffer, size));
|
|
42
|
+
}
|
|
22
43
|
|
|
23
44
|
console.log(`Image generated, url: ${imageUrl}`);
|
|
24
45
|
return imageUrl;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { requireExplanations, temperature } from '../cli/cli-params.js';
|
|
1
|
+
import { requireExplanations, temperature, cheap } from '../cli/cli-params.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Function definitions for function calling feature
|
|
@@ -7,7 +7,7 @@ export const functionDefs = [
|
|
|
7
7
|
{
|
|
8
8
|
name: 'getSourceCode',
|
|
9
9
|
description:
|
|
10
|
-
'This function returns source code of the application in Map format, where absolute file path is the key, and
|
|
10
|
+
'This function returns source code of the application in Map format, where absolute file path is the key, and the value is an object, where one of the properties may be the content of the file. Some keys may not provide content. This function can be called only once during the conversation, and only if suggested by the user.',
|
|
11
11
|
parameters: {
|
|
12
12
|
type: 'object',
|
|
13
13
|
properties: {
|
|
@@ -66,19 +66,28 @@ export const functionDefs = [
|
|
|
66
66
|
'moveFile',
|
|
67
67
|
'generateImage',
|
|
68
68
|
'downloadFile',
|
|
69
|
+
'splitImage',
|
|
70
|
+
'resizeImage',
|
|
71
|
+
'imglyRemoveBackground',
|
|
69
72
|
],
|
|
70
73
|
description: 'A name of the tool that will be used to perform the update.',
|
|
71
74
|
},
|
|
72
75
|
temperature: {
|
|
73
76
|
type: 'number',
|
|
74
77
|
description:
|
|
75
|
-
'Temperature parameter that will be used for LLM request. The value is adjusted to the characteristic of the update. If there is a need for more creative solution, the value should be lower, but stil within [0.0, 2.0] range. The default value is: ' +
|
|
78
|
+
'Temperature parameter that will be used for LLM request. The value is adjusted to the characteristic of the update. If there is a need for a more creative solution, the value should be lower, but stil within [0.0, 2.0] range. The default value is: ' +
|
|
76
79
|
temperature,
|
|
77
80
|
},
|
|
81
|
+
cheap: {
|
|
82
|
+
type: 'boolean',
|
|
83
|
+
description:
|
|
84
|
+
'true value means that the prompt will be executed with cheaper model, which work faster, but provides lower quality results, so please use it only in situation when lower quality results are acceptable for the prompt. The default value is: ' +
|
|
85
|
+
!!cheap,
|
|
86
|
+
},
|
|
78
87
|
prompt: {
|
|
79
88
|
type: 'string',
|
|
80
89
|
description:
|
|
81
|
-
'Prompt that will be
|
|
90
|
+
'Prompt that will be passed to the model request together with the tool request. It summarizes the planned changes for this particular file, so it should be detailed enough for the model to generate necessary changes.',
|
|
82
91
|
},
|
|
83
92
|
contextImageAssets: {
|
|
84
93
|
type: 'array',
|
|
@@ -87,13 +96,13 @@ export const functionDefs = [
|
|
|
87
96
|
items: { type: 'string' },
|
|
88
97
|
},
|
|
89
98
|
},
|
|
90
|
-
required: ['path', 'updateToolName', 'temperature', 'prompt', 'contextImageAssets'],
|
|
99
|
+
required: ['path', 'updateToolName', 'temperature', 'prompt', 'contextImageAssets', 'cheap'],
|
|
91
100
|
},
|
|
92
101
|
},
|
|
93
102
|
contextPaths: {
|
|
94
103
|
type: 'array',
|
|
95
104
|
description:
|
|
96
|
-
'An array of absolute paths of files that should be used to
|
|
105
|
+
'An array of absolute paths of files that should be used to provide context for the following updates. Context files could be for example the dependencies, or files that depend on one of the files that we want to update in the next step.',
|
|
97
106
|
items: {
|
|
98
107
|
type: 'string',
|
|
99
108
|
},
|
|
@@ -268,23 +277,44 @@ Index: filename.js
|
|
|
268
277
|
properties: {
|
|
269
278
|
prompt: {
|
|
270
279
|
type: 'string',
|
|
271
|
-
description:
|
|
280
|
+
description:
|
|
281
|
+
'The prompt that will be used to generate the image. This prompt must be detailed, it will be used by image generation model.',
|
|
272
282
|
},
|
|
273
283
|
filePath: {
|
|
274
284
|
type: 'string',
|
|
275
285
|
description: 'The file path to save the generated image.',
|
|
276
286
|
},
|
|
277
|
-
|
|
287
|
+
contextImagePath: {
|
|
278
288
|
type: 'string',
|
|
279
|
-
|
|
289
|
+
description:
|
|
290
|
+
'Path to a image file that will be used as a context for image generation. It is useful if there is a need to edit an image with genAI.',
|
|
291
|
+
},
|
|
292
|
+
size: {
|
|
293
|
+
type: 'object',
|
|
294
|
+
properties: {
|
|
295
|
+
width: {
|
|
296
|
+
type: 'number',
|
|
297
|
+
description: 'width of the image',
|
|
298
|
+
},
|
|
299
|
+
height: {
|
|
300
|
+
type: 'number',
|
|
301
|
+
description: 'height of the image',
|
|
302
|
+
},
|
|
303
|
+
},
|
|
304
|
+
required: ['width', 'height'],
|
|
280
305
|
description: 'The size of the image to generate.',
|
|
281
306
|
},
|
|
307
|
+
cheap: {
|
|
308
|
+
type: 'boolean',
|
|
309
|
+
description:
|
|
310
|
+
'true value means that the prompt will be executed with cheaper model, which work faster, but provides lower quality results, so please use it only in situation when lower quality results are acceptable for the prompt.',
|
|
311
|
+
},
|
|
282
312
|
explanation: {
|
|
283
313
|
type: 'string',
|
|
284
314
|
description: 'The explanation of the reasoning behind generating this image',
|
|
285
315
|
},
|
|
286
316
|
},
|
|
287
|
-
required: ['prompt', 'filePath', 'size'],
|
|
317
|
+
required: ['prompt', 'filePath', 'size', 'cheap'],
|
|
288
318
|
},
|
|
289
319
|
},
|
|
290
320
|
{
|
|
@@ -309,6 +339,103 @@ Index: filename.js
|
|
|
309
339
|
required: ['filePath', 'downloadUrl'],
|
|
310
340
|
},
|
|
311
341
|
},
|
|
342
|
+
{
|
|
343
|
+
name: 'imglyRemoveBackground',
|
|
344
|
+
description: 'Removes background from an image using @imgly/background-removal-node',
|
|
345
|
+
parameters: {
|
|
346
|
+
type: 'object',
|
|
347
|
+
properties: {
|
|
348
|
+
inputFilePath: {
|
|
349
|
+
type: 'string',
|
|
350
|
+
description: 'The file path of the input image.',
|
|
351
|
+
},
|
|
352
|
+
outputFilePath: {
|
|
353
|
+
type: 'string',
|
|
354
|
+
description: 'The file path to save the output image with removed background.',
|
|
355
|
+
},
|
|
356
|
+
explanation: {
|
|
357
|
+
type: 'string',
|
|
358
|
+
description: 'The explanation of the reasoning behind removing the background from this image',
|
|
359
|
+
},
|
|
360
|
+
},
|
|
361
|
+
required: ['inputFilePath', 'outputFilePath'],
|
|
362
|
+
},
|
|
363
|
+
},
|
|
364
|
+
{
|
|
365
|
+
name: 'resizeImage',
|
|
366
|
+
description: 'Resize image to the desired size',
|
|
367
|
+
parameters: {
|
|
368
|
+
type: 'object',
|
|
369
|
+
properties: {
|
|
370
|
+
filePath: {
|
|
371
|
+
type: 'string',
|
|
372
|
+
description: 'The file path of the image.',
|
|
373
|
+
},
|
|
374
|
+
size: {
|
|
375
|
+
type: 'object',
|
|
376
|
+
properties: {
|
|
377
|
+
width: {
|
|
378
|
+
type: 'number',
|
|
379
|
+
description: 'width of the image',
|
|
380
|
+
},
|
|
381
|
+
height: {
|
|
382
|
+
type: 'number',
|
|
383
|
+
description: 'height of the image',
|
|
384
|
+
},
|
|
385
|
+
},
|
|
386
|
+
required: ['width', 'height'],
|
|
387
|
+
description: 'The size of the image to generate.',
|
|
388
|
+
},
|
|
389
|
+
explanation: {
|
|
390
|
+
type: 'string',
|
|
391
|
+
description: 'The explanation of the reasoning behind removing the background from this image',
|
|
392
|
+
},
|
|
393
|
+
},
|
|
394
|
+
required: ['filePath', 'size'],
|
|
395
|
+
},
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
name: 'splitImage',
|
|
399
|
+
description: 'Split an image into multiple parts and save them as separate files.',
|
|
400
|
+
parameters: {
|
|
401
|
+
type: 'object',
|
|
402
|
+
properties: {
|
|
403
|
+
inputFilePath: {
|
|
404
|
+
type: 'string',
|
|
405
|
+
description: 'The file path of the input image to be split.',
|
|
406
|
+
},
|
|
407
|
+
parts: {
|
|
408
|
+
type: 'array',
|
|
409
|
+
items: {
|
|
410
|
+
type: 'object',
|
|
411
|
+
properties: {
|
|
412
|
+
rect: {
|
|
413
|
+
type: 'object',
|
|
414
|
+
properties: {
|
|
415
|
+
x: { type: 'number', description: 'The x-coordinate of the top-left corner of the rectangle.' },
|
|
416
|
+
y: { type: 'number', description: 'The y-coordinate of the top-left corner of the rectangle.' },
|
|
417
|
+
width: { type: 'number', description: 'The width of the rectangle.' },
|
|
418
|
+
height: { type: 'number', description: 'The height of the rectangle.' },
|
|
419
|
+
},
|
|
420
|
+
required: ['x', 'y', 'width', 'height'],
|
|
421
|
+
},
|
|
422
|
+
outputFilePath: {
|
|
423
|
+
type: 'string',
|
|
424
|
+
description: 'The file path to save the extracted part of the image.',
|
|
425
|
+
},
|
|
426
|
+
},
|
|
427
|
+
required: ['rect', 'outputFilePath'],
|
|
428
|
+
},
|
|
429
|
+
description: 'An array of parts to extract from the image, each with a rectangle and output file path.',
|
|
430
|
+
},
|
|
431
|
+
explanation: {
|
|
432
|
+
type: 'string',
|
|
433
|
+
description: 'The explanation of the reasoning behind splitting this image',
|
|
434
|
+
},
|
|
435
|
+
},
|
|
436
|
+
required: ['inputFilePath', 'parts'],
|
|
437
|
+
},
|
|
438
|
+
},
|
|
312
439
|
].map((fd) => {
|
|
313
440
|
if (requireExplanations && fd.parameters.properties.explanation && !fd.parameters.required.includes('explanation')) {
|
|
314
441
|
fd.parameters.required.push('explanation');
|
|
@@ -4,7 +4,7 @@ import { printTokenUsageAndCost, processFunctionCalls } from './common.js';
|
|
|
4
4
|
/**
|
|
5
5
|
* This function generates content using the Anthropic Claude model via Vertex AI.
|
|
6
6
|
*/
|
|
7
|
-
export async function generateContent(prompt, functionDefs, requiredFunctionName, temperature) {
|
|
7
|
+
export async function generateContent(prompt, functionDefs, requiredFunctionName, temperature, cheap = false) {
|
|
8
8
|
const projectId = process.env.GOOGLE_CLOUD_PROJECT;
|
|
9
9
|
const region = process.env.GOOGLE_CLOUD_REGION;
|
|
10
10
|
|
|
@@ -56,8 +56,11 @@ export async function generateContent(prompt, functionDefs, requiredFunctionName
|
|
|
56
56
|
}
|
|
57
57
|
});
|
|
58
58
|
|
|
59
|
+
const model = cheap ? 'claude-3-haiku@20240307' : 'claude-3-5-sonnet@20240620';
|
|
60
|
+
console.log(`Using Vertex AI Claude model: ${model}`);
|
|
61
|
+
|
|
59
62
|
const response = await client.messages.create({
|
|
60
|
-
model:
|
|
63
|
+
model: model,
|
|
61
64
|
max_tokens: 4096,
|
|
62
65
|
temperature: temperature,
|
|
63
66
|
system: prompt.find((item) => item.type === 'systemPrompt').systemPrompt,
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import mime from 'mime-types';
|
|
3
|
+
import { PredictionServiceClient, helpers } from '@google-cloud/aiplatform';
|
|
4
|
+
import { setTempBuffer } from '../files/temp-buffer.js';
|
|
5
|
+
import { resizeImageBuffer } from '../images/resize-image.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Generate an image using Vertex AI's Imagen model and return the image URL
|
|
9
|
+
* @param {string} prompt - The description of the image to generate
|
|
10
|
+
* @param {string|undefined} contextImagePath - The image to be used as a context
|
|
11
|
+
* @param {{width: number, height: number}} size - The size of the image to generate
|
|
12
|
+
* @param {boolean} cheap - Whether to use a cheaper model
|
|
13
|
+
* @returns {Promise<string>} - The url of the generated image
|
|
14
|
+
*/
|
|
15
|
+
export async function generateImage(prompt, contextImagePath, size, cheap = false) {
|
|
16
|
+
// Initialize the PredictionServiceClient
|
|
17
|
+
const client = new PredictionServiceClient({
|
|
18
|
+
apiEndpoint: `${process.env.GOOGLE_CLOUD_REGION}-aiplatform.googleapis.com`,
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
// Set the project and location
|
|
22
|
+
const projectId = process.env.GOOGLE_CLOUD_PROJECT;
|
|
23
|
+
const location = process.env.GOOGLE_CLOUD_REGION;
|
|
24
|
+
|
|
25
|
+
// Set the model name based on the cheap parameter
|
|
26
|
+
const modelName = contextImagePath
|
|
27
|
+
? 'imagegeneration@002'
|
|
28
|
+
: cheap
|
|
29
|
+
? 'imagen-3.0-fast-generate-001'
|
|
30
|
+
: 'imagen-3.0-generate-001';
|
|
31
|
+
console.log(`Using Vertex AI Imagen model: ${modelName}`);
|
|
32
|
+
|
|
33
|
+
try {
|
|
34
|
+
// Prepare the request
|
|
35
|
+
const request = {
|
|
36
|
+
endpoint: `projects/${projectId}/locations/${location}/publishers/google/models/${modelName}`,
|
|
37
|
+
instances: [
|
|
38
|
+
helpers.toValue({
|
|
39
|
+
prompt: prompt,
|
|
40
|
+
...(contextImagePath
|
|
41
|
+
? {
|
|
42
|
+
image: {
|
|
43
|
+
bytesBase64Encoded: fs.readFileSync(contextImagePath, 'base64'),
|
|
44
|
+
mediaType: mime.lookup(contextImagePath),
|
|
45
|
+
},
|
|
46
|
+
}
|
|
47
|
+
: {}),
|
|
48
|
+
}),
|
|
49
|
+
],
|
|
50
|
+
parameters: helpers.toValue({
|
|
51
|
+
sampleCount: 1,
|
|
52
|
+
safetySetting: 'block_most',
|
|
53
|
+
personGeneration: 'allow_adult',
|
|
54
|
+
includeRaiReason: true,
|
|
55
|
+
language: 'auto',
|
|
56
|
+
aspectRatio: '1:1',
|
|
57
|
+
addWatermark: false,
|
|
58
|
+
}),
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
// Make the prediction request
|
|
62
|
+
const [response] = await client.predict(request);
|
|
63
|
+
|
|
64
|
+
if (response.predictions && response.predictions.length > 0) {
|
|
65
|
+
const prediction = helpers.fromValue(response.predictions[0]);
|
|
66
|
+
if (prediction.bytesBase64Encoded) {
|
|
67
|
+
console.log(`Image generated successfully, resizing to desired dimension`, size);
|
|
68
|
+
const buffer = Buffer.from(prediction.bytesBase64Encoded, 'base64');
|
|
69
|
+
return setTempBuffer(await resizeImageBuffer(buffer, size));
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
throw new Error('No image generated in the response');
|
|
74
|
+
} catch (error) {
|
|
75
|
+
console.error('Error generating image:', error);
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
@@ -7,7 +7,7 @@ import { geminiBlockNone } from '../cli/cli-params.js';
|
|
|
7
7
|
* This function generates content using the Gemini Pro model.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
export async function generateContent(prompt, functionDefs, requiredFunctionName, temperature) {
|
|
10
|
+
export async function generateContent(prompt, functionDefs, requiredFunctionName, temperature, cheap = false) {
|
|
11
11
|
const messages = prompt
|
|
12
12
|
.filter((item) => item.type !== 'systemPrompt')
|
|
13
13
|
.map((item) => {
|
|
@@ -53,16 +53,15 @@ export async function generateContent(prompt, functionDefs, requiredFunctionName
|
|
|
53
53
|
functionDeclarations: functionDefs,
|
|
54
54
|
},
|
|
55
55
|
],
|
|
56
|
-
// TODO: add tool_config once [it is supported](https://github.com/googleapis/nodejs-vertexai/issues/331)
|
|
57
56
|
toolConfig: {
|
|
58
57
|
functionCallingConfig: {
|
|
59
|
-
mode: 'ANY',
|
|
60
|
-
...(requiredFunctionName ? { allowedFunctionNames: [requiredFunctionName] } : {}),
|
|
58
|
+
mode: cheap ? undefined : 'ANY',
|
|
59
|
+
...(!cheap && requiredFunctionName ? { allowedFunctionNames: [requiredFunctionName] } : {}),
|
|
61
60
|
},
|
|
62
61
|
},
|
|
63
62
|
};
|
|
64
63
|
|
|
65
|
-
const model = await getGenModel(prompt.find((item) => item.type === 'systemPrompt').systemPrompt, temperature);
|
|
64
|
+
const model = await getGenModel(prompt.find((item) => item.type === 'systemPrompt').systemPrompt, temperature, cheap);
|
|
66
65
|
|
|
67
66
|
assert(await verifyVertexMonkeyPatch(), 'Vertex AI Tool Config was not monkey patched');
|
|
68
67
|
|
|
@@ -105,11 +104,13 @@ export async function generateContent(prompt, functionDefs, requiredFunctionName
|
|
|
105
104
|
}
|
|
106
105
|
|
|
107
106
|
// A function to get the generative model
|
|
108
|
-
// Modified to accept temperature parameter
|
|
109
|
-
export function getGenModel(systemPrompt, temperature) {
|
|
107
|
+
// Modified to accept temperature parameter and cheap flag
|
|
108
|
+
export function getGenModel(systemPrompt, temperature, cheap = false) {
|
|
110
109
|
// Initialize Vertex with your Cloud project and location
|
|
111
110
|
const vertex_ai = new VertexAI({});
|
|
112
|
-
const model = 'gemini-1.5-pro-001';
|
|
111
|
+
const model = cheap ? 'gemini-1.5-flash-001' : 'gemini-1.5-pro-001';
|
|
112
|
+
|
|
113
|
+
console.log(`Using Vertex AI model: ${model}`);
|
|
113
114
|
|
|
114
115
|
// Instantiate the models
|
|
115
116
|
return vertex_ai.preview.getGenerativeModel({
|
package/src/cli/cli-options.js
CHANGED
|
@@ -86,6 +86,20 @@ const cliOptions = [
|
|
|
86
86
|
description:
|
|
87
87
|
'Enable vision capabilities for processing image inputs. This option allows the tool to analyze and generate code based on image content when used with compatible AI models.',
|
|
88
88
|
},
|
|
89
|
+
{
|
|
90
|
+
name: '--imagen=<service>',
|
|
91
|
+
description:
|
|
92
|
+
'Enable image generation functionality and specify the service to use (either "vertex-ai" or "dall-e").',
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
name: '--cheap',
|
|
96
|
+
description: 'Switch to cheaper models in AI services for content and image generation.',
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
name: '--content-mask=<path>',
|
|
100
|
+
description:
|
|
101
|
+
'Apply a content mask to limit the initial source code files included in the request. The value should be a prefix of the path relative to rootDir.',
|
|
102
|
+
},
|
|
89
103
|
];
|
|
90
104
|
|
|
91
105
|
/**
|