@ai-setting/roy-agent-core 1.7.3 → 1.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/index.js +5 -5
- package/dist/env/agent/index.js +6 -6
- package/dist/env/event-source/index.js +6 -6
- package/dist/env/index.js +43 -43
- package/dist/env/llm/index.js +4 -3
- package/dist/env/mcp/index.js +9 -9
- package/dist/env/memory/index.js +4 -4
- package/dist/env/session/index.js +2 -2
- package/dist/env/task/delegate/index.js +2 -2
- package/dist/env/task/index.js +5 -5
- package/dist/env/task/plugins/index.js +4 -4
- package/dist/env/tool/built-in/index.js +3 -2
- package/dist/env/tool/index.js +6 -5
- package/dist/env/workflow/engine/index.js +3 -3
- package/dist/env/workflow/index.js +27 -27
- package/dist/env/workflow/nodes/index.js +1 -1
- package/dist/env/workflow/tools/index.js +6 -6
- package/dist/env/workflow/utils/index.js +3 -3
- package/dist/index.js +100 -98
- package/dist/shared/@ai-setting/{roy-agent-core-qatz3q31.js → roy-agent-core-29ph0e2k.js} +6 -6
- package/dist/shared/@ai-setting/{roy-agent-core-watkmj7d.js → roy-agent-core-32226m35.js} +1 -1
- package/dist/shared/@ai-setting/{roy-agent-core-y9q3jdcx.js → roy-agent-core-4m4awz0m.js} +15 -15
- package/dist/shared/@ai-setting/{roy-agent-core-pdnp5kj0.js → roy-agent-core-62mvbqmj.js} +2 -2
- package/dist/shared/@ai-setting/{roy-agent-core-xmacq00w.js → roy-agent-core-7bskb5j2.js} +9 -9
- package/dist/shared/@ai-setting/{roy-agent-core-y3g3ar7a.js → roy-agent-core-adg9ghbt.js} +109 -1
- package/dist/shared/@ai-setting/{roy-agent-core-s4r8f6se.js → roy-agent-core-cpfdvzqk.js} +63 -9
- package/dist/shared/@ai-setting/{roy-agent-core-yak43g6m.js → roy-agent-core-d11714ee.js} +56 -9
- package/dist/shared/@ai-setting/{roy-agent-core-0qjw25e8.js → roy-agent-core-hmqmznjc.js} +33 -1
- package/dist/shared/@ai-setting/{roy-agent-core-g7nmwg63.js → roy-agent-core-hncxk9d1.js} +5 -111
- package/dist/shared/@ai-setting/{roy-agent-core-cy8azvpb.js → roy-agent-core-krapd8x2.js} +2 -1
- package/dist/shared/@ai-setting/{roy-agent-core-kv7frw13.js → roy-agent-core-mgp3284f.js} +1 -1
- package/dist/shared/@ai-setting/{roy-agent-core-3dwcwgn5.js → roy-agent-core-q6mq7gmc.js} +14 -14
- package/dist/shared/@ai-setting/{roy-agent-core-wr1pgw0x.js → roy-agent-core-w4dy6zzf.js} +1 -1
- package/dist/shared/@ai-setting/roy-agent-core-xwm8zavb.js +89 -0
- package/dist/shared/@ai-setting/{roy-agent-core-t7rw68q8.js → roy-agent-core-zjdzk1gg.js} +34 -84
- package/package.json +1 -1
- package/dist/shared/@ai-setting/{roy-agent-core-50jdqb7r.js → roy-agent-core-2yt4jmzv.js} +3 -3
- package/dist/shared/@ai-setting/{roy-agent-core-h28gcd77.js → roy-agent-core-71wde8ce.js} +0 -0
- package/dist/shared/@ai-setting/{roy-agent-core-3e0w0ekw.js → roy-agent-core-7xek618f.js} +4 -4
- package/dist/shared/@ai-setting/{roy-agent-core-xvag5pd0.js → roy-agent-core-v13dq3nc.js} +4 -4
- package/dist/shared/@ai-setting/{roy-agent-core-gns3wa97.js → roy-agent-core-zhv8efqh.js} +7 -7
- package/dist/shared/@ai-setting/{roy-agent-core-s07nere5.js → roy-agent-core-zqeq8m9z.js} +7 -7
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
import {
|
|
2
|
-
EnvSource
|
|
3
|
-
|
|
2
|
+
EnvSource,
|
|
3
|
+
parseJSONC
|
|
4
|
+
} from "./roy-agent-core-adg9ghbt.js";
|
|
4
5
|
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
} from "./roy-agent-core-ctdhjv68.js";
|
|
6
|
+
sniffImageMime
|
|
7
|
+
} from "./roy-agent-core-zjdzk1gg.js";
|
|
8
8
|
import {
|
|
9
9
|
invoke,
|
|
10
10
|
invokeNonStream
|
|
11
11
|
} from "./roy-agent-core-2xb3f3ye.js";
|
|
12
|
+
import {
|
|
13
|
+
ContextError,
|
|
14
|
+
ErrorCodes
|
|
15
|
+
} from "./roy-agent-core-ctdhjv68.js";
|
|
12
16
|
import {
|
|
13
17
|
envKeyToConfigKey,
|
|
14
18
|
toEnvKey
|
|
@@ -107,7 +111,7 @@ function toPiMessages(messages) {
|
|
|
107
111
|
if (c.type === "text")
|
|
108
112
|
return [{ type: "text", text: c.text }];
|
|
109
113
|
if (c.type === "image")
|
|
110
|
-
return [{ type: "image", data: c.image instanceof Uint8Array ? Buffer.from(c.image).toString("base64") : c.image, mimeType:
|
|
114
|
+
return [{ type: "image", data: c.image instanceof Uint8Array ? Buffer.from(c.image).toString("base64") : c.image, mimeType: sniffImageMime(c.image, c.mediaType) }];
|
|
111
115
|
return [{ type: "text", text: `[file:${c.mediaType}] ${typeof c.file === "string" ? c.file : Buffer.from(c.file).toString("base64")}` }];
|
|
112
116
|
}), timestamp: ts });
|
|
113
117
|
}
|
|
@@ -125,7 +129,7 @@ function toPiMessages(messages) {
|
|
|
125
129
|
if (c.type === "text")
|
|
126
130
|
content.push({ type: "text", text: c.text });
|
|
127
131
|
else if (c.type === "image")
|
|
128
|
-
content.push({ type: "image", data: c.image instanceof Uint8Array ? Buffer.from(c.image).toString("base64") : c.image, mimeType:
|
|
132
|
+
content.push({ type: "image", data: c.image instanceof Uint8Array ? Buffer.from(c.image).toString("base64") : c.image, mimeType: sniffImageMime(c.image, c.mediaType) });
|
|
129
133
|
else
|
|
130
134
|
content.push({ type: "text", text: `[file:${c.mediaType}] ${typeof c.file === "string" ? c.file : Buffer.from(c.file).toString("base64")}` });
|
|
131
135
|
}
|
|
@@ -903,7 +907,38 @@ class LLMHooks {
|
|
|
903
907
|
init_logger();
|
|
904
908
|
init_decorator();
|
|
905
909
|
import { randomUUID } from "crypto";
|
|
910
|
+
import * as fs from "node:fs/promises";
|
|
911
|
+
import * as os from "node:os";
|
|
912
|
+
import * as path from "node:path";
|
|
906
913
|
var logger2 = createLogger("llm");
|
|
914
|
+
async function resolveFallbackProviderConfig(providerId) {
|
|
915
|
+
const home = os.homedir();
|
|
916
|
+
const authPath = path.join(home, ".roy-agent", "auth.json");
|
|
917
|
+
const configPath = path.join(home, ".roy-agent", "config.jsonc");
|
|
918
|
+
let authText;
|
|
919
|
+
let configText;
|
|
920
|
+
try {
|
|
921
|
+
[authText, configText] = await Promise.all([
|
|
922
|
+
fs.readFile(authPath, "utf-8"),
|
|
923
|
+
fs.readFile(configPath, "utf-8")
|
|
924
|
+
]);
|
|
925
|
+
} catch (err) {
|
|
926
|
+
throw new Error(`resolveFallbackProviderConfig: cannot read ${authPath} or ${configPath}: ` + err.message);
|
|
927
|
+
}
|
|
928
|
+
const auth = JSON.parse(authText);
|
|
929
|
+
const authEntry = auth.providers?.[providerId];
|
|
930
|
+
if (!authEntry?.apiKey || !authEntry?.baseURL) {
|
|
931
|
+
throw new Error(`resolveFallbackProviderConfig: no config found for provider "${providerId}" ` + `in ${authPath}. Add it under "providers.${providerId}".`);
|
|
932
|
+
}
|
|
933
|
+
const config = parseJSONC(configText);
|
|
934
|
+
const configEntry = config.llm?.providers?.[providerId];
|
|
935
|
+
const providerType = configEntry?.provider ?? "openai-compatible";
|
|
936
|
+
return {
|
|
937
|
+
baseURL: authEntry.baseURL,
|
|
938
|
+
apiKey: authEntry.apiKey,
|
|
939
|
+
providerType
|
|
940
|
+
};
|
|
941
|
+
}
|
|
907
942
|
|
|
908
943
|
class LLMComponent extends BaseComponent {
|
|
909
944
|
name = "llm";
|
|
@@ -1193,9 +1228,21 @@ class LLMComponent extends BaseComponent {
|
|
|
1193
1228
|
}
|
|
1194
1229
|
} catch (adapterErr) {
|
|
1195
1230
|
logger2.warn(`[LLMComponent.invoke] adapter(${adapter.name}) failed, falling back to AI SDK invoke: ${adapterErr.message}`);
|
|
1196
|
-
|
|
1231
|
+
let fallbackInvokeConfig = ctx.invokeConfig;
|
|
1232
|
+
try {
|
|
1233
|
+
const fallback = await resolveFallbackProviderConfig(ctx.providerId);
|
|
1234
|
+
fallbackInvokeConfig = {
|
|
1235
|
+
...ctx.invokeConfig,
|
|
1236
|
+
apiKey: fallback.apiKey,
|
|
1237
|
+
baseURL: fallback.baseURL
|
|
1238
|
+
};
|
|
1239
|
+
logger2.warn(`[LLMComponent.invoke] fallback re-resolved baseURL from auth.json: ${fallback.baseURL}`);
|
|
1240
|
+
} catch (configErr) {
|
|
1241
|
+
logger2.warn(`[LLMComponent.invoke] resolveFallbackProviderConfig failed, using ctx.invokeConfig as-is: ${configErr.message}`);
|
|
1242
|
+
}
|
|
1243
|
+
const result = await invoke(fallbackInvokeConfig, {
|
|
1197
1244
|
messages: ctx.messages,
|
|
1198
|
-
model:
|
|
1245
|
+
model: fallbackInvokeConfig.model,
|
|
1199
1246
|
temperature: ctx.temperature,
|
|
1200
1247
|
maxTokens: ctx.maxTokens,
|
|
1201
1248
|
tools: request.tools,
|
|
@@ -125,9 +125,10 @@ var init_tool_node = __esm(() => {
|
|
|
125
125
|
}
|
|
126
126
|
}
|
|
127
127
|
const extractedOutput = this.extractFromWrapper(output);
|
|
128
|
+
const wrapperError = this.extractErrorFromWrapper(output);
|
|
128
129
|
return {
|
|
129
130
|
output: extractedOutput,
|
|
130
|
-
error:
|
|
131
|
+
error: wrapperError,
|
|
131
132
|
durationMs: Date.now() - startTime
|
|
132
133
|
};
|
|
133
134
|
} catch (error) {
|
|
@@ -158,6 +159,37 @@ var init_tool_node = __esm(() => {
|
|
|
158
159
|
}
|
|
159
160
|
return result;
|
|
160
161
|
}
|
|
162
|
+
extractErrorFromWrapper(value) {
|
|
163
|
+
if (value === null || value === undefined) {
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
if (typeof value !== "object") {
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
const obj = value;
|
|
170
|
+
if (obj.success !== false) {
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
const err = obj.error;
|
|
174
|
+
if (typeof err === "string" && err.length > 0) {
|
|
175
|
+
return err;
|
|
176
|
+
}
|
|
177
|
+
if (err instanceof Error) {
|
|
178
|
+
return err;
|
|
179
|
+
}
|
|
180
|
+
const md = obj.metadata;
|
|
181
|
+
if (md) {
|
|
182
|
+
if (md.timed_out === true) {
|
|
183
|
+
const ms = typeof md.execution_time_ms === "number" ? md.execution_time_ms : 0;
|
|
184
|
+
return `Tool execution timed out after ${ms}ms`;
|
|
185
|
+
}
|
|
186
|
+
if (typeof md.exit_code === "number" && md.exit_code !== 0) {
|
|
187
|
+
const stderr = typeof md.stderr === "string" ? md.stderr : "";
|
|
188
|
+
return stderr.length > 0 ? stderr : `Tool exited with code ${md.exit_code}`;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return `Tool execution failed (success=false, no error message)`;
|
|
192
|
+
}
|
|
161
193
|
};
|
|
162
194
|
__legacyDecorateClassTS([
|
|
163
195
|
TracedAs("workflow.tool.execute")
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
|
-
EnvSource
|
|
3
|
-
|
|
2
|
+
EnvSource,
|
|
3
|
+
parseJSONCWithEnv,
|
|
4
|
+
parseJSONCWithProtocols
|
|
5
|
+
} from "./roy-agent-core-adg9ghbt.js";
|
|
4
6
|
import {
|
|
5
7
|
XDG_PATHS
|
|
6
8
|
} from "./roy-agent-core-qxnbvgwe.js";
|
|
@@ -11,114 +13,6 @@ import {
|
|
|
11
13
|
__require
|
|
12
14
|
} from "./roy-agent-core-fs0mn2jk.js";
|
|
13
15
|
|
|
14
|
-
// src/config/config-parser.ts
|
|
15
|
-
import { parse as parseJSONCInternal, ParseErrorCode } from "jsonc-parser";
|
|
16
|
-
function parseJSONC(content) {
|
|
17
|
-
const errors = [];
|
|
18
|
-
const result = parseJSONCInternal(content, errors, {
|
|
19
|
-
allowTrailingComma: true,
|
|
20
|
-
disallowComments: false
|
|
21
|
-
});
|
|
22
|
-
if (errors.length > 0) {
|
|
23
|
-
const hasOnlyValueExpected = result === undefined && errors.length === 1 && errors[0].error === ParseErrorCode.ValueExpected;
|
|
24
|
-
if (!hasOnlyValueExpected) {
|
|
25
|
-
const error = errors[0];
|
|
26
|
-
const offset = error.offset;
|
|
27
|
-
const lines = content.substring(0, offset).split(`
|
|
28
|
-
`);
|
|
29
|
-
const line = lines.length;
|
|
30
|
-
const col = lines[lines.length - 1].length + 1;
|
|
31
|
-
const errorMessages = {
|
|
32
|
-
[ParseErrorCode.UnexpectedEndOfString]: "Unexpected end of string",
|
|
33
|
-
[ParseErrorCode.InvalidCharacter]: "Invalid character",
|
|
34
|
-
[ParseErrorCode.PropertyNameExpected]: "Property name expected",
|
|
35
|
-
[ParseErrorCode.ValueExpected]: "Value expected",
|
|
36
|
-
[ParseErrorCode.ColonExpected]: "Colon expected",
|
|
37
|
-
[ParseErrorCode.CommaExpected]: "Comma expected",
|
|
38
|
-
[ParseErrorCode.CloseBraceExpected]: "Close brace expected",
|
|
39
|
-
[ParseErrorCode.CloseBracketExpected]: "Close bracket expected",
|
|
40
|
-
[ParseErrorCode.EndOfFileExpected]: "Unexpected end of input",
|
|
41
|
-
[ParseErrorCode.InvalidSymbol]: "Invalid symbol"
|
|
42
|
-
};
|
|
43
|
-
throw new Error(`JSONC parse error at line ${line}, col ${col}: ${errorMessages[error.error] || "Unknown error"}`);
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
if (result === undefined || result === null) {
|
|
47
|
-
return {};
|
|
48
|
-
}
|
|
49
|
-
return result;
|
|
50
|
-
}
|
|
51
|
-
function substituteEnvVars(value) {
|
|
52
|
-
if (typeof value === "string") {
|
|
53
|
-
return value.replace(/\$\{([^}]+)\}/g, (match, varExpr) => {
|
|
54
|
-
const [varName, defaultValue] = varExpr.split(":-");
|
|
55
|
-
const envValue = process.env[varName.trim()];
|
|
56
|
-
if (envValue !== undefined) {
|
|
57
|
-
return envValue;
|
|
58
|
-
}
|
|
59
|
-
if (defaultValue !== undefined) {
|
|
60
|
-
return defaultValue;
|
|
61
|
-
}
|
|
62
|
-
return "";
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
if (Array.isArray(value)) {
|
|
66
|
-
return value.map((item) => substituteEnvVars(item));
|
|
67
|
-
}
|
|
68
|
-
if (value !== null && typeof value === "object") {
|
|
69
|
-
const result = {};
|
|
70
|
-
for (const [key, val] of Object.entries(value)) {
|
|
71
|
-
result[key] = substituteEnvVars(val);
|
|
72
|
-
}
|
|
73
|
-
return result;
|
|
74
|
-
}
|
|
75
|
-
return value;
|
|
76
|
-
}
|
|
77
|
-
function parseJSONCWithEnv(content) {
|
|
78
|
-
const parsed = parseJSONC(content);
|
|
79
|
-
return substituteEnvVars(parsed);
|
|
80
|
-
}
|
|
81
|
-
var PROTOCOL_REF_PATTERN = /^\$\{([^:]+):\/\/[^}]+\}$/;
|
|
82
|
-
var PROTOCOL_REF_WITH_DEFAULT_PATTERN = /^\$\{([^:]+):\/\/([^:#]+)(?:#([^:-]+))?(?::-(.*))?\}$/;
|
|
83
|
-
function isProtocolRefString(value) {
|
|
84
|
-
return PROTOCOL_REF_PATTERN.test(value);
|
|
85
|
-
}
|
|
86
|
-
function extractDefaultValue(ref) {
|
|
87
|
-
const match = ref.match(PROTOCOL_REF_WITH_DEFAULT_PATTERN);
|
|
88
|
-
return match ? match[4] : undefined;
|
|
89
|
-
}
|
|
90
|
-
function substituteProtocolRefs(value, resolver) {
|
|
91
|
-
if (typeof value === "string") {
|
|
92
|
-
if (isProtocolRefString(value)) {
|
|
93
|
-
const result = resolver.resolve(value);
|
|
94
|
-
if (result !== null) {
|
|
95
|
-
return result.value;
|
|
96
|
-
}
|
|
97
|
-
const defaultValue = extractDefaultValue(value);
|
|
98
|
-
if (defaultValue !== undefined) {
|
|
99
|
-
return defaultValue;
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
return value;
|
|
103
|
-
}
|
|
104
|
-
if (Array.isArray(value)) {
|
|
105
|
-
return value.map((item) => substituteProtocolRefs(item, resolver));
|
|
106
|
-
}
|
|
107
|
-
if (value !== null && typeof value === "object") {
|
|
108
|
-
const result = {};
|
|
109
|
-
for (const [key, val] of Object.entries(value)) {
|
|
110
|
-
result[key] = substituteProtocolRefs(val, resolver);
|
|
111
|
-
}
|
|
112
|
-
return result;
|
|
113
|
-
}
|
|
114
|
-
return value;
|
|
115
|
-
}
|
|
116
|
-
function parseJSONCWithProtocols(content, resolver) {
|
|
117
|
-
const parsed = parseJSONC(content);
|
|
118
|
-
const withProtocols = substituteProtocolRefs(parsed, resolver);
|
|
119
|
-
return substituteEnvVars(withProtocols);
|
|
120
|
-
}
|
|
121
|
-
|
|
122
16
|
// src/config/file-source.ts
|
|
123
17
|
import * as fsSync from "fs";
|
|
124
18
|
import * as fsPromises from "fs/promises";
|
|
@@ -1054,4 +948,4 @@ class ConfigComponent extends BaseComponent {
|
|
|
1054
948
|
}
|
|
1055
949
|
}
|
|
1056
950
|
|
|
1057
|
-
export {
|
|
951
|
+
export { FileSource, ConfigSourceNotFoundError, ConfigFileNotFoundError, MemorySource, ConfigComponent };
|
|
@@ -3,7 +3,8 @@ import {
|
|
|
3
3
|
resetUnderstandImageLLMProvider,
|
|
4
4
|
setUnderstandImageLLMProvider,
|
|
5
5
|
understandImageTool
|
|
6
|
-
} from "./roy-agent-core-
|
|
6
|
+
} from "./roy-agent-core-xwm8zavb.js";
|
|
7
|
+
import"./roy-agent-core-zjdzk1gg.js";
|
|
7
8
|
import"./roy-agent-core-fs0mn2jk.js";
|
|
8
9
|
export {
|
|
9
10
|
understandImageTool,
|
|
@@ -1,22 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
truncateOutputInline
|
|
3
|
-
} from "./roy-agent-core-
|
|
4
|
-
import {
|
|
5
|
-
enqueueToAddFakeUserMessage
|
|
6
|
-
} from "./roy-agent-core-t3yqzgse.js";
|
|
7
|
-
import {
|
|
8
|
-
AskUserError,
|
|
9
|
-
init_workflow_hil
|
|
10
|
-
} from "./roy-agent-core-e25xkv53.js";
|
|
3
|
+
} from "./roy-agent-core-71wde8ce.js";
|
|
11
4
|
import {
|
|
12
5
|
AgentRegistry
|
|
13
6
|
} from "./roy-agent-core-x0yztmm6.js";
|
|
14
7
|
import {
|
|
15
|
-
|
|
16
|
-
} from "./roy-agent-core-
|
|
17
|
-
import {
|
|
18
|
-
ContextError
|
|
19
|
-
} from "./roy-agent-core-ctdhjv68.js";
|
|
8
|
+
enqueueToAddFakeUserMessage
|
|
9
|
+
} from "./roy-agent-core-t3yqzgse.js";
|
|
20
10
|
import {
|
|
21
11
|
SessionMessageConverter
|
|
22
12
|
} from "./roy-agent-core-zcx8nf4a.js";
|
|
@@ -24,6 +14,16 @@ import {
|
|
|
24
14
|
safeParseToolArgs,
|
|
25
15
|
withTimeout
|
|
26
16
|
} from "./roy-agent-core-2xb3f3ye.js";
|
|
17
|
+
import {
|
|
18
|
+
getCurrentBgTaskId
|
|
19
|
+
} from "./roy-agent-core-c1v263jn.js";
|
|
20
|
+
import {
|
|
21
|
+
AskUserError,
|
|
22
|
+
init_workflow_hil
|
|
23
|
+
} from "./roy-agent-core-e25xkv53.js";
|
|
24
|
+
import {
|
|
25
|
+
ContextError
|
|
26
|
+
} from "./roy-agent-core-ctdhjv68.js";
|
|
27
27
|
import {
|
|
28
28
|
envKeyToConfigKey
|
|
29
29
|
} from "./roy-agent-core-7hh0brvs.js";
|
|
@@ -585,7 +585,7 @@ class AgentComponent extends BaseComponent {
|
|
|
585
585
|
this.toolComponent = tool ? tool : null;
|
|
586
586
|
if (this.llmComponent) {
|
|
587
587
|
try {
|
|
588
|
-
const mod = await import("./roy-agent-core-
|
|
588
|
+
const mod = await import("./roy-agent-core-krapd8x2.js");
|
|
589
589
|
mod.setUnderstandImageLLMProvider(async (messages) => {
|
|
590
590
|
const result = await this.llmComponent.invoke({
|
|
591
591
|
messages,
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import {
|
|
2
|
+
parseImageInput
|
|
3
|
+
} from "./roy-agent-core-zjdzk1gg.js";
|
|
4
|
+
|
|
5
|
+
// src/env/tool/built-in/understand-image.ts
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
var injectedProvider = null;
|
|
8
|
+
function setUnderstandImageLLMProvider(provider) {
|
|
9
|
+
injectedProvider = provider;
|
|
10
|
+
}
|
|
11
|
+
function getUnderstandImageLLMProvider() {
|
|
12
|
+
return injectedProvider;
|
|
13
|
+
}
|
|
14
|
+
async function resetUnderstandImageLLMProvider() {
|
|
15
|
+
injectedProvider = null;
|
|
16
|
+
}
|
|
17
|
+
function normalizeImageSource(raw) {
|
|
18
|
+
if (typeof raw !== "string")
|
|
19
|
+
return raw;
|
|
20
|
+
return raw.startsWith("@") ? raw.slice(1) : raw;
|
|
21
|
+
}
|
|
22
|
+
var understandImageTool = {
|
|
23
|
+
name: "understand_image",
|
|
24
|
+
description: "Analyze or describe the contents of an image using a multimodal LLM. " + "Provide a prompt describing what you want to extract or understand " + "(e.g. 'Describe this image in detail', 'Read the text in the receipt', " + "'What color is the car?'). The image_source can be an HTTP(S) URL, a " + "data:base64 URL, or a local file path (optionally prefixed with '@').",
|
|
25
|
+
parameters: z.object({
|
|
26
|
+
prompt: z.string().min(1).describe("A text prompt describing what you want to analyze or extract from the image."),
|
|
27
|
+
image_source: z.string().min(1).describe("The location of the image to analyze. Accepts HTTP(S) URL, data:base64 URL, or local file path. Optional '@' prefix is stripped.")
|
|
28
|
+
}),
|
|
29
|
+
metadata: {
|
|
30
|
+
category: "ai",
|
|
31
|
+
tags: ["multimodal", "vision", "image", "llm", "ocr"],
|
|
32
|
+
version: "1.0.0"
|
|
33
|
+
},
|
|
34
|
+
permission: {
|
|
35
|
+
level: "safe"
|
|
36
|
+
},
|
|
37
|
+
execute: async (args, _ctx) => {
|
|
38
|
+
const startTime = Date.now();
|
|
39
|
+
const { prompt, image_source } = args;
|
|
40
|
+
if (!injectedProvider) {
|
|
41
|
+
return {
|
|
42
|
+
success: false,
|
|
43
|
+
output: "",
|
|
44
|
+
error: "understand_image is not initialized: no LLM provider has been " + "registered. AgentComponent should call setUnderstandImageLLMProvider() " + "at agent startup.",
|
|
45
|
+
metadata: { execution_time_ms: Date.now() - startTime }
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
let imagePart;
|
|
49
|
+
try {
|
|
50
|
+
imagePart = await parseImageInput(normalizeImageSource(image_source));
|
|
51
|
+
} catch (err) {
|
|
52
|
+
return {
|
|
53
|
+
success: false,
|
|
54
|
+
output: "",
|
|
55
|
+
error: `failed to read image: ${err.message}`,
|
|
56
|
+
metadata: { execution_time_ms: Date.now() - startTime }
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
const messages = [
|
|
60
|
+
{
|
|
61
|
+
role: "user",
|
|
62
|
+
content: [
|
|
63
|
+
{ type: "text", text: prompt },
|
|
64
|
+
imagePart
|
|
65
|
+
]
|
|
66
|
+
}
|
|
67
|
+
];
|
|
68
|
+
let responseText;
|
|
69
|
+
try {
|
|
70
|
+
responseText = await injectedProvider(messages);
|
|
71
|
+
} catch (err) {
|
|
72
|
+
return {
|
|
73
|
+
success: false,
|
|
74
|
+
output: "",
|
|
75
|
+
error: `LLM invocation failed: ${err.message}`,
|
|
76
|
+
metadata: { execution_time_ms: Date.now() - startTime }
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
success: true,
|
|
81
|
+
output: responseText,
|
|
82
|
+
metadata: {
|
|
83
|
+
execution_time_ms: Date.now() - startTime
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
export { setUnderstandImageLLMProvider, getUnderstandImageLLMProvider, resetUnderstandImageLLMProvider, understandImageTool };
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
// src/env/tool/built-in/understand-image.ts
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
|
|
4
1
|
// src/utils/media.ts
|
|
5
2
|
import * as fs from "node:fs/promises";
|
|
6
3
|
import * as path from "node:path";
|
|
@@ -15,6 +12,9 @@ function startsWith(buf, sig) {
|
|
|
15
12
|
return true;
|
|
16
13
|
}
|
|
17
14
|
function sniffMimeType(buffer, fallbackName) {
|
|
15
|
+
return sniffMimeFromBuffer(buffer, fallbackName);
|
|
16
|
+
}
|
|
17
|
+
function sniffMimeFromBuffer(buffer, fallbackName) {
|
|
18
18
|
if (startsWith(buffer, [137, 80, 78, 71, 13, 10, 26, 10])) {
|
|
19
19
|
return "image/png";
|
|
20
20
|
}
|
|
@@ -98,88 +98,38 @@ async function parseLocalImagePath(filePath) {
|
|
|
98
98
|
}
|
|
99
99
|
return { type: "image", image: buffer };
|
|
100
100
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
}
|
|
110
|
-
async function resetUnderstandImageLLMProvider() {
|
|
111
|
-
injectedProvider = null;
|
|
112
|
-
}
|
|
113
|
-
function normalizeImageSource(raw) {
|
|
114
|
-
if (typeof raw !== "string")
|
|
115
|
-
return raw;
|
|
116
|
-
return raw.startsWith("@") ? raw.slice(1) : raw;
|
|
117
|
-
}
|
|
118
|
-
var understandImageTool = {
|
|
119
|
-
name: "understand_image",
|
|
120
|
-
description: "Analyze or describe the contents of an image using a multimodal LLM. " + "Provide a prompt describing what you want to extract or understand " + "(e.g. 'Describe this image in detail', 'Read the text in the receipt', " + "'What color is the car?'). The image_source can be an HTTP(S) URL, a " + "data:base64 URL, or a local file path (optionally prefixed with '@').",
|
|
121
|
-
parameters: z.object({
|
|
122
|
-
prompt: z.string().min(1).describe("A text prompt describing what you want to analyze or extract from the image."),
|
|
123
|
-
image_source: z.string().min(1).describe("The location of the image to analyze. Accepts HTTP(S) URL, data:base64 URL, or local file path. Optional '@' prefix is stripped.")
|
|
124
|
-
}),
|
|
125
|
-
metadata: {
|
|
126
|
-
category: "ai",
|
|
127
|
-
tags: ["multimodal", "vision", "image", "llm", "ocr"],
|
|
128
|
-
version: "1.0.0"
|
|
129
|
-
},
|
|
130
|
-
permission: {
|
|
131
|
-
level: "safe"
|
|
132
|
-
},
|
|
133
|
-
execute: async (args, _ctx) => {
|
|
134
|
-
const startTime = Date.now();
|
|
135
|
-
const { prompt, image_source } = args;
|
|
136
|
-
if (!injectedProvider) {
|
|
137
|
-
return {
|
|
138
|
-
success: false,
|
|
139
|
-
output: "",
|
|
140
|
-
error: "understand_image is not initialized: no LLM provider has been " + "registered. AgentComponent should call setUnderstandImageLLMProvider() " + "at agent startup.",
|
|
141
|
-
metadata: { execution_time_ms: Date.now() - startTime }
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
let imagePart;
|
|
145
|
-
try {
|
|
146
|
-
imagePart = await parseImageInput(normalizeImageSource(image_source));
|
|
147
|
-
} catch (err) {
|
|
148
|
-
return {
|
|
149
|
-
success: false,
|
|
150
|
-
output: "",
|
|
151
|
-
error: `failed to read image: ${err.message}`,
|
|
152
|
-
metadata: { execution_time_ms: Date.now() - startTime }
|
|
153
|
-
};
|
|
101
|
+
function sniffImageMime(image, explicitMediaType) {
|
|
102
|
+
if (explicitMediaType && typeof explicitMediaType === "string" && explicitMediaType !== "image/*" && explicitMediaType.length > 0) {
|
|
103
|
+
return explicitMediaType;
|
|
104
|
+
}
|
|
105
|
+
if (typeof image === "string" && image.startsWith("data:")) {
|
|
106
|
+
const match = image.match(/^data:([^;,]+)/);
|
|
107
|
+
if (match && match[1]) {
|
|
108
|
+
return match[1];
|
|
154
109
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
return
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
110
|
+
}
|
|
111
|
+
if (typeof image === "string" && /^https?:\/\//i.test(image)) {
|
|
112
|
+
const m = image.match(/\.([a-zA-Z0-9]+)(?:\?|$)/);
|
|
113
|
+
const ext = m?.[1]?.toLowerCase();
|
|
114
|
+
if (ext === "jpg" || ext === "jpeg")
|
|
115
|
+
return "image/jpeg";
|
|
116
|
+
if (ext === "png")
|
|
117
|
+
return "image/png";
|
|
118
|
+
if (ext === "gif")
|
|
119
|
+
return "image/gif";
|
|
120
|
+
if (ext === "webp")
|
|
121
|
+
return "image/webp";
|
|
122
|
+
if (ext === "bmp")
|
|
123
|
+
return "image/bmp";
|
|
124
|
+
}
|
|
125
|
+
if (image instanceof Uint8Array) {
|
|
126
|
+
const detected = sniffMimeFromBuffer(image);
|
|
127
|
+
if (detected && detected !== "application/octet-stream") {
|
|
128
|
+
return detected;
|
|
174
129
|
}
|
|
175
|
-
return {
|
|
176
|
-
success: true,
|
|
177
|
-
output: responseText,
|
|
178
|
-
metadata: {
|
|
179
|
-
execution_time_ms: Date.now() - startTime
|
|
180
|
-
}
|
|
181
|
-
};
|
|
182
130
|
}
|
|
183
|
-
}
|
|
131
|
+
const desc = typeof image === "string" ? `URL "${image.slice(0, 60)}${image.length > 60 ? "..." : ""}"` : `buffer (${image.length} bytes, no recognized magic bytes)`;
|
|
132
|
+
throw new Error(`sniffImageMime: cannot determine image MIME from ${desc}. ` + `Provide an explicit mediaType (e.g. "image/jpeg") or use a data: URL with explicit MIME prefix.`);
|
|
133
|
+
}
|
|
184
134
|
|
|
185
|
-
export { ATTACH_FILE_MAX_BYTES, sniffMimeType, isMediaType, parseImageInput,
|
|
135
|
+
export { ATTACH_FILE_MAX_BYTES, sniffMimeType, isMediaType, parseImageInput, sniffImageMime };
|
package/package.json
CHANGED
|
@@ -2,6 +2,9 @@ import {
|
|
|
2
2
|
ProcessRegistry,
|
|
3
3
|
runWithProcessRegistryAsync
|
|
4
4
|
} from "./roy-agent-core-mvs5sen6.js";
|
|
5
|
+
import {
|
|
6
|
+
TaskHookPoints
|
|
7
|
+
} from "./roy-agent-core-1pg0fepg.js";
|
|
5
8
|
import {
|
|
6
9
|
builtInPrompts
|
|
7
10
|
} from "./roy-agent-core-skqt40yy.js";
|
|
@@ -11,9 +14,6 @@ import {
|
|
|
11
14
|
import {
|
|
12
15
|
runWithBgTaskIdAsync
|
|
13
16
|
} from "./roy-agent-core-c1v263jn.js";
|
|
14
|
-
import {
|
|
15
|
-
TaskHookPoints
|
|
16
|
-
} from "./roy-agent-core-1pg0fepg.js";
|
|
17
17
|
import {
|
|
18
18
|
globalHookManager,
|
|
19
19
|
init_global_hook_manager
|
|
File without changes
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
getFromMapWithNormalization,
|
|
3
|
-
init_node_id_utils
|
|
4
|
-
} from "./roy-agent-core-ddwa0n6h.js";
|
|
5
1
|
import {
|
|
6
2
|
DecoratorNode,
|
|
7
3
|
init_decorator_node
|
|
8
4
|
} from "./roy-agent-core-1ce3fqrk.js";
|
|
5
|
+
import {
|
|
6
|
+
getFromMapWithNormalization,
|
|
7
|
+
init_node_id_utils
|
|
8
|
+
} from "./roy-agent-core-ddwa0n6h.js";
|
|
9
9
|
import {
|
|
10
10
|
TracedAs,
|
|
11
11
|
init_decorator
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
+
import {
|
|
2
|
+
MemorySessionStore,
|
|
3
|
+
SQLiteSessionStore
|
|
4
|
+
} from "./roy-agent-core-wgp3q62x.js";
|
|
1
5
|
import {
|
|
2
6
|
init_search_query_parser,
|
|
3
7
|
matchesQuery,
|
|
4
8
|
parseSearchQuery
|
|
5
9
|
} from "./roy-agent-core-rvxg1wps.js";
|
|
6
|
-
import {
|
|
7
|
-
MemorySessionStore,
|
|
8
|
-
SQLiteSessionStore
|
|
9
|
-
} from "./roy-agent-core-wgp3q62x.js";
|
|
10
10
|
import {
|
|
11
11
|
envKeyToConfigKey
|
|
12
12
|
} from "./roy-agent-core-7hh0brvs.js";
|