@downcity/plugins 1.0.96 → 1.0.108
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/asr/Plugin.d.ts +1 -15
- package/bin/asr/Plugin.d.ts.map +1 -1
- package/bin/asr/Plugin.js +36 -5
- package/bin/asr/Plugin.js.map +1 -1
- package/bin/chat/runtime/ChatAuthorizationRuntime.d.ts.map +1 -1
- package/bin/chat/runtime/ChatAuthorizationRuntime.js +53 -12
- package/bin/chat/runtime/ChatAuthorizationRuntime.js.map +1 -1
- package/bin/chat/runtime/ChatPluginActionRegistry.d.ts.map +1 -1
- package/bin/chat/runtime/ChatPluginActionRegistry.js +114 -42
- package/bin/chat/runtime/ChatPluginActionRegistry.js.map +1 -1
- package/bin/contact/Action.d.ts.map +1 -1
- package/bin/contact/Action.js +159 -37
- package/bin/contact/Action.js.map +1 -1
- package/bin/image/ImagePlugin.d.ts +14 -67
- package/bin/image/ImagePlugin.d.ts.map +1 -1
- package/bin/image/ImagePlugin.js +432 -151
- package/bin/image/ImagePlugin.js.map +1 -1
- package/bin/image/types/ImagePlugin.d.ts +87 -32
- package/bin/image/types/ImagePlugin.d.ts.map +1 -1
- package/bin/image/types/ImagePlugin.js +1 -1
- package/bin/index.d.ts +1 -1
- package/bin/index.d.ts.map +1 -1
- package/bin/memory/MemoryPlugin.d.ts.map +1 -1
- package/bin/memory/MemoryPlugin.js +130 -17
- package/bin/memory/MemoryPlugin.js.map +1 -1
- package/bin/skill/Plugin.d.ts.map +1 -1
- package/bin/skill/Plugin.js +90 -11
- package/bin/skill/Plugin.js.map +1 -1
- package/bin/task/runtime/TaskPluginActionRegistry.d.ts.map +1 -1
- package/bin/task/runtime/TaskPluginActionRegistry.js +202 -24
- package/bin/task/runtime/TaskPluginActionRegistry.js.map +1 -1
- package/bin/tts/Plugin.d.ts +1 -15
- package/bin/tts/Plugin.d.ts.map +1 -1
- package/bin/tts/Plugin.js +38 -5
- package/bin/tts/Plugin.js.map +1 -1
- package/bin/web/Plugin.d.ts +2 -19
- package/bin/web/Plugin.d.ts.map +1 -1
- package/bin/web/Plugin.js +39 -5
- package/bin/web/Plugin.js.map +1 -1
- package/bin/workboard/Plugin.d.ts.map +1 -1
- package/bin/workboard/Plugin.js +10 -2
- package/bin/workboard/Plugin.js.map +1 -1
- package/package.json +3 -3
- package/src/asr/Plugin.ts +37 -5
- package/src/chat/runtime/ChatAuthorizationRuntime.ts +53 -12
- package/src/chat/runtime/ChatPluginActionRegistry.ts +114 -42
- package/src/contact/Action.ts +159 -37
- package/src/image/ImagePlugin.ts +477 -222
- package/src/image/types/ImagePlugin.ts +91 -32
- package/src/index.ts +3 -1
- package/src/memory/MemoryPlugin.ts +130 -17
- package/src/skill/Plugin.ts +101 -21
- package/src/task/runtime/TaskPluginActionRegistry.ts +209 -24
- package/src/tts/Plugin.ts +39 -5
- package/src/web/Plugin.ts +39 -5
- package/src/workboard/Plugin.ts +10 -2
package/src/image/ImagePlugin.ts
CHANGED
|
@@ -2,11 +2,15 @@
|
|
|
2
2
|
* ImagePlugin:图片生成插件。
|
|
3
3
|
*
|
|
4
4
|
* 关键点(中文)
|
|
5
|
-
* - 对 Agent
|
|
5
|
+
* - 对 Agent 暴露 `image_create` / `image_result` 两步式任务 action。
|
|
6
6
|
* - City / provider 的图片能力通过 image_create / image_result 任务函数注入。
|
|
7
|
-
* -
|
|
7
|
+
* - 成功结果返回 AI SDK UIMessage,后续由 plugin tool bridge 抽取 file parts 写回 assistant 消息。
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import fs from "node:fs/promises";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
import { createAction } from "@downcity/agent/internal/plugin/core/PluginActionFactory.js";
|
|
10
14
|
import { BasePlugin } from "@downcity/agent/internal/plugin/core/BasePlugin.js";
|
|
11
15
|
import type { AgentContext } from "@downcity/agent/internal/types/runtime/agent/AgentContext.js";
|
|
12
16
|
import type {
|
|
@@ -18,9 +22,12 @@ import type {
|
|
|
18
22
|
ImagePluginJobCreateResult,
|
|
19
23
|
ImagePluginJobResult,
|
|
20
24
|
ImagePluginJobResultInput,
|
|
25
|
+
ImagePluginContent,
|
|
21
26
|
ImagePluginModel,
|
|
22
27
|
ImagePluginModelsResult,
|
|
23
28
|
ImagePluginOptions,
|
|
29
|
+
ImagePluginResolvedContent,
|
|
30
|
+
ImagePluginResolvedInput,
|
|
24
31
|
ImagePluginResult,
|
|
25
32
|
} from "@/image/types/ImagePlugin.js";
|
|
26
33
|
|
|
@@ -28,9 +35,116 @@ const DEFAULT_IMAGE_PLUGIN_NAME = "image";
|
|
|
28
35
|
const DEFAULT_IMAGE_PLUGIN_TITLE = "Image";
|
|
29
36
|
const DEFAULT_IMAGE_PLUGIN_DESCRIPTION =
|
|
30
37
|
"Generate images and return them as assistant file parts.";
|
|
31
|
-
const
|
|
32
|
-
const
|
|
33
|
-
|
|
38
|
+
const HTTP_URL_RE = /^https?:\/\//i;
|
|
39
|
+
const DEFAULT_IMAGE_MEDIA_TYPE = "image/png";
|
|
40
|
+
/**
|
|
41
|
+
* `image_result` 阻塞等待时的默认参数。
|
|
42
|
+
*
|
|
43
|
+
* 关键点(中文)
|
|
44
|
+
* - DEFAULT_IMAGE_WAIT_MS:总等待上限(毫秒),避免 agent 单次调用挂太久。
|
|
45
|
+
* - DEFAULT_IMAGE_POLL_MS:相邻两次轮询的最小间隔(毫秒)。
|
|
46
|
+
* - MAX_IMAGE_WAIT_MS:用户参数硬上限,防止异常值导致 agent 永远等下去。
|
|
47
|
+
*/
|
|
48
|
+
const DEFAULT_IMAGE_WAIT_MS = 60_000;
|
|
49
|
+
const DEFAULT_IMAGE_POLL_MS = 1_500;
|
|
50
|
+
const MAX_IMAGE_WAIT_MS = 10 * 60_000;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* 判断任务状态是否已到达终态。
|
|
54
|
+
*/
|
|
55
|
+
function is_terminal_status(status: string | undefined): boolean {
|
|
56
|
+
return status === "succeeded" || status === "failed";
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* sleep 工具。
|
|
61
|
+
*/
|
|
62
|
+
function delay_ms(ms: number): Promise<void> {
|
|
63
|
+
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* 把外部传入的等待参数夹到合法区间。
|
|
68
|
+
*/
|
|
69
|
+
function clamp_wait_ms(value: number): number {
|
|
70
|
+
if (!Number.isFinite(value) || value < 0) return 0;
|
|
71
|
+
if (value > MAX_IMAGE_WAIT_MS) return MAX_IMAGE_WAIT_MS;
|
|
72
|
+
return Math.floor(value);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* 把异常完整描述为字符串,保留 `error.cause` 链上的诊断信息。
|
|
77
|
+
*
|
|
78
|
+
* 关键点(中文)
|
|
79
|
+
* - Node fetch 抛出的 `TypeError: fetch failed` 把真正的根因放在 `error.cause`;
|
|
80
|
+
* `String(error)` 只会拿到 message,会让上层只看到一个干瘪的 "fetch failed"。
|
|
81
|
+
* - 这里递归读取 cause 链,并取每一层的 `code` + `message`,方便在 agent 输出里直接定位问题。
|
|
82
|
+
*/
|
|
83
|
+
function describe_error(error: unknown): string {
|
|
84
|
+
if (!(error instanceof Error)) return String(error);
|
|
85
|
+
const parts: string[] = [error.message || error.name || "Error"];
|
|
86
|
+
let current: unknown = (error as { cause?: unknown }).cause;
|
|
87
|
+
let depth = 0;
|
|
88
|
+
while (current && depth < 3) {
|
|
89
|
+
if (current instanceof Error) {
|
|
90
|
+
const code = (current as { code?: unknown }).code;
|
|
91
|
+
const code_text = typeof code === "string" && code ? `[${code}] ` : "";
|
|
92
|
+
parts.push(`${code_text}${current.message || current.name}`.trim());
|
|
93
|
+
current = (current as { cause?: unknown }).cause;
|
|
94
|
+
} else {
|
|
95
|
+
parts.push(String(current));
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
depth += 1;
|
|
99
|
+
}
|
|
100
|
+
return parts.filter(Boolean).join(" :: ");
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const IMAGE_MEDIA_TYPES: Record<string, string> = {
|
|
104
|
+
".apng": "image/apng",
|
|
105
|
+
".avif": "image/avif",
|
|
106
|
+
".gif": "image/gif",
|
|
107
|
+
".jpg": "image/jpeg",
|
|
108
|
+
".jpeg": "image/jpeg",
|
|
109
|
+
".png": "image/png",
|
|
110
|
+
".webp": "image/webp",
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
const IMAGE_TEXT_CONTENT_SCHEMA = z.object({
|
|
114
|
+
type: z.literal("text"),
|
|
115
|
+
text: z.string(),
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
const IMAGE_FILE_CONTENT_SCHEMA = z.object({
|
|
119
|
+
type: z.literal("image"),
|
|
120
|
+
url: z.string(),
|
|
121
|
+
media_type: z.string().optional(),
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
const IMAGE_CREATE_INPUT_SCHEMA = z.object({
|
|
125
|
+
model: z.string().optional(),
|
|
126
|
+
prompt: z.string().optional(),
|
|
127
|
+
content: z.array(z.union([
|
|
128
|
+
IMAGE_TEXT_CONTENT_SCHEMA,
|
|
129
|
+
IMAGE_FILE_CONTENT_SCHEMA,
|
|
130
|
+
])).optional(),
|
|
131
|
+
n: z.number().optional(),
|
|
132
|
+
count: z.number().optional(),
|
|
133
|
+
size: z.string().optional(),
|
|
134
|
+
aspect_ratio: z.string().optional(),
|
|
135
|
+
ratio: z.string().optional(),
|
|
136
|
+
quality: z.string().optional(),
|
|
137
|
+
seed: z.number().optional(),
|
|
138
|
+
client_job_id: z.string().optional(),
|
|
139
|
+
provider_options: z.object({}).passthrough().optional(),
|
|
140
|
+
}).passthrough();
|
|
141
|
+
|
|
142
|
+
const IMAGE_RESULT_INPUT_SCHEMA = z.object({
|
|
143
|
+
job_id: z.string(),
|
|
144
|
+
until_done: z.boolean().optional(),
|
|
145
|
+
max_wait_ms: z.number().optional(),
|
|
146
|
+
poll_interval_ms: z.number().optional(),
|
|
147
|
+
}).passthrough();
|
|
34
148
|
|
|
35
149
|
/**
|
|
36
150
|
* 判断值是否为普通对象。
|
|
@@ -53,6 +167,139 @@ function normalize_image_payload(
|
|
|
53
167
|
return { ...record } as ImagePluginInput;
|
|
54
168
|
}
|
|
55
169
|
|
|
170
|
+
/**
|
|
171
|
+
* 根据文件扩展名推断图片 MIME 类型。
|
|
172
|
+
*/
|
|
173
|
+
function infer_image_media_type(file_path: string, fallback?: string): string {
|
|
174
|
+
if (fallback && fallback.trim()) return fallback.trim();
|
|
175
|
+
const ext = path.extname(file_path).toLowerCase();
|
|
176
|
+
return IMAGE_MEDIA_TYPES[ext] ?? DEFAULT_IMAGE_MEDIA_TYPE;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* 解析图片本地路径。
|
|
181
|
+
*/
|
|
182
|
+
function resolve_image_file_path(root_path: string, image_url: string): string {
|
|
183
|
+
const raw = image_url.trim();
|
|
184
|
+
if (!raw) throw new TypeError("ImagePlugin image content url is required");
|
|
185
|
+
return path.isAbsolute(raw) ? raw : path.resolve(root_path, raw);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* 把本地图片读取为 data URL。
|
|
190
|
+
*/
|
|
191
|
+
async function local_image_to_data_url(input: {
|
|
192
|
+
/**
|
|
193
|
+
* 当前 Agent 项目根目录。
|
|
194
|
+
*/
|
|
195
|
+
root_path: string;
|
|
196
|
+
/**
|
|
197
|
+
* 本地绝对路径或相对路径。
|
|
198
|
+
*/
|
|
199
|
+
image_url: string;
|
|
200
|
+
/**
|
|
201
|
+
* 可选 MIME 类型。
|
|
202
|
+
*/
|
|
203
|
+
media_type?: string;
|
|
204
|
+
}): Promise<{ data_url: string; media_type: string }> {
|
|
205
|
+
const file_path = resolve_image_file_path(input.root_path, input.image_url);
|
|
206
|
+
const media_type = infer_image_media_type(file_path, input.media_type);
|
|
207
|
+
const bytes = await fs.readFile(file_path);
|
|
208
|
+
return {
|
|
209
|
+
data_url: `${media_type.includes("/") ? `data:${media_type};base64,` : "data:image/png;base64,"}${bytes.toString("base64")}`,
|
|
210
|
+
media_type,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* 归一化单个图片内容片段。
|
|
216
|
+
*/
|
|
217
|
+
async function normalize_image_content_part(
|
|
218
|
+
context: AgentContext,
|
|
219
|
+
part: ImagePluginContent,
|
|
220
|
+
): Promise<ImagePluginResolvedContent> {
|
|
221
|
+
if (part.type === "text") return part;
|
|
222
|
+
const url = String(part.url || "").trim();
|
|
223
|
+
if (!url) throw new TypeError("ImagePlugin image content url is required");
|
|
224
|
+
if (url.startsWith("data:")) {
|
|
225
|
+
throw new TypeError(
|
|
226
|
+
"ImagePlugin content image url does not accept data URLs; pass an online URL or a local file path",
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
if (HTTP_URL_RE.test(url)) {
|
|
230
|
+
return {
|
|
231
|
+
type: "image",
|
|
232
|
+
url,
|
|
233
|
+
...(part.media_type ? { media_type: part.media_type } : {}),
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
const local = await local_image_to_data_url({
|
|
237
|
+
root_path: context.rootPath,
|
|
238
|
+
image_url: url,
|
|
239
|
+
media_type: part.media_type,
|
|
240
|
+
});
|
|
241
|
+
return {
|
|
242
|
+
type: "image",
|
|
243
|
+
data_url: local.data_url,
|
|
244
|
+
media_type: local.media_type,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* 拒绝旧版或内部协议字段,避免 Agent 继续依赖兼容层。
|
|
250
|
+
*/
|
|
251
|
+
function assert_public_image_create_input(input: ImagePluginInput): void {
|
|
252
|
+
const record = input as Record<string, unknown>;
|
|
253
|
+
if ("messages" in record) {
|
|
254
|
+
throw new TypeError("ImagePlugin image_create uses prompt or content; messages is not supported");
|
|
255
|
+
}
|
|
256
|
+
const content = record.content;
|
|
257
|
+
if (!Array.isArray(content)) return;
|
|
258
|
+
for (const part of content) {
|
|
259
|
+
const part_record = to_record(part);
|
|
260
|
+
if (part_record && "data_url" in part_record) {
|
|
261
|
+
throw new TypeError(
|
|
262
|
+
"ImagePlugin content image uses url only; data_url is not supported",
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* 复制公开输入中的通用字段,剥离 Agent 不应传给下游的公开 content。
|
|
270
|
+
*/
|
|
271
|
+
function copy_resolved_image_input(input: ImagePluginInput): ImagePluginResolvedInput {
|
|
272
|
+
const { content: _content, messages: _messages, ...rest } = input as ImagePluginInput & {
|
|
273
|
+
/** 旧版字段,显式丢弃。 */
|
|
274
|
+
messages?: unknown;
|
|
275
|
+
};
|
|
276
|
+
return rest as ImagePluginResolvedInput;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* 把 Agent 友好的公开输入转成 City 图片任务使用的输入。
|
|
281
|
+
*/
|
|
282
|
+
async function normalize_image_create_input(
|
|
283
|
+
context: AgentContext,
|
|
284
|
+
input: ImagePluginInput,
|
|
285
|
+
): Promise<ImagePluginResolvedInput> {
|
|
286
|
+
assert_public_image_create_input(input);
|
|
287
|
+
if (!Array.isArray(input.content)) return copy_resolved_image_input(input);
|
|
288
|
+
const content = await Promise.all(
|
|
289
|
+
input.content.map((part) => normalize_image_content_part(context, part)),
|
|
290
|
+
);
|
|
291
|
+
const { prompt: _prompt, ...rest } = copy_resolved_image_input(input);
|
|
292
|
+
return {
|
|
293
|
+
...rest,
|
|
294
|
+
messages: [
|
|
295
|
+
{
|
|
296
|
+
role: "user",
|
|
297
|
+
content,
|
|
298
|
+
},
|
|
299
|
+
],
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
|
|
56
303
|
/**
|
|
57
304
|
* 归一化图片任务查询 payload。
|
|
58
305
|
*/
|
|
@@ -67,9 +314,22 @@ function normalize_image_result_payload(
|
|
|
67
314
|
if (!job_id) {
|
|
68
315
|
throw new TypeError("ImagePlugin.image_result payload must include job_id");
|
|
69
316
|
}
|
|
317
|
+
const until_done = record.until_done === true;
|
|
318
|
+
const max_wait_ms =
|
|
319
|
+
typeof record.max_wait_ms === "number" && Number.isFinite(record.max_wait_ms)
|
|
320
|
+
? Math.max(0, Math.floor(record.max_wait_ms as number))
|
|
321
|
+
: undefined;
|
|
322
|
+
const poll_interval_ms =
|
|
323
|
+
typeof record.poll_interval_ms === "number" &&
|
|
324
|
+
Number.isFinite(record.poll_interval_ms)
|
|
325
|
+
? Math.max(0, Math.floor(record.poll_interval_ms as number))
|
|
326
|
+
: undefined;
|
|
70
327
|
return {
|
|
71
328
|
...record,
|
|
72
329
|
job_id,
|
|
330
|
+
...(until_done ? { until_done: true } : {}),
|
|
331
|
+
...(max_wait_ms !== undefined ? { max_wait_ms } : {}),
|
|
332
|
+
...(poll_interval_ms !== undefined ? { poll_interval_ms } : {}),
|
|
73
333
|
} as ImagePluginJobResultInput;
|
|
74
334
|
}
|
|
75
335
|
|
|
@@ -84,41 +344,6 @@ function normalize_image_result(result: ImagePluginResult): ImagePluginResult {
|
|
|
84
344
|
return result;
|
|
85
345
|
}
|
|
86
346
|
|
|
87
|
-
/**
|
|
88
|
-
* 等待指定毫秒数。
|
|
89
|
-
*/
|
|
90
|
-
function sleep(ms: number): Promise<void> {
|
|
91
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/**
|
|
95
|
-
* 限制轮询间隔,避免服务端异常值导致过快或过慢轮询。
|
|
96
|
-
*/
|
|
97
|
-
function clamp_poll_interval(
|
|
98
|
-
value: unknown,
|
|
99
|
-
min_ms: number,
|
|
100
|
-
max_ms: number,
|
|
101
|
-
): number {
|
|
102
|
-
const n = typeof value === "number" && Number.isFinite(value) ? value : min_ms;
|
|
103
|
-
return Math.max(min_ms, Math.min(max_ms, n));
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
/**
|
|
107
|
-
* 归一化正数配置。
|
|
108
|
-
*/
|
|
109
|
-
function normalize_positive_number(value: unknown, fallback: number): number {
|
|
110
|
-
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
111
|
-
? value
|
|
112
|
-
: fallback;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
/**
|
|
116
|
-
* 读取可选布尔值。
|
|
117
|
-
*/
|
|
118
|
-
function normalize_boolean(value: unknown, fallback: boolean): boolean {
|
|
119
|
-
return typeof value === "boolean" ? value : fallback;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
347
|
/**
|
|
123
348
|
* 归一化模型元数据为 JSON 对象。
|
|
124
349
|
*/
|
|
@@ -239,9 +464,7 @@ export class ImagePlugin extends BasePlugin {
|
|
|
239
464
|
private readonly image_create: NonNullable<ImagePluginOptions["image_create"]>;
|
|
240
465
|
private readonly image_result: NonNullable<ImagePluginOptions["image_result"]>;
|
|
241
466
|
private readonly list_models?: ImagePluginOptions["list_models"];
|
|
242
|
-
private
|
|
243
|
-
private readonly min_poll_interval_ms: number;
|
|
244
|
-
private readonly max_poll_interval_ms: number;
|
|
467
|
+
private default_model_id?: string;
|
|
245
468
|
|
|
246
469
|
constructor(options: ImagePluginOptions) {
|
|
247
470
|
super();
|
|
@@ -263,18 +486,6 @@ export class ImagePlugin extends BasePlugin {
|
|
|
263
486
|
this.image_create = options.image_create;
|
|
264
487
|
this.image_result = options.image_result;
|
|
265
488
|
this.list_models = options.list_models;
|
|
266
|
-
this.timeout_ms = normalize_positive_number(
|
|
267
|
-
options.timeout_ms,
|
|
268
|
-
DEFAULT_TIMEOUT_MS,
|
|
269
|
-
);
|
|
270
|
-
this.min_poll_interval_ms = normalize_positive_number(
|
|
271
|
-
options.min_poll_interval_ms,
|
|
272
|
-
DEFAULT_MIN_POLL_INTERVAL_MS,
|
|
273
|
-
);
|
|
274
|
-
this.max_poll_interval_ms = normalize_positive_number(
|
|
275
|
-
options.max_poll_interval_ms,
|
|
276
|
-
DEFAULT_MAX_POLL_INTERVAL_MS,
|
|
277
|
-
);
|
|
278
489
|
}
|
|
279
490
|
|
|
280
491
|
/**
|
|
@@ -284,164 +495,107 @@ export class ImagePlugin extends BasePlugin {
|
|
|
284
495
|
return [
|
|
285
496
|
"# Image Plugin",
|
|
286
497
|
"",
|
|
287
|
-
"Use this plugin only when the user asks to
|
|
288
|
-
"Do not call it for ordinary
|
|
498
|
+
"Use this plugin only when the user asks to create, edit, or otherwise produce an image.",
|
|
499
|
+
"Do not call it for ordinary text answers, even if the message mentions visual ideas.",
|
|
500
|
+
"",
|
|
501
|
+
"## Actions",
|
|
289
502
|
"",
|
|
290
|
-
"
|
|
503
|
+
"- `models`:列出当前可用的图片模型,返回 `{ models, default_model_id }`。",
|
|
504
|
+
" 仅当你需要让用户挑模型,或当前没有默认模型可用时才调用。",
|
|
505
|
+
"- `image_create`:创建一个异步图片任务,返回 `{ job_id, status }`。",
|
|
506
|
+
" - 纯文本生成:填 `prompt`(字符串)。",
|
|
507
|
+
" - 编辑 / 参考图:填 `content`,格式:",
|
|
508
|
+
" `[{ type: \"text\", text }, { type: \"image\", url }]`。可以多段文本和多张图任意顺序混排。",
|
|
509
|
+
" - 同时给了 `content` 和 `prompt` 时,`content` 生效,`prompt` 被忽略。",
|
|
510
|
+
" - `url` 支持三种写法:在线 URL、绝对本地路径、相对 Agent 项目根目录的相对路径。",
|
|
511
|
+
" 不要传 base64 / data URL,也不要再用旧的 `messages` 字段。",
|
|
512
|
+
" - 可选参数:`model`、`aspect_ratio`(如 `16:9`)、`size`(如 `1024x1024`)、`quality`、`seed`。",
|
|
513
|
+
"- `image_result`:用 `job_id` 读取任务状态。",
|
|
514
|
+
" - 默认读取一次。`queued` / `running` 时保存好 `job_id`,下一轮再查,不要在同一轮里反复轮询。",
|
|
515
|
+
" - 想直接等到出图,可以传 `until_done: true`,可选 `max_wait_ms`(默认 60000,最长 600000)与 `poll_interval_ms`(默认 1500)。超时仍未完成会返回最后一次的中间状态。",
|
|
516
|
+
" - `succeeded`:返回的图片 file part 会自动落盘并附加到下一条 assistant 消息,无需自己再拼图。",
|
|
517
|
+
" - `failed`:把 `error` 信息如实回报给用户,不要编造图片结果。",
|
|
291
518
|
"",
|
|
292
|
-
"
|
|
293
|
-
"plugin_call({",
|
|
294
|
-
` plugin: "${this.name}",`,
|
|
295
|
-
' action: "generate",',
|
|
296
|
-
" payload: {",
|
|
297
|
-
' prompt: "...",',
|
|
298
|
-
" },",
|
|
299
|
-
"});",
|
|
300
|
-
"```",
|
|
519
|
+
"## Flow",
|
|
301
520
|
"",
|
|
302
|
-
"
|
|
303
|
-
"
|
|
304
|
-
"
|
|
305
|
-
"
|
|
306
|
-
"
|
|
307
|
-
"- For a two-step flow, call `image_create` first, then call `image_result` with `job_id`; `image_result` waits until the job finishes by default.",
|
|
308
|
-
"- `generate` is a convenience action that creates the job and waits for the final image in one call.",
|
|
309
|
-
"- Generated image file parts are saved under project `.downcity/resources` and attached to the final assistant message automatically.",
|
|
521
|
+
"1. 需要选模型时先 `models`,否则直接进入第 2 步。",
|
|
522
|
+
"2. `image_create` 拿到 `job_id`。",
|
|
523
|
+
"3. `image_result` 查询;短任务可加 `until_done: true` 一次拿结果,长任务保存 `job_id` 下一轮再查。",
|
|
524
|
+
"",
|
|
525
|
+
"如有疑问可用 `plugin_read { plugin: \"image\", action: \"...\" }` 查看每个 action 完整的输入 schema 与示例。",
|
|
310
526
|
].join("\n");
|
|
311
527
|
}
|
|
312
528
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
529
|
+
/**
|
|
530
|
+
* 当调用方未显式指定模型时,使用模型目录中的图片默认模型。
|
|
531
|
+
*/
|
|
532
|
+
private async with_default_model(
|
|
533
|
+
input: ImagePluginResolvedInput,
|
|
534
|
+
): Promise<ImagePluginResolvedInput> {
|
|
535
|
+
if (typeof input.model === "string" && input.model.trim()) return input;
|
|
536
|
+
const default_model_id = await this.resolve_default_model_id();
|
|
537
|
+
return default_model_id ? { ...input, model: default_model_id } : input;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
private async resolve_default_model_id(): Promise<string | undefined> {
|
|
541
|
+
if (this.default_model_id) return this.default_model_id;
|
|
542
|
+
if (!this.list_models) return undefined;
|
|
543
|
+
const result = normalize_image_models(await this.list_models());
|
|
544
|
+
this.default_model_id = result.default_model_id;
|
|
545
|
+
return this.default_model_id;
|
|
326
546
|
}
|
|
327
547
|
|
|
328
548
|
/**
|
|
329
|
-
*
|
|
549
|
+
* 查询图片任务当前状态。
|
|
330
550
|
*/
|
|
331
551
|
private async read_image_result(
|
|
332
552
|
input: ImagePluginJobResultInput,
|
|
333
|
-
options: {
|
|
334
|
-
/**
|
|
335
|
-
* 默认是否等待终态。
|
|
336
|
-
*/
|
|
337
|
-
default_until_finish: boolean;
|
|
338
|
-
},
|
|
339
553
|
): Promise<ImagePluginJobResult> {
|
|
340
|
-
const
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
);
|
|
349
|
-
const until_finish = normalize_boolean(
|
|
350
|
-
input.until_finish,
|
|
351
|
-
options.default_until_finish,
|
|
554
|
+
const first = await this.fetch_job_once(input.job_id);
|
|
555
|
+
if (!input.until_done) return first;
|
|
556
|
+
if (is_terminal_status(first.status)) return first;
|
|
557
|
+
|
|
558
|
+
const deadline =
|
|
559
|
+
Date.now() + clamp_wait_ms(input.max_wait_ms ?? DEFAULT_IMAGE_WAIT_MS);
|
|
560
|
+
const base_interval = clamp_wait_ms(
|
|
561
|
+
input.poll_interval_ms ?? DEFAULT_IMAGE_POLL_MS,
|
|
352
562
|
);
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
);
|
|
563
|
+
let current = first;
|
|
564
|
+
while (Date.now() < deadline) {
|
|
565
|
+
const provider_hint =
|
|
566
|
+
typeof current.poll_after_ms === "number" && current.poll_after_ms > 0
|
|
567
|
+
? Math.floor(current.poll_after_ms)
|
|
568
|
+
: 0;
|
|
569
|
+
const wait_ms = Math.max(base_interval, provider_hint);
|
|
570
|
+
const remaining = Math.max(0, deadline - Date.now());
|
|
571
|
+
const sleep_ms = Math.min(wait_ms, remaining);
|
|
572
|
+
if (sleep_ms === 0) break;
|
|
573
|
+
await delay_ms(sleep_ms);
|
|
574
|
+
current = await this.fetch_job_once(input.job_id);
|
|
575
|
+
if (is_terminal_status(current.status)) return current;
|
|
365
576
|
}
|
|
366
|
-
return
|
|
367
|
-
job_id: input.job_id,
|
|
368
|
-
poll_after_ms: first.poll_after_ms,
|
|
369
|
-
timeout_ms,
|
|
370
|
-
min_poll_interval_ms,
|
|
371
|
-
max_poll_interval_ms,
|
|
372
|
-
});
|
|
577
|
+
return current;
|
|
373
578
|
}
|
|
374
579
|
|
|
375
580
|
/**
|
|
376
|
-
*
|
|
581
|
+
* 拉取一次任务状态并校验。
|
|
377
582
|
*/
|
|
378
|
-
private async
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
/**
|
|
384
|
-
* 首次建议轮询间隔。
|
|
385
|
-
*/
|
|
386
|
-
poll_after_ms?: number;
|
|
387
|
-
/**
|
|
388
|
-
* 最大等待时间。
|
|
389
|
-
*/
|
|
390
|
-
timeout_ms?: number;
|
|
391
|
-
/**
|
|
392
|
-
* 轮询间隔下限。
|
|
393
|
-
*/
|
|
394
|
-
min_poll_interval_ms?: number;
|
|
395
|
-
/**
|
|
396
|
-
* 轮询间隔上限。
|
|
397
|
-
*/
|
|
398
|
-
max_poll_interval_ms?: number;
|
|
399
|
-
}): Promise<ImagePluginJobResult> {
|
|
400
|
-
const timeout_ms = normalize_positive_number(input.timeout_ms, this.timeout_ms);
|
|
401
|
-
const min_poll_interval_ms = normalize_positive_number(
|
|
402
|
-
input.min_poll_interval_ms,
|
|
403
|
-
this.min_poll_interval_ms,
|
|
404
|
-
);
|
|
405
|
-
const max_poll_interval_ms = normalize_positive_number(
|
|
406
|
-
input.max_poll_interval_ms,
|
|
407
|
-
this.max_poll_interval_ms,
|
|
408
|
-
);
|
|
409
|
-
const deadline = Date.now() + timeout_ms;
|
|
410
|
-
let poll_after_ms = input.poll_after_ms;
|
|
411
|
-
|
|
412
|
-
while (Date.now() < deadline) {
|
|
413
|
-
await sleep(
|
|
414
|
-
clamp_poll_interval(
|
|
415
|
-
poll_after_ms,
|
|
416
|
-
min_poll_interval_ms,
|
|
417
|
-
max_poll_interval_ms,
|
|
418
|
-
),
|
|
419
|
-
);
|
|
420
|
-
const current = await this.image_result({ job_id: input.job_id });
|
|
421
|
-
validate_job_result(current);
|
|
422
|
-
poll_after_ms = current.poll_after_ms;
|
|
423
|
-
if (current.status === "succeeded") {
|
|
424
|
-
if (!current.result) {
|
|
425
|
-
throw new Error(`Image job ${input.job_id} succeeded without result`);
|
|
426
|
-
}
|
|
427
|
-
normalize_image_result(current.result);
|
|
428
|
-
return current;
|
|
429
|
-
}
|
|
430
|
-
if (current.status === "failed") {
|
|
431
|
-
throw new Error(
|
|
432
|
-
`Image job failed: ${current.error ?? current.message ?? input.job_id}`,
|
|
433
|
-
);
|
|
434
|
-
}
|
|
583
|
+
private async fetch_job_once(job_id: string): Promise<ImagePluginJobResult> {
|
|
584
|
+
const current = await this.image_result({ job_id });
|
|
585
|
+
validate_job_result(current);
|
|
586
|
+
if (current.status === "succeeded" && current.result) {
|
|
587
|
+
normalize_image_result(current.result);
|
|
435
588
|
}
|
|
436
|
-
|
|
437
|
-
throw new Error(`Image job timed out: ${input.job_id}`);
|
|
589
|
+
return current;
|
|
438
590
|
}
|
|
439
591
|
|
|
440
592
|
/**
|
|
441
593
|
* 显式 action 集合。
|
|
442
594
|
*/
|
|
443
595
|
readonly actions = {
|
|
444
|
-
models: {
|
|
596
|
+
models: createAction({
|
|
597
|
+
description: "List image-capable models available to ImagePlugin.",
|
|
598
|
+
input_schema: z.object({}).passthrough(),
|
|
445
599
|
execute: async () => {
|
|
446
600
|
try {
|
|
447
601
|
if (!this.list_models) {
|
|
@@ -461,17 +615,85 @@ export class ImagePlugin extends BasePlugin {
|
|
|
461
615
|
} catch (error) {
|
|
462
616
|
return {
|
|
463
617
|
success: false,
|
|
464
|
-
error:
|
|
465
|
-
message:
|
|
618
|
+
error: describe_error(error),
|
|
619
|
+
message: describe_error(error),
|
|
466
620
|
};
|
|
467
621
|
}
|
|
468
622
|
},
|
|
469
|
-
},
|
|
470
|
-
image_create: {
|
|
471
|
-
|
|
623
|
+
}),
|
|
624
|
+
image_create: createAction({
|
|
625
|
+
description:
|
|
626
|
+
"Create an async image job. Use prompt for text-only generation, or content for reference images and edits.",
|
|
627
|
+
input_schema: {
|
|
628
|
+
zod: IMAGE_CREATE_INPUT_SCHEMA,
|
|
629
|
+
json_schema: {
|
|
630
|
+
type: "object",
|
|
631
|
+
additionalProperties: true,
|
|
632
|
+
properties: {
|
|
633
|
+
model: { type: "string", description: "Image model id." },
|
|
634
|
+
prompt: {
|
|
635
|
+
type: "string",
|
|
636
|
+
description: "Text-only image prompt. Ignored when content is present.",
|
|
637
|
+
},
|
|
638
|
+
content: {
|
|
639
|
+
type: "array",
|
|
640
|
+
description: "Multimodal content for image edits or reference images.",
|
|
641
|
+
items: {
|
|
642
|
+
oneOf: [
|
|
643
|
+
{
|
|
644
|
+
type: "object",
|
|
645
|
+
required: ["type", "text"],
|
|
646
|
+
properties: {
|
|
647
|
+
type: { const: "text" },
|
|
648
|
+
text: { type: "string" },
|
|
649
|
+
},
|
|
650
|
+
},
|
|
651
|
+
{
|
|
652
|
+
type: "object",
|
|
653
|
+
required: ["type", "url"],
|
|
654
|
+
properties: {
|
|
655
|
+
type: { const: "image" },
|
|
656
|
+
url: {
|
|
657
|
+
type: "string",
|
|
658
|
+
description:
|
|
659
|
+
"Online URL, absolute local path, or path relative to the Agent project root.",
|
|
660
|
+
},
|
|
661
|
+
media_type: { type: "string" },
|
|
662
|
+
},
|
|
663
|
+
},
|
|
664
|
+
],
|
|
665
|
+
},
|
|
666
|
+
},
|
|
667
|
+
aspect_ratio: { type: "string", description: "Aspect ratio, for example 16:9." },
|
|
668
|
+
size: { type: "string", description: "Image size, for example 1024x1024." },
|
|
669
|
+
quality: { type: "string", description: "Image quality." },
|
|
670
|
+
seed: { type: "number", description: "Random seed." },
|
|
671
|
+
},
|
|
672
|
+
},
|
|
673
|
+
},
|
|
674
|
+
examples: [
|
|
675
|
+
{
|
|
676
|
+
title: "Text-only image",
|
|
677
|
+
payload: {
|
|
678
|
+
prompt: "A cinematic illustration of a rainy city corner at night",
|
|
679
|
+
aspect_ratio: "16:9",
|
|
680
|
+
},
|
|
681
|
+
},
|
|
682
|
+
{
|
|
683
|
+
title: "Edit image with local reference",
|
|
684
|
+
payload: {
|
|
685
|
+
content: [
|
|
686
|
+
{ type: "text", text: "Change this image to a white studio background" },
|
|
687
|
+
{ type: "image", url: "./input.png" },
|
|
688
|
+
],
|
|
689
|
+
},
|
|
690
|
+
},
|
|
691
|
+
],
|
|
692
|
+
execute: async ({ context, payload }: { context: AgentContext; payload: JsonValue }) => {
|
|
472
693
|
try {
|
|
473
694
|
const input = normalize_image_payload(payload);
|
|
474
|
-
const
|
|
695
|
+
const normalized_input = await normalize_image_create_input(context, input);
|
|
696
|
+
const created = await this.image_create(await this.with_default_model(normalized_input));
|
|
475
697
|
validate_created_job(created);
|
|
476
698
|
return {
|
|
477
699
|
success: true,
|
|
@@ -481,19 +703,71 @@ export class ImagePlugin extends BasePlugin {
|
|
|
481
703
|
} catch (error) {
|
|
482
704
|
return {
|
|
483
705
|
success: false,
|
|
484
|
-
error:
|
|
485
|
-
message:
|
|
706
|
+
error: describe_error(error),
|
|
707
|
+
message: describe_error(error),
|
|
486
708
|
};
|
|
487
709
|
}
|
|
488
710
|
},
|
|
489
|
-
},
|
|
490
|
-
image_result: {
|
|
711
|
+
}),
|
|
712
|
+
image_result: createAction({
|
|
713
|
+
description:
|
|
714
|
+
"Read an async image job. By default reads once; pass until_done=true to block-wait until succeeded/failed or max_wait_ms times out.",
|
|
715
|
+
input_schema: {
|
|
716
|
+
zod: IMAGE_RESULT_INPUT_SCHEMA,
|
|
717
|
+
json_schema: {
|
|
718
|
+
type: "object",
|
|
719
|
+
required: ["job_id"],
|
|
720
|
+
properties: {
|
|
721
|
+
job_id: {
|
|
722
|
+
type: "string",
|
|
723
|
+
description: "Image job id returned by image_create.",
|
|
724
|
+
},
|
|
725
|
+
until_done: {
|
|
726
|
+
type: "boolean",
|
|
727
|
+
description:
|
|
728
|
+
"If true, the plugin internally polls until the job reaches a terminal state (succeeded/failed) or max_wait_ms elapses.",
|
|
729
|
+
},
|
|
730
|
+
max_wait_ms: {
|
|
731
|
+
type: "number",
|
|
732
|
+
description:
|
|
733
|
+
"Total wait budget in milliseconds when until_done is true. Default 60000, hard cap 600000.",
|
|
734
|
+
},
|
|
735
|
+
poll_interval_ms: {
|
|
736
|
+
type: "number",
|
|
737
|
+
description:
|
|
738
|
+
"Minimum delay between polls in milliseconds when until_done is true. Default 1500. Provider poll_after_ms overrides when larger.",
|
|
739
|
+
},
|
|
740
|
+
},
|
|
741
|
+
},
|
|
742
|
+
},
|
|
743
|
+
examples: [
|
|
744
|
+
{
|
|
745
|
+
title: "Read image job",
|
|
746
|
+
payload: {
|
|
747
|
+
job_id: "img_123",
|
|
748
|
+
},
|
|
749
|
+
},
|
|
750
|
+
{
|
|
751
|
+
title: "Wait until done",
|
|
752
|
+
payload: {
|
|
753
|
+
job_id: "img_123",
|
|
754
|
+
until_done: true,
|
|
755
|
+
max_wait_ms: 30000,
|
|
756
|
+
},
|
|
757
|
+
},
|
|
758
|
+
],
|
|
491
759
|
execute: async ({ payload }: { payload: JsonValue }) => {
|
|
492
760
|
try {
|
|
493
761
|
const input = normalize_image_result_payload(payload);
|
|
494
|
-
const current = await this.read_image_result(input
|
|
495
|
-
|
|
496
|
-
|
|
762
|
+
const current = await this.read_image_result(input);
|
|
763
|
+
if (current.status === "failed") {
|
|
764
|
+
return {
|
|
765
|
+
success: false,
|
|
766
|
+
data: current as unknown as JsonObject,
|
|
767
|
+
error: current.error ?? current.message ?? input.job_id,
|
|
768
|
+
message: current.error ?? current.message ?? "image job failed",
|
|
769
|
+
};
|
|
770
|
+
}
|
|
497
771
|
const data = current.status === "succeeded" && current.result
|
|
498
772
|
? current.result as unknown as JsonObject
|
|
499
773
|
: current as unknown as JsonObject;
|
|
@@ -508,30 +782,11 @@ export class ImagePlugin extends BasePlugin {
|
|
|
508
782
|
} catch (error) {
|
|
509
783
|
return {
|
|
510
784
|
success: false,
|
|
511
|
-
error:
|
|
512
|
-
message:
|
|
513
|
-
};
|
|
514
|
-
}
|
|
515
|
-
},
|
|
516
|
-
},
|
|
517
|
-
generate: {
|
|
518
|
-
execute: async ({ payload }: { payload: JsonValue }) => {
|
|
519
|
-
try {
|
|
520
|
-
const input = normalize_image_payload(payload);
|
|
521
|
-
const message = await this.generate_image(input);
|
|
522
|
-
return {
|
|
523
|
-
success: true,
|
|
524
|
-
data: message as unknown as JsonObject,
|
|
525
|
-
message: "image generated",
|
|
526
|
-
};
|
|
527
|
-
} catch (error) {
|
|
528
|
-
return {
|
|
529
|
-
success: false,
|
|
530
|
-
error: String(error),
|
|
531
|
-
message: String(error),
|
|
785
|
+
error: describe_error(error),
|
|
786
|
+
message: describe_error(error),
|
|
532
787
|
};
|
|
533
788
|
}
|
|
534
789
|
},
|
|
535
|
-
},
|
|
790
|
+
}),
|
|
536
791
|
};
|
|
537
792
|
}
|