@eum-agent/llm-capability-image 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1323 -0
- package/package.json +56 -3
- package/ui/ui.js +228 -0
- package/README.md +0 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,1323 @@
|
|
|
1
|
+
import { constants, existsSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { open, realpath, stat } from "node:fs/promises";
|
|
6
|
+
//#region src/runtime/models/redact.ts
|
|
7
|
+
function redactables(type, secrets) {
|
|
8
|
+
const out = { ...secrets };
|
|
9
|
+
for (const field of type.fields) {
|
|
10
|
+
if (field.kind !== "login") continue;
|
|
11
|
+
const value = secrets[field.key];
|
|
12
|
+
if (value === void 0) continue;
|
|
13
|
+
try {
|
|
14
|
+
const parsed = JSON.parse(value);
|
|
15
|
+
if (typeof parsed !== "object" || parsed === null) continue;
|
|
16
|
+
for (const [member, token] of Object.entries(parsed)) if (typeof token === "string" && token !== "") out[`${field.key}.${member}`] = token;
|
|
17
|
+
} catch {}
|
|
18
|
+
}
|
|
19
|
+
return out;
|
|
20
|
+
}
|
|
21
|
+
/** Adds every opaque token string to the call's redaction set, including rotated tokens. */
|
|
22
|
+
function trackLogin(handle, values) {
|
|
23
|
+
let count = 0;
|
|
24
|
+
function record(value) {
|
|
25
|
+
if (!value) return;
|
|
26
|
+
values[`login.${count++}`] = value;
|
|
27
|
+
try {
|
|
28
|
+
const parsed = JSON.parse(value);
|
|
29
|
+
if (parsed && typeof parsed === "object") {
|
|
30
|
+
for (const member of Object.values(parsed)) if (typeof member === "string") record(member);
|
|
31
|
+
}
|
|
32
|
+
} catch {}
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
async get(signal) {
|
|
36
|
+
const result = await handle.get(signal);
|
|
37
|
+
record(result.credential);
|
|
38
|
+
return result;
|
|
39
|
+
},
|
|
40
|
+
expire: (credential) => handle.expire(credential)
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
//#endregion
|
|
44
|
+
//#region src/runtime/models/binding-port.ts
|
|
45
|
+
/** Keeps construction, generation and deferred asset errors inside the same redaction lifetime. */
|
|
46
|
+
function createImageBindingPort(adapter, call) {
|
|
47
|
+
const values = { ...call.secrets };
|
|
48
|
+
for (const [key, value] of Object.entries(call.secrets)) try {
|
|
49
|
+
const parsed = JSON.parse(value);
|
|
50
|
+
if (parsed && typeof parsed === "object") {
|
|
51
|
+
for (const [member, token] of Object.entries(parsed)) if (typeof token === "string" && token) values[`${key}.${member}`] = token;
|
|
52
|
+
}
|
|
53
|
+
} catch {}
|
|
54
|
+
function redact(value) {
|
|
55
|
+
for (const secret of Object.values(values).toSorted((a, b) => b.length - a.length)) if (secret) value = value.split(secret).join("****");
|
|
56
|
+
return value;
|
|
57
|
+
}
|
|
58
|
+
function failure(error) {
|
|
59
|
+
const source = error !== null && typeof error === "object" ? error : {};
|
|
60
|
+
const safe = new Error(typeof source.message === "string" ? redact(source.message) : "Image adapter failed");
|
|
61
|
+
if (typeof source.name === "string") safe.name = redact(source.name);
|
|
62
|
+
for (const key of [
|
|
63
|
+
"code",
|
|
64
|
+
"resultState",
|
|
65
|
+
"providerText",
|
|
66
|
+
"problem"
|
|
67
|
+
]) if (typeof source[key] === "string") Object.assign(safe, { [key]: redact(source[key]) });
|
|
68
|
+
for (const key of ["status", "retryAt"]) if (typeof source[key] === "number") Object.assign(safe, { [key]: source[key] });
|
|
69
|
+
if (typeof source.authored === "boolean") Object.assign(safe, { authored: source.authored });
|
|
70
|
+
return safe;
|
|
71
|
+
}
|
|
72
|
+
let port;
|
|
73
|
+
try {
|
|
74
|
+
port = adapter.createPort({
|
|
75
|
+
...call,
|
|
76
|
+
...call.login ? { login: trackLogin(call.login, values) } : {}
|
|
77
|
+
});
|
|
78
|
+
} catch (error) {
|
|
79
|
+
throw failure(error);
|
|
80
|
+
}
|
|
81
|
+
return { async generate(request, signal) {
|
|
82
|
+
try {
|
|
83
|
+
const result = await port.generate(request, signal);
|
|
84
|
+
if (!result || !Array.isArray(result.assets)) return result;
|
|
85
|
+
return {
|
|
86
|
+
...result,
|
|
87
|
+
assets: result.assets.map((asset) => ({
|
|
88
|
+
...asset,
|
|
89
|
+
body: { async *[Symbol.asyncIterator]() {
|
|
90
|
+
try {
|
|
91
|
+
yield* asset.body;
|
|
92
|
+
} catch (error) {
|
|
93
|
+
throw failure(error);
|
|
94
|
+
}
|
|
95
|
+
} }
|
|
96
|
+
}))
|
|
97
|
+
};
|
|
98
|
+
} catch (error) {
|
|
99
|
+
throw failure(error);
|
|
100
|
+
}
|
|
101
|
+
} };
|
|
102
|
+
}
|
|
103
|
+
//#endregion
|
|
104
|
+
//#region src/capability.ts
|
|
105
|
+
/** Validates the Image adapter shape without touching credentials or network. */
|
|
106
|
+
function assertImageAdapter(adapter) {
|
|
107
|
+
const value = adapter;
|
|
108
|
+
if (!value || typeof value.models !== "function" || typeof value.createPort !== "function") throw new Error("unsupported-binding: invalid Image adapter");
|
|
109
|
+
}
|
|
110
|
+
function containsCustomCheck(value, seen = /* @__PURE__ */ new Set()) {
|
|
111
|
+
if (value === null || typeof value !== "object" || seen.has(value)) return false;
|
|
112
|
+
seen.add(value);
|
|
113
|
+
const internal = value._zod;
|
|
114
|
+
if (internal) return containsCustomCheck(internal.def, seen);
|
|
115
|
+
const record = value;
|
|
116
|
+
return record.check === "custom" || Object.values(record).some((child) => containsCustomCheck(child, seen));
|
|
117
|
+
}
|
|
118
|
+
const unique = (values) => new Set(values).size === values.length;
|
|
119
|
+
const descriptors = z.array(z.strictObject({
|
|
120
|
+
id: z.string().trim().min(1),
|
|
121
|
+
label: z.string().trim().min(1),
|
|
122
|
+
outputKinds: z.array(z.enum([
|
|
123
|
+
"image",
|
|
124
|
+
"audio",
|
|
125
|
+
"video"
|
|
126
|
+
])).nonempty().refine(unique),
|
|
127
|
+
artifactOrigins: z.array(z.string().url().refine((value) => {
|
|
128
|
+
const url = new URL(value);
|
|
129
|
+
return url.protocol === "https:" && !url.username && !url.password && url.origin === value;
|
|
130
|
+
})).refine(unique).optional(),
|
|
131
|
+
operations: z.array(z.strictObject({
|
|
132
|
+
id: z.string().trim().min(1),
|
|
133
|
+
inputKinds: z.array(z.string().min(1)).refine(unique),
|
|
134
|
+
minInputs: z.number().int().nonnegative(),
|
|
135
|
+
maxInputs: z.number().int().nonnegative(),
|
|
136
|
+
paramsSchema: z.custom((value) => value instanceof z.ZodType)
|
|
137
|
+
}).refine((value) => value.minInputs <= value.maxInputs)).nonempty().refine((values) => unique(values.map((value) => value.id)))
|
|
138
|
+
})).refine((values) => unique(values.map((value) => value.id)));
|
|
139
|
+
/** Validates descriptors at their capability boundary, including strict lossless operation schemas. */
|
|
140
|
+
function generationModelsOf(protocol, context) {
|
|
141
|
+
const adapter = protocol.adapters.image;
|
|
142
|
+
if (!adapter) return [];
|
|
143
|
+
assertImageAdapter(adapter);
|
|
144
|
+
const models = adapter.models(context);
|
|
145
|
+
if (!descriptors.safeParse(models).success) throw new Error("invalid Image model descriptors");
|
|
146
|
+
for (const model of models) {
|
|
147
|
+
if (!model.id || !model.label || !Array.isArray(model.outputKinds) || !Array.isArray(model.operations)) throw new Error("invalid Image model descriptor");
|
|
148
|
+
for (const operation of model.operations) {
|
|
149
|
+
if (!(operation.paramsSchema instanceof z.ZodObject) || !(operation.paramsSchema._zod.def.catchall instanceof z.ZodNever) || containsCustomCheck(operation.paramsSchema)) throw new Error("Image operation parameters must be a strict, lossless object schema");
|
|
150
|
+
operation.paramsSchema.toJSONSchema();
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return models;
|
|
154
|
+
}
|
|
155
|
+
const ImageDiscoveryFacts = z.object({ output: z.array(z.string()).optional() });
|
|
156
|
+
const ImageModelFacts = z.object({
|
|
157
|
+
modalities: z.object({ output: z.array(z.string()).optional() }).optional(),
|
|
158
|
+
generation: z.object({
|
|
159
|
+
outputKinds: z.array(z.string()).optional(),
|
|
160
|
+
operations: z.array(z.object({ id: z.string() })).optional()
|
|
161
|
+
}).optional()
|
|
162
|
+
});
|
|
163
|
+
function imageModelFor(input) {
|
|
164
|
+
assertImageAdapter(input.adapter);
|
|
165
|
+
const discovery = ImageDiscoveryFacts.safeParse(input.descriptor?.discoveryFacts).data;
|
|
166
|
+
const facts = ImageModelFacts.safeParse(input.descriptor?.modelFacts ?? input.facts).data;
|
|
167
|
+
const output = discovery?.output ?? facts?.modalities?.output ?? facts?.generation?.outputKinds;
|
|
168
|
+
if (output !== void 0 && !output.includes("image")) return void 0;
|
|
169
|
+
const declaredOperations = facts?.generation?.operations;
|
|
170
|
+
const context = {
|
|
171
|
+
providerId: input.ref.providerId,
|
|
172
|
+
config: input.config,
|
|
173
|
+
configuredModels: [{
|
|
174
|
+
id: input.ref.model,
|
|
175
|
+
...input.execution ? { execution: input.execution } : {}
|
|
176
|
+
}]
|
|
177
|
+
};
|
|
178
|
+
const model = input.adapter.models(context).find((item) => item.id === input.ref.model && item.outputKinds.includes("image"));
|
|
179
|
+
if (!model) return void 0;
|
|
180
|
+
const operations = model.operations.filter((operation) => (operation.id === "create" || operation.id === "edit") && (declaredOperations === void 0 || declaredOperations.some((one) => one.id === operation.id)));
|
|
181
|
+
return operations.length ? {
|
|
182
|
+
...model,
|
|
183
|
+
operations
|
|
184
|
+
} : void 0;
|
|
185
|
+
}
|
|
186
|
+
/** Registered Image ability; ordinary parameter edits affect only the next submission. */
|
|
187
|
+
const imageCapability = {
|
|
188
|
+
id: "image",
|
|
189
|
+
tools: ["model_image_generate"],
|
|
190
|
+
label: "Image",
|
|
191
|
+
order: 20,
|
|
192
|
+
uiContributionKey: "image-models",
|
|
193
|
+
configurationPolicy: {
|
|
194
|
+
params: "retain",
|
|
195
|
+
connection: "cancel"
|
|
196
|
+
},
|
|
197
|
+
bindingSchema: z.record(z.string(), z.unknown()),
|
|
198
|
+
descriptorSchema: z.record(z.string(), z.unknown()),
|
|
199
|
+
paramsSchema(input) {
|
|
200
|
+
const operations = input.descriptor?.operations;
|
|
201
|
+
const fields = {};
|
|
202
|
+
if (Array.isArray(operations)) for (const operation of operations) {
|
|
203
|
+
if (!operation || typeof operation !== "object") continue;
|
|
204
|
+
const value = operation;
|
|
205
|
+
if (typeof value.id === "string" && value.parameterSchema && typeof value.parameterSchema === "object") fields[value.id] = z.fromJSONSchema(value.parameterSchema).optional();
|
|
206
|
+
}
|
|
207
|
+
return Object.keys(fields).length ? z.strictObject(fields) : z.strictObject({
|
|
208
|
+
create: z.record(z.string(), z.unknown()).optional(),
|
|
209
|
+
edit: z.record(z.string(), z.unknown()).optional()
|
|
210
|
+
});
|
|
211
|
+
},
|
|
212
|
+
validateAdapter: assertImageAdapter,
|
|
213
|
+
describeBinding(input) {
|
|
214
|
+
const model = imageModelFor(input);
|
|
215
|
+
return {
|
|
216
|
+
availability: {
|
|
217
|
+
available: !!model,
|
|
218
|
+
...!model ? { reason: "execution-unverified" } : {}
|
|
219
|
+
},
|
|
220
|
+
descriptor: { operations: model?.operations.map((operation) => ({
|
|
221
|
+
id: operation.id,
|
|
222
|
+
inputKinds: [...operation.inputKinds],
|
|
223
|
+
parameterSchema: operation.paramsSchema.toJSONSchema({ io: "input" })
|
|
224
|
+
})) ?? [] }
|
|
225
|
+
};
|
|
226
|
+
},
|
|
227
|
+
createBinding(input) {
|
|
228
|
+
const model = imageModelFor(input);
|
|
229
|
+
return {
|
|
230
|
+
...imageCapability.describeBinding(input),
|
|
231
|
+
model,
|
|
232
|
+
async createPort() {
|
|
233
|
+
return createImageBindingPort(input.adapter, input.getCall());
|
|
234
|
+
},
|
|
235
|
+
async dispose() {}
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
};
|
|
239
|
+
//#endregion
|
|
240
|
+
//#region src/guidance.ts
|
|
241
|
+
/** Image generation guidance withdraws with the owning tool. */
|
|
242
|
+
const GUIDANCE = {
|
|
243
|
+
title: "Images",
|
|
244
|
+
entries: [{
|
|
245
|
+
tools: ["model_image_generate"],
|
|
246
|
+
text: "Use model_image_generate for image creation or editing. model_list with capabilityId image reports configured targets and operation parameter schemas. Explicit target overrides the retained image default. A failed default is never silently replaced. File references are returned after publication. If the conversation model cannot view images, do not claim to have inspected the generated result. Never automatically retry an unknown or failed submission.\nAfter successful generation or editing, deliver the image in your final reply using its canonical private attachment URI from the tool result's File reference. Choose presentation from the person's explicit request, task context and effective prompts. By default, show ordinary generation or editing inline as . When the person explicitly requests only a link, use [label](uri) with no inline image. To change only the presentation of an already delivered image, reuse its same URI without calling `model_image_generate` again.\nCopy the URI exactly as returned for that file; never guess or construct it, or substitute a local path, temporary download address, Base64, data URI, browser Blob URL or credential-bearing URL. If the image was saved and delivered as an attachment but no usable URI was returned, say it was generated but a body reference is currently unavailable; retain the attachment viewing entry, do not describe generation as failed, invent a link or generate again just to display it.\nFor failed, cancelled or unknown results, do not invent success, an image or a link, and do not automatically retry generation. Explain the actual result only while the turn can continue. If the whole turn was cancelled, preserve its cancellation state: do not append a reply, resume or restart the turn."
|
|
247
|
+
}]
|
|
248
|
+
};
|
|
249
|
+
//#endregion
|
|
250
|
+
//#region src/output-schema.ts
|
|
251
|
+
/** Validates complete owner business results and bounded recoverable delivery failures. */
|
|
252
|
+
/** Adds the standard failure envelope and bounded text refusals to an owner's result schema. */
|
|
253
|
+
function toolOutputSchema(body, input, name) {
|
|
254
|
+
return z.union([
|
|
255
|
+
body,
|
|
256
|
+
z.strictObject({
|
|
257
|
+
ok: z.literal(false),
|
|
258
|
+
error: z.strictObject({
|
|
259
|
+
code: z.string().min(1).max(128),
|
|
260
|
+
message: z.string().refine((value) => [...value].length <= 256).meta({ maxLength: 256 }),
|
|
261
|
+
retryable: z.boolean(),
|
|
262
|
+
field: z.string().max(256).optional(),
|
|
263
|
+
maximum: z.number().finite().optional()
|
|
264
|
+
}),
|
|
265
|
+
next: z.strictObject({
|
|
266
|
+
tool: z.literal(name),
|
|
267
|
+
arguments: input
|
|
268
|
+
}).nullable(),
|
|
269
|
+
execution: z.strictObject({ state: z.enum([
|
|
270
|
+
"not-started",
|
|
271
|
+
"committed",
|
|
272
|
+
"unknown"
|
|
273
|
+
]) }).optional()
|
|
274
|
+
}),
|
|
275
|
+
z.string().max(1024)
|
|
276
|
+
]);
|
|
277
|
+
}
|
|
278
|
+
//#endregion
|
|
279
|
+
//#region src/image-generation.ts
|
|
280
|
+
const target = z.strictObject({
|
|
281
|
+
providerId: z.string().min(1).max(512).describe("Exact configured provider connection ID from model_list."),
|
|
282
|
+
model: z.string().min(1).max(256).describe("Exact configured image model ID from model_list.")
|
|
283
|
+
});
|
|
284
|
+
const common = {
|
|
285
|
+
target: target.optional().describe("Exact configured target; otherwise use the global image default."),
|
|
286
|
+
prompt: z.string().refine((value) => value.trim().length > 0 && [...value.trim()].length <= 32e3, { message: "The prompt must contain 1–32000 Unicode characters after trimming." }).meta({
|
|
287
|
+
minLength: 1,
|
|
288
|
+
maxLength: 32e3
|
|
289
|
+
}).describe("Image instructions; 1–32000 Unicode characters after trimming."),
|
|
290
|
+
params: z.record(z.string(), z.unknown()).default({}).describe("Parameters must match the selected model operation schema; omit to use that model default.")
|
|
291
|
+
};
|
|
292
|
+
const reference = z.union([z.strictObject({ fileId: z.string().regex(/^[0-9a-f]{64}$/).describe("Stored image file ID from this session.") }), z.strictObject({ path: z.string().min(1).max(4096).describe("Local image path to authorize before submission.") })]);
|
|
293
|
+
const create = {
|
|
294
|
+
...common,
|
|
295
|
+
operation: z.literal("create").default("create").describe("Create a new image without references.")
|
|
296
|
+
};
|
|
297
|
+
const edit = {
|
|
298
|
+
...common,
|
|
299
|
+
operation: z.literal("edit").describe("Edit existing images selected by one reference mechanism.")
|
|
300
|
+
};
|
|
301
|
+
const explicit = { reference_images: z.array(reference).min(1).max(14).describe("1–14 ordered references, subject to the selected model limit; mutually exclusive with recent count.") };
|
|
302
|
+
const recent = { num_last_images_to_include: z.number().int().min(1).max(14).describe("1–14 most recent distinct session images; mutually exclusive with explicit references.") };
|
|
303
|
+
/** Strict create/edit shapes, with paired explicit targets and mutually exclusive reference selectors. */
|
|
304
|
+
const IMAGE_INPUT_SCHEMA = z.union([
|
|
305
|
+
z.strictObject(create),
|
|
306
|
+
z.strictObject({
|
|
307
|
+
...edit,
|
|
308
|
+
...explicit
|
|
309
|
+
}),
|
|
310
|
+
z.strictObject({
|
|
311
|
+
...edit,
|
|
312
|
+
...recent
|
|
313
|
+
})
|
|
314
|
+
]);
|
|
315
|
+
function refusal(content) {
|
|
316
|
+
return {
|
|
317
|
+
isError: true,
|
|
318
|
+
content: content.slice(0, 1024)
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
const FAILURE_TEXT = {
|
|
322
|
+
unsupported: "This connection does not support the requested image operation.",
|
|
323
|
+
"invalid-params": "The image request parameters are invalid.",
|
|
324
|
+
"invalid-input": "The selected reference images are missing, stale or unsupported.",
|
|
325
|
+
unavailable: "The current image generation connection is unavailable.",
|
|
326
|
+
busy: "Too many image calls are active; wait for one to finish.",
|
|
327
|
+
cancelled: "Image generation was cancelled.",
|
|
328
|
+
"credential-unavailable": "The connection credential is unavailable; check model settings.",
|
|
329
|
+
"not-signed-in": "Sign in to the selected connection in model settings.",
|
|
330
|
+
expired: "The provider sign-in expired; sign in again in model settings.",
|
|
331
|
+
forbidden: "This account or request is not permitted to generate an image.",
|
|
332
|
+
quota: "The image service rate or quota limit has been reached.",
|
|
333
|
+
timeout: "Image generation timed out.",
|
|
334
|
+
network: "The connection to the image service failed.",
|
|
335
|
+
provider: "The image service request failed.",
|
|
336
|
+
"invalid-output": "The returned image could not be read or did not meet output requirements.",
|
|
337
|
+
"too-large": "The image inputs or output exceeded the size limit.",
|
|
338
|
+
storage: "The generated image could not be saved."
|
|
339
|
+
};
|
|
340
|
+
const STATE_TEXT = {
|
|
341
|
+
"not-submitted": "No image request was submitted.",
|
|
342
|
+
unknown: "The server result is unknown; another attempt creates a new request.",
|
|
343
|
+
"not-delivered": "No image was delivered."
|
|
344
|
+
};
|
|
345
|
+
/** Displays only host-normalized detail; unexpected exceptions never expose their messages. */
|
|
346
|
+
function failureOf(error, cancelled) {
|
|
347
|
+
const failure = error;
|
|
348
|
+
if (failure?.name !== "GenerationFailure" || typeof failure.code !== "string" || !Object.hasOwn(FAILURE_TEXT, failure.code)) return {
|
|
349
|
+
isError: true,
|
|
350
|
+
content: cancelled ? "Image generation was cancelled. No image was delivered." : "Image generation failed. No image was delivered."
|
|
351
|
+
};
|
|
352
|
+
const state = failure.resultState ?? "not-delivered";
|
|
353
|
+
const prefix = `${FAILURE_TEXT[failure.code]} ${STATE_TEXT[state] ?? STATE_TEXT["not-delivered"]}`;
|
|
354
|
+
const retryAt = failure.retryAt;
|
|
355
|
+
const retry = typeof retryAt === "number" && Number.isFinite(retryAt) && retryAt > 0 && retryAt <= 864e13 ? ` Retry after ${new Date(retryAt).toISOString()} UTC.` : "";
|
|
356
|
+
return {
|
|
357
|
+
isError: true,
|
|
358
|
+
content: (prefix + (typeof failure.message === "string" ? ` ${failure.message}` : "")).slice(0, 511 - retry.length) + retry + " Do not automatically retry."
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
/** Creates the shared image tool with explicit, default, then unique-candidate routing. */
|
|
362
|
+
function imageGenerationTool(deps) {
|
|
363
|
+
const candidates = (operation) => deps.generation.connections().flatMap((connection) => connection.models.filter((model) => model.outputKinds.includes("image")).flatMap((model) => model.operations.filter((one) => one.id === operation).map((descriptor) => ({
|
|
364
|
+
ref: {
|
|
365
|
+
providerId: connection.providerId,
|
|
366
|
+
model: model.id
|
|
367
|
+
},
|
|
368
|
+
descriptor
|
|
369
|
+
}))));
|
|
370
|
+
return {
|
|
371
|
+
name: "model_image_generate",
|
|
372
|
+
preload: false,
|
|
373
|
+
deliveryKind: "preview",
|
|
374
|
+
description: "Generate or edit an image using a configured image model. Inspect model_list for operation parameter schemas. Supply target to override the global image default. Never automatically retry a failed submission.",
|
|
375
|
+
inputSchema: IMAGE_INPUT_SCHEMA,
|
|
376
|
+
outputSchema: toolOutputSchema(z.strictObject({
|
|
377
|
+
ok: z.literal(true),
|
|
378
|
+
committed: z.literal(true),
|
|
379
|
+
text: z.string(),
|
|
380
|
+
target,
|
|
381
|
+
fileIds: z.array(z.string().regex(/^[0-9a-f]{64}$/)).min(1).max(8)
|
|
382
|
+
}), IMAGE_INPUT_SCHEMA, "model_image_generate"),
|
|
383
|
+
available(view) {
|
|
384
|
+
return view.model !== null && deps.generation.connections().some((connection) => connection.models.some((model) => model.outputKinds.includes("image") && model.operations.some((operation) => operation.id === "create" || operation.id === "edit")));
|
|
385
|
+
},
|
|
386
|
+
async execute(input, ctx) {
|
|
387
|
+
try {
|
|
388
|
+
const current = deps.generation.current(ctx);
|
|
389
|
+
const requested = input.target ?? deps.defaultModel();
|
|
390
|
+
const all = candidates(input.operation);
|
|
391
|
+
const eligible = requested ? all.filter((one) => one.ref.providerId === requested.providerId && one.ref.model === requested.model) : all;
|
|
392
|
+
if (!eligible.length) return refusal("The selected or default image model is unavailable for this operation. Choose an image model in settings or pass an explicit target. No request was submitted.");
|
|
393
|
+
if (eligible.length !== 1) return refusal("Multiple image models are available. Set a default image model or pass an explicit target from model_list.");
|
|
394
|
+
const selected = eligible[0];
|
|
395
|
+
const identity = deps.identity(selected.ref);
|
|
396
|
+
if (!identity) return refusal("The image connection is unavailable. No request was submitted.");
|
|
397
|
+
const merged = structuredClone({
|
|
398
|
+
...deps.params(selected.ref, input.operation),
|
|
399
|
+
...input.params
|
|
400
|
+
});
|
|
401
|
+
const params = selected.descriptor.paramsSchema.safeParse(merged);
|
|
402
|
+
if (!params.success) return refusal("The selected image model does not support these parameters. Inspect model_list for its operation schema.");
|
|
403
|
+
let prepared;
|
|
404
|
+
if (input.operation === "edit") {
|
|
405
|
+
const count = "reference_images" in input ? input.reference_images.length : input.num_last_images_to_include;
|
|
406
|
+
if (count < selected.descriptor.minInputs || count > selected.descriptor.maxInputs) return refusal("The selected image model does not support this number of reference images.");
|
|
407
|
+
const selection = "reference_images" in input ? input.reference_images : { recent: input.num_last_images_to_include };
|
|
408
|
+
const result = await deps.inputs.prepare(ctx, selection);
|
|
409
|
+
if (result.kind === "ask") return result.result;
|
|
410
|
+
if (result.kind === "refused") return refusal(result.message);
|
|
411
|
+
prepared = result.inputs;
|
|
412
|
+
}
|
|
413
|
+
const authorization = await deps.generation.authorizeArtifacts(ctx, selected.ref);
|
|
414
|
+
if (authorization.kind === "ask") return authorization.result;
|
|
415
|
+
if (authorization.kind === "refused") return refusal(authorization.message);
|
|
416
|
+
const next = deps.generation.current(ctx);
|
|
417
|
+
if (JSON.stringify(next?.ref) !== JSON.stringify(current?.ref) || next?.connectionRevision !== current?.connectionRevision || next?.registrationEpoch !== current?.registrationEpoch || deps.identity(selected.ref) !== identity) return refusal("The current conversation connection changed. No image request was submitted.");
|
|
418
|
+
if (!candidates(input.operation).some((one) => one.ref.providerId === selected.ref.providerId && one.ref.model === selected.ref.model)) return refusal("The selected image model became unavailable. No image request was submitted.");
|
|
419
|
+
ctx.signal.throwIfAborted();
|
|
420
|
+
const result = await deps.generation.generate(ctx, {
|
|
421
|
+
ref: selected.ref,
|
|
422
|
+
operation: input.operation,
|
|
423
|
+
outputKind: "image",
|
|
424
|
+
prompt: input.prompt.trim(),
|
|
425
|
+
params: params.data,
|
|
426
|
+
...prepared === void 0 ? {} : { inputs: prepared }
|
|
427
|
+
});
|
|
428
|
+
return {
|
|
429
|
+
content: JSON.stringify({
|
|
430
|
+
ok: true,
|
|
431
|
+
committed: true,
|
|
432
|
+
target: selected.ref,
|
|
433
|
+
text: result.text?.trim() || "Image generated.",
|
|
434
|
+
fileIds: result.files.map((file) => file.id)
|
|
435
|
+
}),
|
|
436
|
+
files: result.files.map((file) => ({
|
|
437
|
+
fileId: file.id,
|
|
438
|
+
name: file.name
|
|
439
|
+
}))
|
|
440
|
+
};
|
|
441
|
+
} catch (error) {
|
|
442
|
+
return failureOf(error, ctx.signal.aborted);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
//#endregion
|
|
448
|
+
//#region src/adapters.ts
|
|
449
|
+
/** Returns the capability-owned image adapter. */
|
|
450
|
+
function imageAdapter(protocol) {
|
|
451
|
+
return protocol?.adapters.image;
|
|
452
|
+
}
|
|
453
|
+
//#endregion
|
|
454
|
+
//#region src/runtime/models/generation.ts
|
|
455
|
+
/** Host admission, validation, cancellation and atomic delivery for generation tools. */
|
|
456
|
+
/** A stable failure whose text never contains raw provider headers or bodies. */
|
|
457
|
+
var GenerationFailure = class extends Error {
|
|
458
|
+
name = "GenerationFailure";
|
|
459
|
+
code;
|
|
460
|
+
resultState;
|
|
461
|
+
status;
|
|
462
|
+
retryAt;
|
|
463
|
+
constructor(code, message, resultState = "not-submitted", status, retryAt) {
|
|
464
|
+
super(message.slice(0, 512));
|
|
465
|
+
this.code = code;
|
|
466
|
+
this.resultState = resultState;
|
|
467
|
+
this.status = status;
|
|
468
|
+
this.retryAt = retryAt;
|
|
469
|
+
}
|
|
470
|
+
};
|
|
471
|
+
const FAILURE_CODES = /* @__PURE__ */ new Set([
|
|
472
|
+
"unsupported",
|
|
473
|
+
"invalid-params",
|
|
474
|
+
"invalid-input",
|
|
475
|
+
"unavailable",
|
|
476
|
+
"busy",
|
|
477
|
+
"cancelled",
|
|
478
|
+
"credential-unavailable",
|
|
479
|
+
"not-signed-in",
|
|
480
|
+
"expired",
|
|
481
|
+
"forbidden",
|
|
482
|
+
"quota",
|
|
483
|
+
"timeout",
|
|
484
|
+
"network",
|
|
485
|
+
"provider",
|
|
486
|
+
"invalid-output",
|
|
487
|
+
"too-large",
|
|
488
|
+
"storage"
|
|
489
|
+
]);
|
|
490
|
+
function fail(code, message) {
|
|
491
|
+
throw new GenerationFailure(code, message);
|
|
492
|
+
}
|
|
493
|
+
/** Races uncooperative promises while still observing their eventual rejection. */
|
|
494
|
+
function untilAbort(promise, signal) {
|
|
495
|
+
if (signal.aborted) {
|
|
496
|
+
promise.catch(() => {});
|
|
497
|
+
signal.throwIfAborted();
|
|
498
|
+
}
|
|
499
|
+
return new Promise((resolve, reject) => {
|
|
500
|
+
const abort = () => reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
501
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
502
|
+
promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
/** Builds independently bound services sharing the installation's admission budget. */
|
|
506
|
+
function createGenerationHost(deps) {
|
|
507
|
+
const prepared = /* @__PURE__ */ new WeakMap();
|
|
508
|
+
const active = /* @__PURE__ */ new Set();
|
|
509
|
+
const authorized = /* @__PURE__ */ new WeakMap();
|
|
510
|
+
const ports = /* @__PURE__ */ new Map();
|
|
511
|
+
let stopped = false;
|
|
512
|
+
function descriptor(owner, ref, crossOwner = false) {
|
|
513
|
+
const target = deps.target(ref);
|
|
514
|
+
if (!target?.enabled || target.lifetime.aborted || !crossOwner && target.owner !== owner || !imageAdapter(target.protocol)) fail("unavailable", "the generation provider is unavailable");
|
|
515
|
+
const model = deps.descriptors(target.protocol, {
|
|
516
|
+
providerId: ref.providerId,
|
|
517
|
+
config: target.config,
|
|
518
|
+
configuredModels: target.configuredModels
|
|
519
|
+
}).find((one) => one.id === ref.model);
|
|
520
|
+
if (!model) fail("unsupported", "the generation model is unsupported");
|
|
521
|
+
return {
|
|
522
|
+
target,
|
|
523
|
+
model
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
function bind(owner, crossOwner = false) {
|
|
527
|
+
return {
|
|
528
|
+
connections: () => deps.connections(crossOwner ? void 0 : owner),
|
|
529
|
+
current(ctx) {
|
|
530
|
+
deps.authenticate(owner, ctx);
|
|
531
|
+
return deps.current(ctx);
|
|
532
|
+
},
|
|
533
|
+
async authorizeArtifacts(ctx, ref) {
|
|
534
|
+
deps.authenticate(owner, ctx);
|
|
535
|
+
const { model, target } = descriptor(owner, ref, crossOwner);
|
|
536
|
+
const result = await deps.authorize(ctx, model.artifactOrigins ?? []);
|
|
537
|
+
if (result.kind === "ready") {
|
|
538
|
+
if (deps.target(ref)?.registrationEpoch !== target.registrationEpoch || deps.target(ref)?.protocolEpoch !== target.protocolEpoch || deps.target(ref)?.connectionRevision !== target.connectionRevision || !deps.target(ref)?.enabled || target.lifetime.aborted || ctx.signal.aborted) fail("cancelled", "the provider changed during authorization");
|
|
539
|
+
authorized.set(ctx, new Set(model.artifactOrigins ?? []));
|
|
540
|
+
}
|
|
541
|
+
return result;
|
|
542
|
+
},
|
|
543
|
+
async generate(ctx, input) {
|
|
544
|
+
deps.authenticate(owner, ctx);
|
|
545
|
+
if (stopped) fail("unavailable", "generation is stopped");
|
|
546
|
+
ctx.signal.throwIfAborted();
|
|
547
|
+
const { target, model } = descriptor(owner, input.ref, crossOwner);
|
|
548
|
+
const operation = model.operations.find((one) => one.id === input.operation);
|
|
549
|
+
if (!operation || !model.outputKinds.includes(input.outputKind)) fail("unsupported", "the generation operation is unsupported");
|
|
550
|
+
const prompt = input.prompt.trim();
|
|
551
|
+
if (!prompt || [...prompt].length > 32e3) fail("invalid-params", "the generation prompt must contain 1–32000 characters");
|
|
552
|
+
const params = operation.paramsSchema.safeParse(input.params);
|
|
553
|
+
if (!params.success) fail("invalid-params", `the generation parameters are invalid (${params.error.issues.slice(0, 4).map((issue) => `${issue.path.map(String).join(".") || "params"}: ${issue.code}`).join("; ")})`);
|
|
554
|
+
const selection = input.inputs === void 0 ? void 0 : prepared.get(input.inputs);
|
|
555
|
+
const historyRevision = deps.sessionRevision(ctx.sessionId);
|
|
556
|
+
if (input.inputs !== void 0 && (!selection || selection.owner !== owner || selection.ctx !== ctx || selection.sessionId !== ctx.sessionId || selection.callId !== ctx.toolCallId || selection.revision !== historyRevision)) fail("invalid-input", "the input selection is invalid or stale");
|
|
557
|
+
const inputs = selection?.files ?? [];
|
|
558
|
+
if (inputs.length < operation.minInputs || inputs.length > operation.maxInputs || inputs.some((file) => !operation.inputKinds.includes(file.mediaType.split("/")[0]))) fail("invalid-input", "the generation input count or media kind is unsupported");
|
|
559
|
+
const origins = model.artifactOrigins ?? [];
|
|
560
|
+
if (origins.some((origin) => !authorized.get(ctx)?.has(origin))) fail("forbidden", "authorize artifact origins before generation");
|
|
561
|
+
if (active.size >= 4 || [...active].filter((one) => one.ctx.sessionId === ctx.sessionId).length >= 2) fail("busy", "too many active generation calls");
|
|
562
|
+
const initial = deps.current(ctx);
|
|
563
|
+
const controller = new AbortController();
|
|
564
|
+
const deadline = AbortSignal.timeout(9e5);
|
|
565
|
+
const signal = AbortSignal.any([
|
|
566
|
+
ctx.signal,
|
|
567
|
+
target.lifetime,
|
|
568
|
+
controller.signal,
|
|
569
|
+
deadline
|
|
570
|
+
]);
|
|
571
|
+
const staged = [];
|
|
572
|
+
let submitted = false;
|
|
573
|
+
let responded = false;
|
|
574
|
+
let phase = "provider";
|
|
575
|
+
const admission = {
|
|
576
|
+
ctx,
|
|
577
|
+
controller,
|
|
578
|
+
assertCurrent() {
|
|
579
|
+
signal.throwIfAborted();
|
|
580
|
+
const now = deps.target(input.ref);
|
|
581
|
+
const current = deps.current(ctx);
|
|
582
|
+
if (!now?.enabled || now.owner !== target.owner || now.registrationEpoch !== target.registrationEpoch || now.protocolEpoch !== target.protocolEpoch || now.connectionRevision !== target.connectionRevision || historyRevision !== deps.sessionRevision(ctx.sessionId) || JSON.stringify(current?.ref) !== JSON.stringify(initial?.ref)) fail("cancelled", "the generation context changed");
|
|
583
|
+
}
|
|
584
|
+
};
|
|
585
|
+
active.add(admission);
|
|
586
|
+
let secrets = {};
|
|
587
|
+
try {
|
|
588
|
+
admission.assertCurrent();
|
|
589
|
+
const key = JSON.stringify([
|
|
590
|
+
owner,
|
|
591
|
+
input.ref.providerId,
|
|
592
|
+
input.ref.model,
|
|
593
|
+
"generation"
|
|
594
|
+
]);
|
|
595
|
+
let kept = ports.get(key);
|
|
596
|
+
if (!kept || kept.epoch !== target.registrationEpoch || kept.protocolEpoch !== target.protocolEpoch || kept.revision !== target.connectionRevision) {
|
|
597
|
+
const credential = deps.credential(input.ref, target.type);
|
|
598
|
+
const redactions = redactables(target.type, credential.secrets);
|
|
599
|
+
const login = credential.login === void 0 ? void 0 : trackLogin(credential.login, redactions);
|
|
600
|
+
secrets = redactions;
|
|
601
|
+
const port = imageAdapter(target.protocol).createPort({
|
|
602
|
+
endpointId: String(target.config.endpointId),
|
|
603
|
+
protocolId: target.protocol.type,
|
|
604
|
+
options: target.options,
|
|
605
|
+
providerId: input.ref.providerId,
|
|
606
|
+
model: input.ref.model,
|
|
607
|
+
config: target.config,
|
|
608
|
+
configuredModels: target.configuredModels,
|
|
609
|
+
secrets: credential.secrets,
|
|
610
|
+
...login ? { login } : {}
|
|
611
|
+
});
|
|
612
|
+
kept = {
|
|
613
|
+
ref: input.ref,
|
|
614
|
+
owner: target.owner,
|
|
615
|
+
epoch: target.registrationEpoch,
|
|
616
|
+
protocolEpoch: target.protocolEpoch,
|
|
617
|
+
revision: target.connectionRevision,
|
|
618
|
+
port,
|
|
619
|
+
secrets: redactions
|
|
620
|
+
};
|
|
621
|
+
ports.set(key, kept);
|
|
622
|
+
}
|
|
623
|
+
secrets = kept.secrets;
|
|
624
|
+
const port = kept.port;
|
|
625
|
+
submitted = true;
|
|
626
|
+
const firstDeadline = new AbortController();
|
|
627
|
+
const firstTimer = setTimeout(() => {
|
|
628
|
+
firstDeadline.abort(new DOMException("The first response timed out", "TimeoutError"));
|
|
629
|
+
}, 3e5);
|
|
630
|
+
firstTimer.unref();
|
|
631
|
+
const first = AbortSignal.any([signal, firstDeadline.signal]);
|
|
632
|
+
const result = await untilAbort((async () => port.generate({
|
|
633
|
+
identity: { callId: randomUUID() },
|
|
634
|
+
artifacts: deps.artifacts(ctx, origins),
|
|
635
|
+
operation: input.operation,
|
|
636
|
+
outputKind: input.outputKind,
|
|
637
|
+
prompt,
|
|
638
|
+
params: params.data,
|
|
639
|
+
inputs
|
|
640
|
+
}, first))(), first).finally(() => clearTimeout(firstTimer));
|
|
641
|
+
responded = true;
|
|
642
|
+
if (result === null || typeof result !== "object" || !Array.isArray(result.assets) || result.assets.length < 1 || result.assets.length > 8 || result.text !== void 0 && (typeof result.text !== "string" || [...result.text].length > 16e3)) fail("invalid-output", "the provider returned an invalid generation result");
|
|
643
|
+
phase = "stage";
|
|
644
|
+
let total = 0;
|
|
645
|
+
for (const asset of result.assets) {
|
|
646
|
+
admission.assertCurrent();
|
|
647
|
+
const body = (async function* () {
|
|
648
|
+
const iterator = asset.body[Symbol.asyncIterator]();
|
|
649
|
+
try {
|
|
650
|
+
for (;;) {
|
|
651
|
+
const next = await untilAbort(iterator.next(), AbortSignal.any([signal, AbortSignal.timeout(6e4)]));
|
|
652
|
+
if (next.done) break;
|
|
653
|
+
if (!(next.value instanceof Uint8Array)) fail("invalid-output", "the provider returned invalid asset bytes");
|
|
654
|
+
total += next.value.byteLength;
|
|
655
|
+
if (total > 268435456) fail("too-large", "the generation result exceeds 256 MiB");
|
|
656
|
+
yield next.value;
|
|
657
|
+
}
|
|
658
|
+
} finally {
|
|
659
|
+
iterator.return?.().catch(() => {});
|
|
660
|
+
}
|
|
661
|
+
})();
|
|
662
|
+
staged.push(await deps.files.stageStream(ctx.address.workspaceId, {
|
|
663
|
+
mediaType: asset.mediaType,
|
|
664
|
+
name: asset.name,
|
|
665
|
+
body
|
|
666
|
+
}, {
|
|
667
|
+
outputKind: input.outputKind,
|
|
668
|
+
signal
|
|
669
|
+
}));
|
|
670
|
+
}
|
|
671
|
+
phase = "commit";
|
|
672
|
+
const files = await deps.files.commitBatch(ctx.address.workspaceId, ctx.sessionId, staged, {
|
|
673
|
+
signal,
|
|
674
|
+
assertCurrent: admission.assertCurrent
|
|
675
|
+
});
|
|
676
|
+
return {
|
|
677
|
+
...result.text === void 0 ? {} : { text: result.text },
|
|
678
|
+
files
|
|
679
|
+
};
|
|
680
|
+
} catch (error) {
|
|
681
|
+
if (error instanceof GenerationFailure) {
|
|
682
|
+
if (submitted && error.resultState === "not-submitted") throw new GenerationFailure(error.code, error.message, responded ? "not-delivered" : "unknown", error.status, error.retryAt);
|
|
683
|
+
throw error;
|
|
684
|
+
}
|
|
685
|
+
const record = error;
|
|
686
|
+
const cancelled = ctx.signal.aborted || target.lifetime.aborted || controller.signal.aborted || record.name === "AbortError";
|
|
687
|
+
const timeout = record.name === "TimeoutError" || deadline.aborted;
|
|
688
|
+
const reported = typeof record.code === "string" && FAILURE_CODES.has(record.code) && (record.authored || record.name === "GenerationFailure") ? record.code : void 0;
|
|
689
|
+
const code = cancelled ? "cancelled" : timeout ? "timeout" : phase === "commit" ? "storage" : phase === "stage" ? record.problem === "too-large" ? "too-large" : "invalid-output" : reported ?? (record.name === "CredentialUnavailableError" ? "credential-unavailable" : record.status === 401 ? "expired" : record.status === 403 ? "forbidden" : record.status === 429 ? "quota" : "provider");
|
|
690
|
+
let message = (record.authored || reported) && typeof record.message === "string" ? record.message : code === "cancelled" ? "generation was cancelled" : code === "timeout" ? "generation timed out" : code === "storage" ? "the generated files could not be saved" : code === "invalid-output" ? "the generated media is invalid" : code === "expired" ? "your sign-in expired; sign in again" : "the generation provider call failed";
|
|
691
|
+
if (typeof record.providerText === "string") message += `: ${record.providerText.slice(0, 65536)}`;
|
|
692
|
+
for (const secret of Object.values(secrets)) if (secret) message = message.split(secret).join("****");
|
|
693
|
+
const retryAt = typeof record.retryAt === "number" && Number.isFinite(record.retryAt) && record.retryAt > 0 && record.retryAt <= 864e13 ? record.retryAt : void 0;
|
|
694
|
+
if (retryAt !== void 0) message += ` Retry after ${new Date(retryAt).toISOString()}.`;
|
|
695
|
+
const declaredState = record.name === "GenerationFailure" && reported !== void 0 && (record.resultState === "not-submitted" || record.resultState === "unknown" || record.resultState === "not-delivered") ? record.resultState : void 0;
|
|
696
|
+
throw new GenerationFailure(code, message, responded ? "not-delivered" : declaredState ?? (submitted ? "unknown" : "not-submitted"), typeof record.status === "number" ? record.status : void 0, retryAt);
|
|
697
|
+
} finally {
|
|
698
|
+
active.delete(admission);
|
|
699
|
+
deps.files.discard(staged);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
};
|
|
703
|
+
}
|
|
704
|
+
return {
|
|
705
|
+
bindInputs(owner) {
|
|
706
|
+
return { async prepare(ctx, selectors) {
|
|
707
|
+
deps.authenticate(owner, ctx);
|
|
708
|
+
ctx.signal.throwIfAborted();
|
|
709
|
+
const revision = deps.sessionRevision(ctx.sessionId);
|
|
710
|
+
const result = await deps.prepare(ctx, selectors);
|
|
711
|
+
ctx.signal.throwIfAborted();
|
|
712
|
+
if (!Array.isArray(result)) return result;
|
|
713
|
+
if (revision !== deps.sessionRevision(ctx.sessionId)) fail("cancelled", "the session history changed");
|
|
714
|
+
const token = Object.freeze({});
|
|
715
|
+
prepared.set(token, {
|
|
716
|
+
owner,
|
|
717
|
+
ctx,
|
|
718
|
+
callId: ctx.toolCallId,
|
|
719
|
+
sessionId: ctx.sessionId,
|
|
720
|
+
revision,
|
|
721
|
+
files: result
|
|
722
|
+
});
|
|
723
|
+
return {
|
|
724
|
+
kind: "ready",
|
|
725
|
+
inputs: token
|
|
726
|
+
};
|
|
727
|
+
} };
|
|
728
|
+
},
|
|
729
|
+
bind,
|
|
730
|
+
bindCoordinator: (owner) => bind(owner, true),
|
|
731
|
+
invalidate() {
|
|
732
|
+
for (const [key, kept] of ports) {
|
|
733
|
+
const target = deps.target(kept.ref);
|
|
734
|
+
if (!target?.enabled || target.owner !== kept.owner || target.registrationEpoch !== kept.epoch || target.protocolEpoch !== kept.protocolEpoch || target.connectionRevision !== kept.revision) ports.delete(key);
|
|
735
|
+
}
|
|
736
|
+
for (const call of active) try {
|
|
737
|
+
call.assertCurrent();
|
|
738
|
+
} catch {
|
|
739
|
+
call.controller.abort();
|
|
740
|
+
}
|
|
741
|
+
},
|
|
742
|
+
stop() {
|
|
743
|
+
stopped = true;
|
|
744
|
+
ports.clear();
|
|
745
|
+
for (const call of active) call.controller.abort();
|
|
746
|
+
}
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
//#endregion
|
|
750
|
+
//#region src/runtime/models/generation-inputs.ts
|
|
751
|
+
/** Ordered, host-authorized image references for generation tool submissions. */
|
|
752
|
+
const FILE_ORIGINAL_LIMIT = 20971520;
|
|
753
|
+
const IMAGE_MEDIA_TYPES = [
|
|
754
|
+
"image/png",
|
|
755
|
+
"image/jpeg",
|
|
756
|
+
"image/gif",
|
|
757
|
+
"image/webp"
|
|
758
|
+
];
|
|
759
|
+
const selectorsSchema = z.union([z.object({ recent: z.number().int().min(1).max(16) }).strict(), z.array(z.union([z.object({ fileId: z.string().regex(/^[0-9a-f]{64}$/) }).strict(), z.object({ path: z.string().min(1) }).strict()])).max(16)]);
|
|
760
|
+
/** Prepares authorized bytes only after every selected path has passed the fence. */
|
|
761
|
+
function createGenerationInputReader(deps) {
|
|
762
|
+
return async (ctx, selectors) => {
|
|
763
|
+
ctx.signal.throwIfAborted();
|
|
764
|
+
const parsed = selectorsSchema.safeParse(selectors);
|
|
765
|
+
if (!parsed.success) return {
|
|
766
|
+
kind: "refused",
|
|
767
|
+
message: "invalid generation input selectors"
|
|
768
|
+
};
|
|
769
|
+
selectors = parsed.data;
|
|
770
|
+
const history = deps.history(ctx);
|
|
771
|
+
const ordered = [];
|
|
772
|
+
for (const message of history) for (const part of message.content) {
|
|
773
|
+
if (part.type === "file") ordered.push(part.file);
|
|
774
|
+
if (part.type === "tool-result") ordered.push(...part.files ?? []);
|
|
775
|
+
}
|
|
776
|
+
const latest = new Map(ordered.map((file, index) => [file.id, {
|
|
777
|
+
file,
|
|
778
|
+
index
|
|
779
|
+
}]));
|
|
780
|
+
let selection;
|
|
781
|
+
if ("recent" in selectors) {
|
|
782
|
+
const recent = [...latest.values()].filter((one) => IMAGE_MEDIA_TYPES.includes(one.file.mediaType)).sort((a, b) => a.index - b.index).slice(-selectors.recent);
|
|
783
|
+
if (recent.length !== selectors.recent) return {
|
|
784
|
+
kind: "refused",
|
|
785
|
+
message: "the session does not contain enough recent images"
|
|
786
|
+
};
|
|
787
|
+
selection = recent.map((one) => ({ fileId: one.file.id }));
|
|
788
|
+
} else selection = selectors;
|
|
789
|
+
const paths = /* @__PURE__ */ new Map();
|
|
790
|
+
for (const item of selection) if ("fileId" in item) {
|
|
791
|
+
const selected = latest.get(item.fileId);
|
|
792
|
+
if (!selected) return {
|
|
793
|
+
kind: "refused",
|
|
794
|
+
message: "the selected file is not in this session history"
|
|
795
|
+
};
|
|
796
|
+
if (!IMAGE_MEDIA_TYPES.includes(selected.file.mediaType)) return {
|
|
797
|
+
kind: "refused",
|
|
798
|
+
message: "the selected file is not an image"
|
|
799
|
+
};
|
|
800
|
+
}
|
|
801
|
+
const requested = selection.flatMap((item, index) => "path" in item ? [{
|
|
802
|
+
index,
|
|
803
|
+
path: item.path
|
|
804
|
+
}] : []);
|
|
805
|
+
if (requested.length > 0) {
|
|
806
|
+
const verdict = deps.scope.allowsBatch(ctx, requested.map((item) => item.path), "read");
|
|
807
|
+
if (!verdict.allowed) {
|
|
808
|
+
if (verdict.reason === "ask" && "awaitHuman" in verdict.ask) return {
|
|
809
|
+
kind: "ask",
|
|
810
|
+
result: verdict.ask
|
|
811
|
+
};
|
|
812
|
+
return {
|
|
813
|
+
kind: "refused",
|
|
814
|
+
message: "message" in verdict ? verdict.message : "content" in verdict.ask ? verdict.ask.content : "reading the selected image paths was refused"
|
|
815
|
+
};
|
|
816
|
+
}
|
|
817
|
+
for (const [offset, item] of requested.entries()) {
|
|
818
|
+
const path = verdict.paths[offset];
|
|
819
|
+
if (path === void 0) return {
|
|
820
|
+
kind: "refused",
|
|
821
|
+
message: "the image path was not authorized"
|
|
822
|
+
};
|
|
823
|
+
paths.set(item.index, path);
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
function pathsCurrent() {
|
|
827
|
+
const verdict = deps.scope.allowsBatch(ctx, requested.map((item) => item.path), "read");
|
|
828
|
+
return verdict.allowed && requested.every((item, offset) => verdict.paths[offset] === paths.get(item.index));
|
|
829
|
+
}
|
|
830
|
+
const files = [];
|
|
831
|
+
try {
|
|
832
|
+
for (const [index, item] of selection.entries()) {
|
|
833
|
+
ctx.signal.throwIfAborted();
|
|
834
|
+
let bytes;
|
|
835
|
+
if ("path" in item) {
|
|
836
|
+
const path = paths.get(index);
|
|
837
|
+
if (path === void 0) return {
|
|
838
|
+
kind: "refused",
|
|
839
|
+
message: "the image path was not authorized"
|
|
840
|
+
};
|
|
841
|
+
const descriptor = await (deps.open ?? open)(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0));
|
|
842
|
+
try {
|
|
843
|
+
ctx.signal.throwIfAborted();
|
|
844
|
+
const before = await descriptor.stat();
|
|
845
|
+
if (!before.isFile() || before.size > FILE_ORIGINAL_LIMIT) return {
|
|
846
|
+
kind: "refused",
|
|
847
|
+
message: "the image exceeds the input size limit"
|
|
848
|
+
};
|
|
849
|
+
if (!pathsCurrent() || await realpath(path) !== path || !sameFile(before, await stat(path))) return {
|
|
850
|
+
kind: "refused",
|
|
851
|
+
message: "the image path changed before it was read"
|
|
852
|
+
};
|
|
853
|
+
const parts = [];
|
|
854
|
+
let size = 0;
|
|
855
|
+
for (;;) {
|
|
856
|
+
ctx.signal.throwIfAborted();
|
|
857
|
+
const chunk = Buffer.alloc(Math.min(65536, FILE_ORIGINAL_LIMIT - size + 1));
|
|
858
|
+
const result = await descriptor.read(chunk, 0, chunk.length, size);
|
|
859
|
+
ctx.signal.throwIfAborted();
|
|
860
|
+
if (result.bytesRead === 0) break;
|
|
861
|
+
size += result.bytesRead;
|
|
862
|
+
if (size > FILE_ORIGINAL_LIMIT) return {
|
|
863
|
+
kind: "refused",
|
|
864
|
+
message: "the image exceeds the input size limit"
|
|
865
|
+
};
|
|
866
|
+
parts.push(chunk.subarray(0, result.bytesRead));
|
|
867
|
+
}
|
|
868
|
+
const after = await descriptor.stat();
|
|
869
|
+
if (!sameFile(before, after) || size !== after.size) return {
|
|
870
|
+
kind: "refused",
|
|
871
|
+
message: "the image changed while it was read"
|
|
872
|
+
};
|
|
873
|
+
if (!pathsCurrent() || await realpath(path) !== path || !sameFile(after, await stat(path))) return {
|
|
874
|
+
kind: "refused",
|
|
875
|
+
message: "the image path changed while it was read"
|
|
876
|
+
};
|
|
877
|
+
bytes = Buffer.concat(parts, size);
|
|
878
|
+
} finally {
|
|
879
|
+
await descriptor.close();
|
|
880
|
+
}
|
|
881
|
+
} else bytes = await deps.files.read(ctx.address.workspaceId, item.fileId);
|
|
882
|
+
if (!bytes || bytes.length > FILE_ORIGINAL_LIMIT) return {
|
|
883
|
+
kind: "refused",
|
|
884
|
+
message: "the selected image is missing or exceeds the input size limit"
|
|
885
|
+
};
|
|
886
|
+
const info = await deps.probeImage(bytes);
|
|
887
|
+
ctx.signal.throwIfAborted();
|
|
888
|
+
const held = bytes;
|
|
889
|
+
files.push({
|
|
890
|
+
mediaType: info.mediaType,
|
|
891
|
+
size: held.length,
|
|
892
|
+
async *read(signal) {
|
|
893
|
+
signal.throwIfAborted();
|
|
894
|
+
yield held;
|
|
895
|
+
signal.throwIfAborted();
|
|
896
|
+
}
|
|
897
|
+
});
|
|
898
|
+
}
|
|
899
|
+
return files;
|
|
900
|
+
} catch {
|
|
901
|
+
ctx.signal.throwIfAborted();
|
|
902
|
+
return {
|
|
903
|
+
kind: "refused",
|
|
904
|
+
message: "the selected image could not be read"
|
|
905
|
+
};
|
|
906
|
+
}
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
function sameFile(before, after) {
|
|
910
|
+
return before.ino === after.ino && before.dev === after.dev && before.size === after.size && before.mtimeMs === after.mtimeMs && before.ctimeMs === after.ctimeMs;
|
|
911
|
+
}
|
|
912
|
+
//#endregion
|
|
913
|
+
//#region src/runtime/models/generation-services.ts
|
|
914
|
+
/** Composes resource authorization with capability-owned image execution and atomic publication. */
|
|
915
|
+
function createGenerationServices(input) {
|
|
916
|
+
const { ctx, manager, host } = input;
|
|
917
|
+
const deps = {
|
|
918
|
+
files: host.files,
|
|
919
|
+
security: host.security,
|
|
920
|
+
contexts: host.contexts,
|
|
921
|
+
approval: input.approval,
|
|
922
|
+
transport: host.transport,
|
|
923
|
+
probeImage: host.media.probeImage
|
|
924
|
+
};
|
|
925
|
+
function session(id) {
|
|
926
|
+
return host.contexts.session(id);
|
|
927
|
+
}
|
|
928
|
+
const identities = /* @__PURE__ */ new Map();
|
|
929
|
+
function target(ref) {
|
|
930
|
+
const row = manager.store.findConnection(ref.providerId);
|
|
931
|
+
if (!row) return void 0;
|
|
932
|
+
const provider = manager.registries.models.entry(row.type);
|
|
933
|
+
const selected = manager.store.findChannel(ref.providerId, "image");
|
|
934
|
+
const capability = manager.registries.capabilities.entry("image");
|
|
935
|
+
const protocol = selected && manager.registries.protocols.entry(selected.protocolId);
|
|
936
|
+
if (!provider || !selected || !protocol || !capability || !protocol.type.adapters.image) return void 0;
|
|
937
|
+
const configuredModels = manager.store.listBindings(ref.providerId, "image").filter((binding) => !binding.invalid).map((binding) => ({
|
|
938
|
+
id: binding.ref.model,
|
|
939
|
+
...binding.execution ? { execution: binding.execution } : {}
|
|
940
|
+
}));
|
|
941
|
+
const fingerprint = JSON.stringify([
|
|
942
|
+
row.revision,
|
|
943
|
+
selected.revision,
|
|
944
|
+
configuredModels,
|
|
945
|
+
provider.registrationEpoch,
|
|
946
|
+
protocol.registrationEpoch,
|
|
947
|
+
capability.registrationEpoch
|
|
948
|
+
]);
|
|
949
|
+
const prior = identities.get(ref.providerId);
|
|
950
|
+
const revision = prior?.fingerprint === fingerprint ? prior.revision : (prior?.revision ?? 0) + 1;
|
|
951
|
+
identities.set(ref.providerId, {
|
|
952
|
+
fingerprint,
|
|
953
|
+
revision
|
|
954
|
+
});
|
|
955
|
+
const original = protocol.type.adapters.image;
|
|
956
|
+
const wrapped = {
|
|
957
|
+
...protocol.type,
|
|
958
|
+
adapters: {
|
|
959
|
+
...protocol.type.adapters,
|
|
960
|
+
image: {
|
|
961
|
+
...original,
|
|
962
|
+
createPort: () => ({ async generate(request, signal) {
|
|
963
|
+
const lease = await manager.acquire({
|
|
964
|
+
...ref,
|
|
965
|
+
capabilityId: "image"
|
|
966
|
+
}, signal);
|
|
967
|
+
try {
|
|
968
|
+
return await (await lease.runtime.createPort()).generate(request, lease.signal);
|
|
969
|
+
} finally {
|
|
970
|
+
await lease.release();
|
|
971
|
+
}
|
|
972
|
+
} })
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
};
|
|
976
|
+
return {
|
|
977
|
+
type: provider.type,
|
|
978
|
+
protocol: wrapped,
|
|
979
|
+
protocolEpoch: protocol.registrationEpoch,
|
|
980
|
+
options: selected.options,
|
|
981
|
+
owner: provider.owner,
|
|
982
|
+
registrationEpoch: provider.registrationEpoch,
|
|
983
|
+
lifetime: AbortSignal.any([
|
|
984
|
+
provider.lifetime,
|
|
985
|
+
protocol.lifetime,
|
|
986
|
+
capability.lifetime
|
|
987
|
+
]),
|
|
988
|
+
enabled: row.enabled,
|
|
989
|
+
config: {
|
|
990
|
+
...row.config,
|
|
991
|
+
endpointId: row.endpointId,
|
|
992
|
+
protocolId: protocol.type.type
|
|
993
|
+
},
|
|
994
|
+
configuredModels,
|
|
995
|
+
connectionRevision: revision
|
|
996
|
+
};
|
|
997
|
+
}
|
|
998
|
+
function descriptors(providerId, found = target({
|
|
999
|
+
providerId,
|
|
1000
|
+
model: ""
|
|
1001
|
+
})) {
|
|
1002
|
+
return found?.enabled ? generationModelsOf(found.protocol, {
|
|
1003
|
+
providerId,
|
|
1004
|
+
config: found.config,
|
|
1005
|
+
configuredModels: found.configuredModels
|
|
1006
|
+
}) : [];
|
|
1007
|
+
}
|
|
1008
|
+
function current(call) {
|
|
1009
|
+
const ref = session(call.sessionId)?.model;
|
|
1010
|
+
if (!ref) return null;
|
|
1011
|
+
const row = manager.store.findConnection(ref.providerId);
|
|
1012
|
+
const entry = row && manager.registries.models.entry(row.type);
|
|
1013
|
+
if (!row || !entry) return null;
|
|
1014
|
+
return {
|
|
1015
|
+
ref,
|
|
1016
|
+
type: row.type,
|
|
1017
|
+
owner: entry.owner,
|
|
1018
|
+
registrationEpoch: entry.registrationEpoch,
|
|
1019
|
+
connectionRevision: row.revision,
|
|
1020
|
+
loginState: null,
|
|
1021
|
+
chatInputKinds: []
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
const approved = /* @__PURE__ */ new WeakMap();
|
|
1025
|
+
const generation = createGenerationHost({
|
|
1026
|
+
files: host.files,
|
|
1027
|
+
descriptors: generationModelsOf,
|
|
1028
|
+
authenticate: (owner, call) => host.contexts.authenticate(owner, call),
|
|
1029
|
+
current,
|
|
1030
|
+
target,
|
|
1031
|
+
connections(owner) {
|
|
1032
|
+
return manager.store.listConnections().flatMap((row) => {
|
|
1033
|
+
const entry = manager.registries.models.entry(row.type);
|
|
1034
|
+
if (!row.enabled || !entry || owner !== void 0 && entry.owner !== owner) return [];
|
|
1035
|
+
const found = target({
|
|
1036
|
+
providerId: row.providerId,
|
|
1037
|
+
model: ""
|
|
1038
|
+
});
|
|
1039
|
+
if (!found) return [];
|
|
1040
|
+
return [{
|
|
1041
|
+
providerId: row.providerId,
|
|
1042
|
+
type: row.type,
|
|
1043
|
+
owner: entry.owner,
|
|
1044
|
+
label: row.name,
|
|
1045
|
+
loginState: null,
|
|
1046
|
+
models: descriptors(row.providerId, found)
|
|
1047
|
+
}];
|
|
1048
|
+
});
|
|
1049
|
+
},
|
|
1050
|
+
credential(ref, type) {
|
|
1051
|
+
if (type.login && manager.connections().find((row) => row.providerId === ref.providerId)?.login?.state !== "signed-in") throw new GenerationFailure("not-signed-in", "Sign in to the selected connection in model settings.");
|
|
1052
|
+
const availability = manager.availability({
|
|
1053
|
+
...ref,
|
|
1054
|
+
capabilityId: "image"
|
|
1055
|
+
});
|
|
1056
|
+
if (!availability.available && availability.reason === "credential-unavailable") throw new GenerationFailure("credential-unavailable", "The connection credential is unavailable; check model settings.");
|
|
1057
|
+
return { secrets: {} };
|
|
1058
|
+
},
|
|
1059
|
+
sessionRevision: (id) => session(id)?.historyGeneration ?? -1,
|
|
1060
|
+
prepare: createGenerationInputReader({
|
|
1061
|
+
files: host.files,
|
|
1062
|
+
scope: host.security,
|
|
1063
|
+
probeImage: host.media.probeImage,
|
|
1064
|
+
history: (call) => host.contexts.history(call)
|
|
1065
|
+
}),
|
|
1066
|
+
async authorize(ctx, origins) {
|
|
1067
|
+
ctx.signal.throwIfAborted();
|
|
1068
|
+
const needs = [];
|
|
1069
|
+
for (const origin of origins) {
|
|
1070
|
+
const url = new URL(origin);
|
|
1071
|
+
if (url.protocol !== "https:" || url.origin !== origin || url.username || url.password) return {
|
|
1072
|
+
kind: "refused",
|
|
1073
|
+
message: "artifact origins must be exact HTTPS origins"
|
|
1074
|
+
};
|
|
1075
|
+
const verdict = await deps.security.judgeAddress(origin);
|
|
1076
|
+
ctx.signal.throwIfAborted();
|
|
1077
|
+
if (verdict.kind !== "allowed") return {
|
|
1078
|
+
kind: "refused",
|
|
1079
|
+
message: "the artifact origin address was refused or unresolved"
|
|
1080
|
+
};
|
|
1081
|
+
const port = Number(url.port || 443);
|
|
1082
|
+
if (!deps.security.holdsHost(verdict.host, port, "connect")) needs.push({
|
|
1083
|
+
host: verdict.host,
|
|
1084
|
+
port
|
|
1085
|
+
});
|
|
1086
|
+
}
|
|
1087
|
+
ctx.signal.throwIfAborted();
|
|
1088
|
+
if (needs.length) {
|
|
1089
|
+
const decision = deps.approval.decide(ctx, {
|
|
1090
|
+
id: "generationArtifactOrigins",
|
|
1091
|
+
subject: "Download generated media",
|
|
1092
|
+
reason: "The provider will return generated media from these hosts.",
|
|
1093
|
+
targets: needs.map((one) => ({
|
|
1094
|
+
access: "connect",
|
|
1095
|
+
target: `${one.host}:${one.port}`
|
|
1096
|
+
})),
|
|
1097
|
+
accepts: [
|
|
1098
|
+
"allowed-once",
|
|
1099
|
+
"allowed-always",
|
|
1100
|
+
"rejected"
|
|
1101
|
+
],
|
|
1102
|
+
globalResource: true
|
|
1103
|
+
});
|
|
1104
|
+
if (decision.kind === "ask") {
|
|
1105
|
+
if ("awaitHuman" in decision.result) return {
|
|
1106
|
+
kind: "ask",
|
|
1107
|
+
result: decision.result
|
|
1108
|
+
};
|
|
1109
|
+
return {
|
|
1110
|
+
kind: "refused",
|
|
1111
|
+
message: "artifact authorization is unavailable"
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
1114
|
+
if (decision.result.kind !== "allowed-once" && decision.result.kind !== "allowed-always") return {
|
|
1115
|
+
kind: "refused",
|
|
1116
|
+
message: "artifact authorization was refused"
|
|
1117
|
+
};
|
|
1118
|
+
ctx.signal.throwIfAborted();
|
|
1119
|
+
if (decision.result.kind === "allowed-always") deps.security.recordHosts(needs.map((one) => ({
|
|
1120
|
+
host: one.host,
|
|
1121
|
+
port: one.port,
|
|
1122
|
+
operations: ["connect"]
|
|
1123
|
+
})));
|
|
1124
|
+
}
|
|
1125
|
+
approved.set(ctx, new Set(origins));
|
|
1126
|
+
return { kind: "ready" };
|
|
1127
|
+
},
|
|
1128
|
+
artifacts(ctx, origins) {
|
|
1129
|
+
return { async fetch(url, signal) {
|
|
1130
|
+
let currentUrl = url;
|
|
1131
|
+
for (let hop = 0; hop <= 5; hop++) {
|
|
1132
|
+
signal.throwIfAborted();
|
|
1133
|
+
const parsed = new URL(currentUrl);
|
|
1134
|
+
if (!origins.includes(parsed.origin) || !approved.get(ctx)?.has(parsed.origin) || parsed.username || parsed.password) throw new GenerationFailure("forbidden", "the artifact URL has an unauthorized origin", "not-delivered");
|
|
1135
|
+
const address = await deps.security.judgeAddress(currentUrl);
|
|
1136
|
+
if (address.kind !== "allowed") throw new GenerationFailure("forbidden", "the artifact address was refused or unresolved", "not-delivered");
|
|
1137
|
+
signal.throwIfAborted();
|
|
1138
|
+
const response = await deps.transport({
|
|
1139
|
+
url: currentUrl,
|
|
1140
|
+
addresses: address.addresses,
|
|
1141
|
+
headers: {
|
|
1142
|
+
"user-agent": "yaoguang",
|
|
1143
|
+
accept: "*/*"
|
|
1144
|
+
},
|
|
1145
|
+
signal
|
|
1146
|
+
});
|
|
1147
|
+
if ([
|
|
1148
|
+
301,
|
|
1149
|
+
302,
|
|
1150
|
+
303,
|
|
1151
|
+
307,
|
|
1152
|
+
308
|
|
1153
|
+
].includes(response.status)) {
|
|
1154
|
+
response.cancel();
|
|
1155
|
+
if (hop === 5 || !response.headers.location) throw new GenerationFailure("network", "the artifact redirect limit was exceeded", "not-delivered");
|
|
1156
|
+
currentUrl = new URL(response.headers.location, currentUrl).href;
|
|
1157
|
+
continue;
|
|
1158
|
+
}
|
|
1159
|
+
if (signal.aborted) {
|
|
1160
|
+
response.cancel();
|
|
1161
|
+
signal.throwIfAborted();
|
|
1162
|
+
}
|
|
1163
|
+
return artifactResponse(response, signal);
|
|
1164
|
+
}
|
|
1165
|
+
throw new GenerationFailure("network", "the artifact download failed", "not-delivered");
|
|
1166
|
+
} };
|
|
1167
|
+
}
|
|
1168
|
+
});
|
|
1169
|
+
const dispose = manager.onChanged(() => generation.invalidate());
|
|
1170
|
+
ctx.onDispose(() => {
|
|
1171
|
+
dispose();
|
|
1172
|
+
});
|
|
1173
|
+
ctx.signal.addEventListener("abort", () => generation.stop(), { once: true });
|
|
1174
|
+
return {
|
|
1175
|
+
...generation,
|
|
1176
|
+
identity(ref) {
|
|
1177
|
+
const found = target(ref);
|
|
1178
|
+
return found?.enabled && !found.lifetime.aborted ? JSON.stringify([
|
|
1179
|
+
found.owner,
|
|
1180
|
+
found.registrationEpoch,
|
|
1181
|
+
found.protocolEpoch,
|
|
1182
|
+
found.connectionRevision
|
|
1183
|
+
]) : void 0;
|
|
1184
|
+
},
|
|
1185
|
+
availability(address, id) {
|
|
1186
|
+
const ref = id ? session(id)?.model ?? null : null;
|
|
1187
|
+
const row = ref ? manager.store.findConnection(ref.providerId) : void 0;
|
|
1188
|
+
const entry = row ? manager.registries.models.entry(row.type) : void 0;
|
|
1189
|
+
let toolsAllowed;
|
|
1190
|
+
let chatInputKinds = [];
|
|
1191
|
+
try {
|
|
1192
|
+
const policy = ctx.services.get("model-tool-policy").availability(address, id);
|
|
1193
|
+
toolsAllowed = policy.toolsAllowed;
|
|
1194
|
+
chatInputKinds = policy.chatInputKinds;
|
|
1195
|
+
} catch {}
|
|
1196
|
+
return {
|
|
1197
|
+
toolsAllowed,
|
|
1198
|
+
sessionId: id ?? "",
|
|
1199
|
+
address,
|
|
1200
|
+
model: ref,
|
|
1201
|
+
type: row?.type ?? null,
|
|
1202
|
+
owner: entry?.owner ?? null,
|
|
1203
|
+
chatInputKinds: [...chatInputKinds],
|
|
1204
|
+
generation: ref ? descriptors(ref.providerId).map(({ id, label, outputKinds }) => ({
|
|
1205
|
+
id,
|
|
1206
|
+
label,
|
|
1207
|
+
outputKinds
|
|
1208
|
+
})) : [],
|
|
1209
|
+
loginState: null
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
};
|
|
1213
|
+
}
|
|
1214
|
+
/** Keeps a downloaded body in the same cancellation lifetime as its generation call. */
|
|
1215
|
+
function artifactResponse(response, signal) {
|
|
1216
|
+
const iterator = response.body?.[Symbol.asyncIterator]();
|
|
1217
|
+
let finished = false;
|
|
1218
|
+
let streamController;
|
|
1219
|
+
const release = () => {
|
|
1220
|
+
if (finished) return;
|
|
1221
|
+
finished = true;
|
|
1222
|
+
signal.removeEventListener("abort", abort);
|
|
1223
|
+
response.cancel();
|
|
1224
|
+
if (iterator?.return) Promise.resolve().then(() => iterator.return?.()).catch(() => void 0);
|
|
1225
|
+
};
|
|
1226
|
+
const abort = () => {
|
|
1227
|
+
release();
|
|
1228
|
+
streamController?.error(signal.reason);
|
|
1229
|
+
};
|
|
1230
|
+
const body = iterator ? new ReadableStream({
|
|
1231
|
+
start(controller) {
|
|
1232
|
+
streamController = controller;
|
|
1233
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
1234
|
+
if (signal.aborted) abort();
|
|
1235
|
+
},
|
|
1236
|
+
async pull(controller) {
|
|
1237
|
+
try {
|
|
1238
|
+
signal.throwIfAborted();
|
|
1239
|
+
const next = await iterator.next();
|
|
1240
|
+
signal.throwIfAborted();
|
|
1241
|
+
if (finished) return;
|
|
1242
|
+
if (next.done) {
|
|
1243
|
+
controller.close();
|
|
1244
|
+
release();
|
|
1245
|
+
} else controller.enqueue(next.value);
|
|
1246
|
+
} catch (error) {
|
|
1247
|
+
if (!finished) controller.error(error);
|
|
1248
|
+
release();
|
|
1249
|
+
}
|
|
1250
|
+
},
|
|
1251
|
+
cancel() {
|
|
1252
|
+
release();
|
|
1253
|
+
}
|
|
1254
|
+
}) : null;
|
|
1255
|
+
if (body === null) release();
|
|
1256
|
+
try {
|
|
1257
|
+
return new Response(body, {
|
|
1258
|
+
status: response.status,
|
|
1259
|
+
statusText: response.statusText,
|
|
1260
|
+
headers: Object.fromEntries(Object.entries(response.headers).filter((entry) => entry[1] !== void 0))
|
|
1261
|
+
});
|
|
1262
|
+
} catch (error) {
|
|
1263
|
+
release();
|
|
1264
|
+
throw error;
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
//#endregion
|
|
1268
|
+
//#region src/index.ts
|
|
1269
|
+
/** Image generation and editing capability, independent from the selected Chat provider. */
|
|
1270
|
+
/** Tools owned by the Image capability. */
|
|
1271
|
+
const TOOL_NAMES = ["model_image_generate"];
|
|
1272
|
+
/** Image tool settings withdraw the tool and its guidance together. */
|
|
1273
|
+
const ImageToolConfig = z.strictObject({ disabled: z.array(z.enum(TOOL_NAMES)).default([]) }).default({ disabled: [] });
|
|
1274
|
+
const plugin = {
|
|
1275
|
+
name: "@eum-agent/llm-capability-image",
|
|
1276
|
+
requires: [
|
|
1277
|
+
"model-management",
|
|
1278
|
+
"agent-capability-host",
|
|
1279
|
+
"approval"
|
|
1280
|
+
],
|
|
1281
|
+
provides: ["generation-runtime"],
|
|
1282
|
+
configSchema: ImageToolConfig,
|
|
1283
|
+
apply(ctx, config) {
|
|
1284
|
+
ctx.modelCapabilities.register({
|
|
1285
|
+
...imageCapability,
|
|
1286
|
+
tools: config.disabled.includes("model_image_generate") ? [] : TOOL_NAMES
|
|
1287
|
+
});
|
|
1288
|
+
const manager = ctx.services.get("model-management");
|
|
1289
|
+
const generation = createGenerationServices({
|
|
1290
|
+
ctx,
|
|
1291
|
+
manager,
|
|
1292
|
+
host: ctx.services.get("agent-capability-host"),
|
|
1293
|
+
approval: ctx.services.get("approval")
|
|
1294
|
+
});
|
|
1295
|
+
ctx.services.provide("generation-runtime", {
|
|
1296
|
+
bind: generation.bind,
|
|
1297
|
+
bindInputs: generation.bindInputs,
|
|
1298
|
+
availability: generation.availability,
|
|
1299
|
+
invalidate: generation.invalidate
|
|
1300
|
+
});
|
|
1301
|
+
if (!config.disabled.includes("model_image_generate")) ctx.tools.register(imageGenerationTool({
|
|
1302
|
+
generation: generation.bindCoordinator(ctx.pluginName),
|
|
1303
|
+
inputs: generation.bindInputs(ctx.pluginName),
|
|
1304
|
+
defaultModel: () => {
|
|
1305
|
+
const ref = manager.store.getDefault("image")?.ref;
|
|
1306
|
+
return ref ? {
|
|
1307
|
+
providerId: ref.providerId,
|
|
1308
|
+
model: ref.model
|
|
1309
|
+
} : null;
|
|
1310
|
+
},
|
|
1311
|
+
params: (ref, operation) => manager.store.findBinding({
|
|
1312
|
+
...ref,
|
|
1313
|
+
capabilityId: "image"
|
|
1314
|
+
})?.params[operation] ?? {},
|
|
1315
|
+
identity: generation.identity
|
|
1316
|
+
}));
|
|
1317
|
+
ctx.tools.guide(GUIDANCE);
|
|
1318
|
+
const ui = fileURLToPath(new URL("../ui/ui.js", import.meta.url));
|
|
1319
|
+
if (existsSync(ui)) ctx.client.registerUi({ module: ui });
|
|
1320
|
+
}
|
|
1321
|
+
};
|
|
1322
|
+
//#endregion
|
|
1323
|
+
export { ImageToolConfig, TOOL_NAMES, plugin as default };
|