@contenthero/mcp 0.4.7 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.js +1 -1
- package/dist/format.d.ts +150 -6
- package/dist/format.d.ts.map +1 -1
- package/dist/format.js +217 -12
- package/dist/format.js.map +1 -1
- package/dist/groups.js +1 -1
- package/dist/groups.js.map +1 -1
- package/dist/server.d.ts +68 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +497 -57
- package/dist/server.js.map +1 -1
- package/dist/widget/generation.html +516 -0
- package/dist/widget-uri.d.ts +12 -0
- package/dist/widget-uri.d.ts.map +1 -0
- package/dist/widget-uri.js +12 -0
- package/dist/widget-uri.js.map +1 -0
- package/package.json +13 -4
package/dist/server.js
CHANGED
|
@@ -33,11 +33,28 @@ import { readFileSync } from 'node:fs';
|
|
|
33
33
|
import { fileURLToPath } from 'node:url';
|
|
34
34
|
import { dirname, join } from 'node:path';
|
|
35
35
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
36
|
+
/**
|
|
37
|
+
* ⚠️⚠️ **THE CONSTANTS ONLY, NOT THE `./server` HELPERS, AND THAT IS DELIBERATE.**
|
|
38
|
+
*
|
|
39
|
+
* `@modelcontextprotocol/ext-apps@2` targets the SPLIT packages (`@modelcontextprotocol/server`), while this
|
|
40
|
+
* server is built on the monolithic `@modelcontextprotocol/sdk@1.26`. Its `registerAppResource` therefore
|
|
41
|
+
* typechecks against a different `ResourceMetadata` than ours and rejects `description`.
|
|
42
|
+
*
|
|
43
|
+
* ⭐ The helper is convenience over a two-line contract: a resource whose mimeType is the app profile, and a
|
|
44
|
+
* tool result whose `_meta` names it. Registering through OUR `server.registerResource` keeps one server
|
|
45
|
+
* abstraction instead of two, and the STRINGS still come from the package, so the part that must match the
|
|
46
|
+
* spec has a single source. Migrating to the split SDK is its own piece of work, not a prerequisite for this.
|
|
47
|
+
*/
|
|
48
|
+
import { RESOURCE_MIME_TYPE, RESOURCE_URI_META_KEY } from '@modelcontextprotocol/ext-apps';
|
|
36
49
|
import { z } from 'zod';
|
|
37
50
|
import { GenerationTimeoutError, pendingOutputId, } from '@contenthero/sdk';
|
|
38
51
|
import { getClient as defaultGetClient } from './client.js';
|
|
52
|
+
/** This module's own directory, so the widget is read from the PACKAGE rather than from the cwd. */
|
|
53
|
+
const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
|
|
54
|
+
export { GENERATION_WIDGET_URI } from './widget-uri.js';
|
|
55
|
+
import { GENERATION_WIDGET_URI } from './widget-uri.js';
|
|
39
56
|
import { resolveModelEnums, BOARD_TYPES, BOARD_TYPE_GUIDANCE, IMAGE_MODEL_GUIDANCE, VIDEO_MODEL_GUIDANCE, AUDIO_MODEL_GUIDANCE, EDIT_AUDIO_MODEL_GUIDANCE, UPSCALE_MODEL_GUIDANCE, LIP_SYNC_MODEL_GUIDANCE, } from './models.js';
|
|
40
|
-
import { audioResult, avatarListResult, avatarResult, avatarPendingResult, balanceResult, brandKitListResult, brandKitResult, brandKnowledgeListResult, brandKnowledgeDetailResult, brandKnowledgeSearchResult, brandKnowledgeItemResult, completedResult, connectedAccountListResult, connectedAccountResult, costResult, accountDetailResult, inspirationContentResult, mediaListResult, mediaSearchResult, folderListResult, folderContentsResult, mediaBatchResult, mediaUploadResult, importedMediaResult, uploadedMediaResult, tagListResult, tagResult, tagDeletedResult, modelListResult, modelResult, platformListResult, platformResult, elementListResult, elementResult, elementDeletedResult, errorResult, generationBatchResult, outlierListResult, enhanceClipsResult, pendingResult, stageListResult, stageResult, stageDeletedResult, spaceDeletedResult, spaceListResult, spaceResult, cardListResult, cardResult, postSummaryResult, publishResult, statusActionResult, editorOpsResult, text, projectDetailResult, liveContextResult, projectListResult, projectCreatedResult, projectDeletedResult, layerTypesResult, timelineTypesResult, editorTranscriptResult, exportJobResult, exportFormatsResult, trackedAccountListResult, transcriptResult, voiceListResult, voiceResult, } from './format.js';
|
|
57
|
+
import { audioResult, avatarListResult, avatarResult, avatarPendingResult, balanceResult, brandKitListResult, brandKitResult, brandKnowledgeListResult, brandKnowledgeDetailResult, brandKnowledgeSearchResult, brandKnowledgeItemResult, completedResult, connectedAccountListResult, connectedAccountResult, costResult, accountDetailResult, inspirationContentResult, mediaListResult, mediaSearchResult, folderListResult, folderContentsResult, mediaBatchResult, mediaUploadResult, importedMediaResult, uploadedMediaResult, tagListResult, tagResult, tagDeletedResult, modelListResult, modelResult, platformListResult, platformResult, elementListResult, elementResult, elementDeletedResult, errorResult, generationBatchResult, outlierListResult, enhanceClipsResult, pendingResult, pollAfterSecondsFor, stageListResult, stageResult, stageDeletedResult, spaceDeletedResult, spaceListResult, spaceResult, cardListResult, cardResult, postSummaryResult, publishResult, statusActionResult, editorOpsResult, text, projectDetailResult, liveContextResult, projectListResult, projectCreatedResult, projectDeletedResult, layerTypesResult, timelineTypesResult, editorTranscriptResult, exportJobResult, exportFormatsResult, trackedAccountListResult, transcriptResult, voiceListResult, voiceResult, } from './format.js';
|
|
41
58
|
/** Platforms a card or one of its posts may target. */
|
|
42
59
|
const POST_PLATFORMS = [
|
|
43
60
|
'youtube',
|
|
@@ -57,6 +74,33 @@ const POST_PLATFORMS = [
|
|
|
57
74
|
* than tripping the client's timeout.
|
|
58
75
|
*/
|
|
59
76
|
const SMART_WAIT_MS = 50_000;
|
|
77
|
+
/**
|
|
78
|
+
* What a still-running generation can already say about the shape of its own result, read from the tool's
|
|
79
|
+
* own ARGUMENTS.
|
|
80
|
+
*
|
|
81
|
+
* ⚠️ **FROM `args`, NOT FROM THE BUILT REQUEST.** Every one of these sites builds its request inside a
|
|
82
|
+
* `try`, so the request is out of scope in the `catch` where a pending outputId surfaces. `args` is the
|
|
83
|
+
* handler's parameter and is always in scope, and it is also the more honest source: it is what the caller
|
|
84
|
+
* asked for, which is exactly what the placeholders should depict.
|
|
85
|
+
*
|
|
86
|
+
* ⚠️ Read defensively because the count is spelled `numImages` on some tools and `numGenerations` on
|
|
87
|
+
* others. A widened type here would be a third spelling; reading both is the whole reconciliation.
|
|
88
|
+
*
|
|
89
|
+
* ⛔ `auto` and `adaptive` are legal aspect inputs meaning "the model decides", so they are NOT ratios.
|
|
90
|
+
* Passing one through would have the widget lay placeholders out against a string it cannot parse. Null
|
|
91
|
+
* lets it fall back to its unshaped box, which is the honest state while nothing is known.
|
|
92
|
+
*/
|
|
93
|
+
function pendingShapeFrom(args, contentType) {
|
|
94
|
+
const a = (args ?? {});
|
|
95
|
+
const ar = a.aspectRatio;
|
|
96
|
+
const displayAspect = !ar || ar === 'auto' || ar === 'adaptive' || !ar.includes(':') ? null : ar;
|
|
97
|
+
return {
|
|
98
|
+
contentType,
|
|
99
|
+
modelId: a.modelId ?? '',
|
|
100
|
+
displayAspect,
|
|
101
|
+
expected: a.numImages ?? a.numGenerations ?? 1,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
60
104
|
/**
|
|
61
105
|
* Tool annotations drive how MCP clients group the surface. readOnlyHint=true
|
|
62
106
|
* tools list under "Read-only"; the rest list under "Interactive". publish is
|
|
@@ -119,6 +163,159 @@ async function fetchSnapshotBase64(url) {
|
|
|
119
163
|
return null;
|
|
120
164
|
}
|
|
121
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* The asset itself, ready to attach to a finished generation.
|
|
168
|
+
*
|
|
169
|
+
* ## The rule, per medium
|
|
170
|
+
*
|
|
171
|
+
* ⭐⭐⭐ **BYTES FOR WHAT MCP CAN CARRY, A LINK FOR WHAT IT CANNOT.** Images and audio have first-class
|
|
172
|
+
* content blocks and modest sizes, so they are embedded: that is what makes them render in the chat, and it
|
|
173
|
+
* is also what makes them permanent, because bytes in a transcript cannot expire. Video has no block of its
|
|
174
|
+
* own and a ten-second 1080p clip would be megabytes of base64 in every future turn of the conversation, so
|
|
175
|
+
* it travels as a `resource_link` pointing at a capability url.
|
|
176
|
+
*
|
|
177
|
+
* ⛔ **FAILING TO FETCH IS NOT AN ERROR.** A generation that succeeded must never be reported as failed
|
|
178
|
+
* because we could not inline a preview of it. Every failure path returns no attachment and the text result
|
|
179
|
+
* stands on its own, which is exactly what the caller got before this existed.
|
|
180
|
+
*
|
|
181
|
+
* ⚠️ ONLY THE FIRST OUTPUT IS EMBEDDED WHEN THERE ARE MANY. A four-image batch as four base64 payloads is a
|
|
182
|
+
* large multiple of the same conversation cost, and the urls for the rest are already in the text. The cap
|
|
183
|
+
* is stated here rather than left implicit, because a silent truncation reads as "that is all there was".
|
|
184
|
+
*/
|
|
185
|
+
/**
|
|
186
|
+
* ⭐⭐⭐ **EVERY OUTPUT, AS A LINK, NOT THE FIRST ONE AS BYTES.**
|
|
187
|
+
*
|
|
188
|
+
* The first version embedded base64 for images and audio and capped at one attachment. Both halves were
|
|
189
|
+
* wrong, and they were wrong together:
|
|
190
|
+
*
|
|
191
|
+
* - **The cap made a four-variation batch show one variation.** Generating four and seeing one is not a
|
|
192
|
+
* smaller version of the feature, it is a broken one: the whole point of a batch is to compare them.
|
|
193
|
+
* - **Base64 is charged to the user's context on every subsequent turn.** A four-image batch embedded as
|
|
194
|
+
* bytes is a large multiple of the same cost, repeated for the rest of the conversation.
|
|
195
|
+
*
|
|
196
|
+
* ⭐ A `resource_link` costs a URL and renders in the host's UI, so ALL of them can come back. Verified
|
|
197
|
+
* against a working implementation: the Higgsfield MCP returns `resource_link` for its media and it renders.
|
|
198
|
+
*
|
|
199
|
+
* ## ⛔ THE THING THIS DELIBERATELY GIVES UP, AND WHERE IT WENT INSTEAD
|
|
200
|
+
*
|
|
201
|
+
* An `image` block feeds the MODEL's vision; a `resource_link` gives the HOST something to render for the
|
|
202
|
+
* human. Measured in this very session: when a link came back from another MCP, the model received text and
|
|
203
|
+
* could not see the picture.
|
|
204
|
+
*
|
|
205
|
+
* So the model can no longer critique a generation it just made from this result alone. That is the correct
|
|
206
|
+
* trade, because **`get_media` already exists to embed bytes for exactly that purpose** and an agent calls
|
|
207
|
+
* it when it actually needs to look. Deciding on every generation that the model probably wants to look was
|
|
208
|
+
* the wrong default: it spent the user's context to answer a question nobody asked.
|
|
209
|
+
*
|
|
210
|
+
* ⚠️ NO SSRF FETCH HAPPENS HERE ANY MORE. Nothing is downloaded, so the allowlist that guards
|
|
211
|
+
* `fetchSnapshotBase64` is not on this path; the url is handed to the host to fetch under its own rules.
|
|
212
|
+
*/
|
|
213
|
+
const LINK_MIME = {
|
|
214
|
+
image: 'image/png',
|
|
215
|
+
video: 'video/mp4',
|
|
216
|
+
audio: 'audio/mpeg',
|
|
217
|
+
};
|
|
218
|
+
/**
|
|
219
|
+
* ⛔⛔⛔ **A `resource_link` DOES NOT RENDER. MEASURED IN BOTH HOSTS, IN PRODUCTION, 2026-09-19.**
|
|
220
|
+
*
|
|
221
|
+
* This returned `kind: 'link'` for every output, including images, and the result was the feature not
|
|
222
|
+
* working at all:
|
|
223
|
+
*
|
|
224
|
+
* - **ChatGPT** showed the output id and a "View the generated image" hyperlink. Clicking it opened the
|
|
225
|
+
* asset in a NEW TAB, which is the opposite of inline.
|
|
226
|
+
* - **Claude** showed NOTHING. No image, no link, no output id.
|
|
227
|
+
*
|
|
228
|
+
* ⚠️ **THE FORMATTER COULD ALWAYS DO THIS AND NOTHING EVER ASKED IT TO.** `completedResult` has handled a
|
|
229
|
+
* `kind: 'bytes'` attachment since it was written, and its own docblock claims images get first-class
|
|
230
|
+
* blocks. This function never produced one, so the branch had no caller. Same shape as the capability-url
|
|
231
|
+
* no-op: a path that exists, typechecks, passes tests, and is unreachable.
|
|
232
|
+
*
|
|
233
|
+
* ⭐ **AN `image` BLOCK IS THE ONLY THING A HOST ACTUALLY RENDERS**, and it feeds the model's vision as
|
|
234
|
+
* well, so the earlier reasoning that `get_media` covers the looking case was answering a different
|
|
235
|
+
* question than the one the user asked: they wanted to SEE it.
|
|
236
|
+
*
|
|
237
|
+
* ## Both, not either
|
|
238
|
+
*
|
|
239
|
+
* Images get a block AND a link. The block is the small `.preview.webp` sibling, so inline display costs a
|
|
240
|
+
* few hundred tokens rather than the megabytes a 2736x1536 original would. The link is the capability url:
|
|
241
|
+
* permanent, full resolution, and the thing to click when the preview is not enough.
|
|
242
|
+
*
|
|
243
|
+
* ⚠️ VIDEO STAYS LINK-ONLY. MCP has no video content block, and base64 video in a transcript is not a
|
|
244
|
+
* trade worth making. Audio likewise has no small derivative to send, so it stays a link too.
|
|
245
|
+
*/
|
|
246
|
+
/**
|
|
247
|
+
* ⛔⛔ **THERE IS NO HOST DETECTION HERE, AND THAT IS NOT AN OVERSIGHT.**
|
|
248
|
+
*
|
|
249
|
+
* The obvious optimization is to skip the bytes when the host will mount the widget, since the widget loads
|
|
250
|
+
* media from a URL and the blocks are only a fallback. I wrote it, and it could never work: MCP Apps
|
|
251
|
+
* declares its support under `capabilities.extensions["io.modelcontextprotocol/ui"]`, and
|
|
252
|
+
* **`@modelcontextprotocol/sdk@1.26` does not know the word `extensions`** (measured: zero occurrences in
|
|
253
|
+
* its types). The schema strips it, so `getClientCapabilities()` returns the same answer for a host that
|
|
254
|
+
* mounts widgets and one that cannot, and the check silently reduced to a constant.
|
|
255
|
+
*
|
|
256
|
+
* ⭐ A CHECK THAT ALWAYS ANSWERS THE SAME WAY IS WORSE THAN NO CHECK: it reads as a decision being made.
|
|
257
|
+
* Deleted, and the budget below is what keeps every result under the host's ceiling on its own.
|
|
258
|
+
*
|
|
259
|
+
* ⏭️ The split packages (`@modelcontextprotocol/server@2`) carry the field. Migrating to them is what
|
|
260
|
+
* unlocks this, and it is its own piece of work rather than a prerequisite for rendering.
|
|
261
|
+
*/
|
|
262
|
+
export async function attachmentsFor(gen) {
|
|
263
|
+
/**
|
|
264
|
+
* ⛔⛔⛔ **ONE BUDGET FOR THE WHOLE RESULT, BECAUSE THE HOST'S CEILING IS PER RESULT.**
|
|
265
|
+
*
|
|
266
|
+
* This was a PER-ITEM cap, which is the same defect one level up from the one it replaced. Four images at
|
|
267
|
+
* 600 KB each pass individually (822 KB encoded, under the budget) and total 3.3 MB, so the host rejects
|
|
268
|
+
* the call and the person pays for four generations they cannot reach. Today's assets are ~3.6 MB apiece
|
|
269
|
+
* and fail the per-item check anyway, so the batch case was safe BY ACCIDENT rather than by design.
|
|
270
|
+
*
|
|
271
|
+
* ⭐ Spending a single budget makes the envelope bounded no matter the count or the resolution: four 4K
|
|
272
|
+
* images, ten variations, a 60 second video. Whatever does not fit degrades to a link, and the widget
|
|
273
|
+
* renders it from a URL regardless, so nothing is lost but the fallback for hosts that cannot mount apps.
|
|
274
|
+
*/
|
|
275
|
+
let budget = MAX_INLINE_BASE64_CHARS;
|
|
276
|
+
const urls = (gen.outputUrls ?? []).filter((u) => typeof u === 'string' && u.length > 0);
|
|
277
|
+
const mimeType = LINK_MIME[gen.contentType];
|
|
278
|
+
if (!mimeType)
|
|
279
|
+
return [];
|
|
280
|
+
const ext = mimeType.split('/')[1];
|
|
281
|
+
const out = [];
|
|
282
|
+
for (const [i, uri] of urls.entries()) {
|
|
283
|
+
/**
|
|
284
|
+
* ⭐ AUDIO HAS A FIRST-CLASS BLOCK TOO, and it plays inline exactly as an image draws. It is fetched the
|
|
285
|
+
* same best-effort way: a miss degrades this one output to a link rather than failing a generation the
|
|
286
|
+
* person already paid for.
|
|
287
|
+
*
|
|
288
|
+
* ⚠️ NO PREVIEW DERIVATIVE EXISTS FOR AUDIO, so this is the real file and the size cap is what stops a
|
|
289
|
+
* long track going into the transcript. A voiceover is small; an hour of music is not, and that one
|
|
290
|
+
* degrades to a link, which the widget renders anyway.
|
|
291
|
+
*/
|
|
292
|
+
if (gen.contentType === 'audio') {
|
|
293
|
+
const bytes = await fetchAudioBytes(uri, budget);
|
|
294
|
+
if (bytes) {
|
|
295
|
+
budget -= bytes.data.length;
|
|
296
|
+
out.push({ kind: 'bytes', type: 'audio', data: bytes.data, mimeType: bytes.mimeType });
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
if (gen.contentType === 'image') {
|
|
300
|
+
// Best-effort: a miss (host not allowlisted, over the budget, network hiccup) degrades this one output
|
|
301
|
+
// to a link instead of failing a generation the user already paid for.
|
|
302
|
+
const bytes = await fetchMediaImageBase64(uri, budget);
|
|
303
|
+
if (bytes) {
|
|
304
|
+
budget -= bytes.data.length;
|
|
305
|
+
out.push({ kind: 'bytes', type: 'image', data: bytes.data, mimeType: bytes.mimeType });
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* ⛔ NO `resource_link` PER OUTPUT ANY MORE. It was a THIRD representation of a url the text list and the
|
|
310
|
+
* widget's `structuredContent` both already carry, and hosts render a run of them as `name: uri` with no
|
|
311
|
+
* separator, producing tokens that read as corrupted. The link added nothing the text did not, and cost
|
|
312
|
+
* a per-output block to say it.
|
|
313
|
+
*/
|
|
314
|
+
void ext;
|
|
315
|
+
void mimeType;
|
|
316
|
+
}
|
|
317
|
+
return out;
|
|
318
|
+
}
|
|
122
319
|
/**
|
|
123
320
|
* True when an image URL is safe to fetch into an image block. SSRF allowlist:
|
|
124
321
|
* our storage hosts plus the finite set of generation-provider CDNs that our
|
|
@@ -134,6 +331,11 @@ function isAllowedImageHost(url) {
|
|
|
134
331
|
if (u.username || u.password)
|
|
135
332
|
return false;
|
|
136
333
|
return (u.host === 'cloud.contenthero.ai' ||
|
|
334
|
+
// ⭐ THE MEDIA GATEWAY. Generated assets now address through it with a capability token rather than a
|
|
335
|
+
// presigned R2 url, so without this every inline attachment would be silently dropped by the SSRF
|
|
336
|
+
// allowlist and the agent would be back to a bare link.
|
|
337
|
+
u.host === 'media.contenthero.ai' ||
|
|
338
|
+
u.host === 'cdn.contenthero.ai' ||
|
|
137
339
|
u.host.endsWith('.supabase.co') ||
|
|
138
340
|
u.host.endsWith('.fal.media') ||
|
|
139
341
|
u.host.endsWith('.cloudinary.com'));
|
|
@@ -161,17 +363,87 @@ function optimizedImageSibling(url) {
|
|
|
161
363
|
return url;
|
|
162
364
|
return query ? `${rewritten}?${query}` : rewritten;
|
|
163
365
|
}
|
|
164
|
-
|
|
366
|
+
/**
|
|
367
|
+
* ⛔⛔⛔ **A HOST ENFORCES A 1 MB CEILING ON A WHOLE TOOL RESULT, AND THIS IS SIZED AGAINST THAT.**
|
|
368
|
+
*
|
|
369
|
+
* Claude Desktop rejects an oversized result outright with "Tool result is too large. Maximum size is 1MB",
|
|
370
|
+
* which fails the CALL rather than degrading the picture. Measured 2026-09-19 on a real `generate_image`:
|
|
371
|
+
* the generation succeeded and was charged, and the person could not retrieve it.
|
|
372
|
+
*
|
|
373
|
+
* ⚠️ I SET THIS CAP WRONG TWICE BEFORE GETTING HERE. 1.5 MB was too low and would have silently degraded
|
|
374
|
+
* real outputs back to links; 8 MB was too high and broke the host. Both were reasoned from what the BYTES
|
|
375
|
+
* cost us. The number that actually governs belongs to the host, and it bounds the ENTIRE result: text,
|
|
376
|
+
* structured content, every block. So the budget is expressed in BASE64 LENGTH, which is what travels, and
|
|
377
|
+
* leaves room for everything else in the envelope.
|
|
378
|
+
*
|
|
379
|
+
* ⭐ Over budget, the item degrades to a link and the widget still renders it, because the widget loads from
|
|
380
|
+
* a url rather than from bytes.
|
|
381
|
+
*
|
|
382
|
+
* ⚠️ **DERIVED, NOT PICKED.** The ceiling is 1,000,000 bytes for the ENTIRE serialized result. Measured on a
|
|
383
|
+
* real one: text, `structuredContent` and the links together weigh about 2 KB. 900,000 base64 characters
|
|
384
|
+
* leaves roughly 100 KB of headroom, and admits a 657 KB source asset. A 613 KB PNG measured in production
|
|
385
|
+
* encodes to about 840 KB, so it fits with room to spare, where the 700,000 I first wrote would have thrown
|
|
386
|
+
* it away. That was the third time I set this number from reasoning instead of from a measurement.
|
|
387
|
+
*/
|
|
388
|
+
const MAX_INLINE_BASE64_CHARS = 900_000;
|
|
389
|
+
/**
|
|
390
|
+
* The audio equivalent of `fetchImageBytes`, sharing its host allowlist, its size cap and its timeout.
|
|
391
|
+
*
|
|
392
|
+
* ⚠️ SEPARATE RATHER THAN A `kind` PARAMETER because the content-type CHECK is the difference, and a single
|
|
393
|
+
* function taking "which prefix do I accept" is the shape that eventually accepts the wrong one.
|
|
394
|
+
*/
|
|
395
|
+
async function fetchAudioBytes(url, budget) {
|
|
165
396
|
if (!isAllowedImageHost(url))
|
|
166
397
|
return null;
|
|
167
398
|
try {
|
|
168
|
-
const res = await fetch(url);
|
|
399
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
|
|
400
|
+
if (!res.ok)
|
|
401
|
+
return null;
|
|
402
|
+
const mimeType = res.headers.get('content-type') || 'audio/mpeg';
|
|
403
|
+
if (!mimeType.startsWith('audio/'))
|
|
404
|
+
return null;
|
|
405
|
+
// Base64 inflates by about a third, so the declared byte length is checked against the budget it will
|
|
406
|
+
// BECOME rather than against itself.
|
|
407
|
+
const declared = Number(res.headers.get('content-length') ?? '');
|
|
408
|
+
if (Number.isFinite(declared) && declared * 1.37 > budget)
|
|
409
|
+
return null;
|
|
410
|
+
const data = Buffer.from(await res.arrayBuffer()).toString('base64');
|
|
411
|
+
if (data.length > budget)
|
|
412
|
+
return null;
|
|
413
|
+
return { data, mimeType };
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
return null;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
async function fetchImageBytes(url, budget) {
|
|
420
|
+
if (!isAllowedImageHost(url))
|
|
421
|
+
return null;
|
|
422
|
+
try {
|
|
423
|
+
/**
|
|
424
|
+
* ⚠️⚠️ **A BARE `fetch` HAS NO TIMEOUT, AND THIS ONE IS ON THE PATH OF EVERY GENERATION RESULT.**
|
|
425
|
+
*
|
|
426
|
+
* One unresponsive asset would hang the whole tool call rather than degrading that item to a link, and
|
|
427
|
+
* the caller would see a dead generation they had already paid for. Found by `verify:inline` hanging on
|
|
428
|
+
* its first run after the status path started attaching.
|
|
429
|
+
*
|
|
430
|
+
* ⭐ Failing is CHEAP here and the fallback is good: no block, keep the link. Waiting is what is
|
|
431
|
+
* expensive, so the budget is deliberately short.
|
|
432
|
+
*/
|
|
433
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
|
|
169
434
|
if (!res.ok)
|
|
170
435
|
return null;
|
|
171
436
|
const mimeType = res.headers.get('content-type') || 'image/jpeg';
|
|
172
437
|
if (!mimeType.startsWith('image/'))
|
|
173
438
|
return null;
|
|
439
|
+
// Checked BEFORE reading the body where the server declares it, and again after, because
|
|
440
|
+
// `content-length` is absent on a chunked response and a header is not a measurement.
|
|
441
|
+
const declared = Number(res.headers.get('content-length') ?? '');
|
|
442
|
+
if (Number.isFinite(declared) && declared * 1.37 > budget)
|
|
443
|
+
return null;
|
|
174
444
|
const data = Buffer.from(await res.arrayBuffer()).toString('base64');
|
|
445
|
+
if (data.length > budget)
|
|
446
|
+
return null;
|
|
175
447
|
return { data, mimeType };
|
|
176
448
|
}
|
|
177
449
|
catch {
|
|
@@ -184,15 +456,111 @@ async function fetchImageBytes(url) {
|
|
|
184
456
|
* auto-upgrades as the optimization pipeline backfills derivatives, with no code
|
|
185
457
|
* change here. Best-effort: any failure returns null and the item stays text-only.
|
|
186
458
|
*/
|
|
187
|
-
async function fetchMediaImageBase64(url) {
|
|
459
|
+
async function fetchMediaImageBase64(url, budget) {
|
|
188
460
|
const optimized = optimizedImageSibling(url);
|
|
189
461
|
if (optimized !== url) {
|
|
190
|
-
const hit = await fetchImageBytes(optimized);
|
|
462
|
+
const hit = await fetchImageBytes(optimized, budget);
|
|
191
463
|
if (hit)
|
|
192
464
|
return hit;
|
|
193
465
|
}
|
|
194
|
-
return fetchImageBytes(url);
|
|
466
|
+
return fetchImageBytes(url, budget);
|
|
195
467
|
}
|
|
468
|
+
/**
|
|
469
|
+
* ⭐⭐⭐ **THE ONE PLACE THAT DECIDES HOW MUCH OF A RESULT MAY BE BYTES.**
|
|
470
|
+
*
|
|
471
|
+
* Two call sites needed this and each had its own answer, which is how the same defect appeared twice at
|
|
472
|
+
* different levels: `attachmentsFor` capped PER ITEM while the host's ceiling is per RESULT, and `get_media`
|
|
473
|
+
* fetched up to TEN images in parallel with no cap at all (10 x 840 KB is 8 MB against a 1 MB limit).
|
|
474
|
+
*
|
|
475
|
+
* ⛔ SEQUENTIAL, DELIBERATELY. The budget is shared state, so a parallel fetch cannot know what the others
|
|
476
|
+
* already spent and every one of them would pass a check the set as a whole fails. Most calls carry one to
|
|
477
|
+
* three items, so the latency is small and the alternative is a ceiling that holds only by luck.
|
|
478
|
+
*
|
|
479
|
+
* Returns one entry per input, `null` where the asset did not fit or could not be read, so callers keep
|
|
480
|
+
* positional alignment with what they asked for.
|
|
481
|
+
*/
|
|
482
|
+
async function inlineImagesWithinBudget(urls) {
|
|
483
|
+
let budget = MAX_INLINE_BASE64_CHARS;
|
|
484
|
+
const out = [];
|
|
485
|
+
for (const url of urls) {
|
|
486
|
+
if (!url) {
|
|
487
|
+
out.push(null);
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
const hit = await fetchMediaImageBase64(url, budget);
|
|
491
|
+
if (hit)
|
|
492
|
+
budget -= hit.data.length;
|
|
493
|
+
out.push(hit);
|
|
494
|
+
}
|
|
495
|
+
return out;
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* ⛔⛔⛔ **THE WIDGET IS DECLARED BY THE TOOL, NOT BY THE RESULT. I HAD IT ON THE RESULT.**
|
|
499
|
+
*
|
|
500
|
+
* A host reads `tool._meta` at `tools/list` time to learn that a tool renders a widget. Putting the binding
|
|
501
|
+
* only on the CallToolResult means the host never knows to mount anything, so the result arrives as plain
|
|
502
|
+
* blocks and the widget silently never appears. Measured in Claude Desktop 2026-09-19: four URLs as text,
|
|
503
|
+
* no viewer, no error anywhere.
|
|
504
|
+
*
|
|
505
|
+
* ⚠️ BOTH SPELLINGS, DELIBERATELY. `_meta.ui.resourceUri` is the current format and `ui/resourceUri` the
|
|
506
|
+
* legacy one, and the spec's own guidance is that hosts must accept either. Emitting both costs nothing and
|
|
507
|
+
* removes a whole class of "works in one client" from the table.
|
|
508
|
+
*
|
|
509
|
+
* ⭐ Spread into the tools whose results are MEDIA. Not onto all 87: a tool that returns a card or a folder
|
|
510
|
+
* has nothing for this widget to show, and claiming otherwise would put an empty frame under every call.
|
|
511
|
+
*/
|
|
512
|
+
const RENDERS_GENERATION = {
|
|
513
|
+
_meta: {
|
|
514
|
+
ui: { resourceUri: GENERATION_WIDGET_URI },
|
|
515
|
+
[RESOURCE_URI_META_KEY]: GENERATION_WIDGET_URI,
|
|
516
|
+
},
|
|
517
|
+
};
|
|
518
|
+
/**
|
|
519
|
+
* ⛔⛔⛔ **WITHOUT THIS THE FRAME LOADS NOTHING, AND THE SPEC SAYS SO PLAINLY:**
|
|
520
|
+
* "Empty or omitted → no network resources (secure default)."
|
|
521
|
+
*
|
|
522
|
+
* Measured in Claude Desktop 2026-09-19: the widget mounted, the chrome rendered, the variation strip and
|
|
523
|
+
* the buttons worked, and every image was a broken icon showing its own filename. The frame was doing
|
|
524
|
+
* exactly what it was told, which was to permit nothing.
|
|
525
|
+
*
|
|
526
|
+
* `resourceDomains` maps to `img-src`, `media-src`, `script-src`, `style-src` and `font-src`, so it is the
|
|
527
|
+
* one field that decides whether an `<img>` or a `<video>` in this widget can reach our storage.
|
|
528
|
+
*
|
|
529
|
+
* ⚠️ NO `connectDomains`. The widget never calls `fetch`: it points element sources at urls and lets the
|
|
530
|
+
* browser load them. Granting network access it does not use would widen the sandbox for nothing.
|
|
531
|
+
*
|
|
532
|
+
* ⚠️ These are the hosts that actually serve generated media, which is a SMALLER set than the server's SSRF
|
|
533
|
+
* allowlist. That list governs what the SERVER may fetch and inline; this governs what the FRAME may load.
|
|
534
|
+
* Two different questions, deliberately not one constant.
|
|
535
|
+
*/
|
|
536
|
+
const WIDGET_CSP = {
|
|
537
|
+
_meta: {
|
|
538
|
+
ui: {
|
|
539
|
+
csp: {
|
|
540
|
+
resourceDomains: [
|
|
541
|
+
// Capability urls for generated assets: the token rides in the query string, so an element src
|
|
542
|
+
// loads one directly with no header to set.
|
|
543
|
+
'https://media.contenthero.ai',
|
|
544
|
+
// Public-class objects (posters, gallery, stock).
|
|
545
|
+
'https://cdn.contenthero.ai',
|
|
546
|
+
],
|
|
547
|
+
/**
|
|
548
|
+
* ⛔⛔ **A SEPARATE FIELD, AND OMITTING IT BLOCKS `fetch` ENTIRELY.**
|
|
549
|
+
*
|
|
550
|
+
* `resourceDomains` maps to `img-src`, `media-src` and friends, which is why the pictures render.
|
|
551
|
+
* `connectDomains` maps to `connect-src`, and the spec's default for an omitted list is "no network
|
|
552
|
+
* connections (secure default)". So the frame could DISPLAY our media and could not READ it, which
|
|
553
|
+
* is exactly the shape needed to save a file: downloading means holding the bytes.
|
|
554
|
+
*
|
|
555
|
+
* ⚠️ Same origins, deliberately repeated rather than shared with a constant. They answer different
|
|
556
|
+
* questions (may the frame paint this, may the frame read this) and a future answer to one is not
|
|
557
|
+
* automatically the answer to the other.
|
|
558
|
+
*/
|
|
559
|
+
connectDomains: ['https://media.contenthero.ai', 'https://cdn.contenthero.ai'],
|
|
560
|
+
},
|
|
561
|
+
},
|
|
562
|
+
},
|
|
563
|
+
};
|
|
196
564
|
/** Drop undefined values so the request payload stays minimal. */
|
|
197
565
|
function compact(obj) {
|
|
198
566
|
return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
|
|
@@ -228,6 +596,57 @@ export function registerTools(server, opts) {
|
|
|
228
596
|
* shape per op type. Strictness here applies to the TOP-LEVEL argument object only, so those keep taking
|
|
229
597
|
* varied op payloads while still rejecting an undeclared top-level parameter.
|
|
230
598
|
*/
|
|
599
|
+
/**
|
|
600
|
+
* ⭐⭐⭐ **THE GENERATION WIDGET: THE ONLY THING THAT PUTS A PLAYING VIDEO IN A CONVERSATION.**
|
|
601
|
+
*
|
|
602
|
+
* MCP's content blocks are `text | image | audio | resource | resource_link`. **There is no video block**,
|
|
603
|
+
* so no arrangement of them can render video, and a `resource_link` renders as a hyperlink in ChatGPT and
|
|
604
|
+
* as NOTHING in Claude. Measured in production 2026-09-19.
|
|
605
|
+
*
|
|
606
|
+
* MCP Apps is the mechanism that works. The server publishes an HTML resource under `ui://`, the host
|
|
607
|
+
* mounts it, and the widget reads the tool's `structuredContent`. It is an open standard with an official
|
|
608
|
+
* SDK (`@modelcontextprotocol/ext-apps`), verified against a working implementation before adoption.
|
|
609
|
+
*
|
|
610
|
+
* ⚠️ **THE HTML IS READ FROM THE PACKAGE, NOT FETCHED.** It ships inside the published tarball, so a local
|
|
611
|
+
* install renders the same thing the hosted server does. Fetching it from our app would make the widget
|
|
612
|
+
* depend on a deploy and break every offline or self-hosted install.
|
|
613
|
+
*
|
|
614
|
+
* ⛔ **REGISTERING THIS COSTS NOTHING FOR HOSTS THAT DO NOT SUPPORT IT.** A client that ignores `ui://`
|
|
615
|
+
* resources simply never reads it, and the image and audio BLOCKS remain the fallback. That is why the
|
|
616
|
+
* blocks stay rather than being replaced: two mechanisms, and the widget is the better one where it exists.
|
|
617
|
+
*/
|
|
618
|
+
/**
|
|
619
|
+
* ⚠️⚠️ **REGISTERED UNCONDITIONALLY, AND READ LAZILY.**
|
|
620
|
+
*
|
|
621
|
+
* This used to read the bundle at startup and register the resource only if it was found. Two problems.
|
|
622
|
+
* A missing bundle produced a server that silently had no widget, which is the failure mode hardest to
|
|
623
|
+
* notice: every tool still worked and nothing rendered. And the resource then did not exist when running
|
|
624
|
+
* from `src/`, so the guard that checks tools point at a real resource could not run at all.
|
|
625
|
+
*
|
|
626
|
+
* ⭐ The resource is part of this server's contract. Advertising it always and throwing a NAMED error at
|
|
627
|
+
* read time turns "no widget, no reason" into one line that says exactly what is missing.
|
|
628
|
+
*/
|
|
629
|
+
server.registerResource('generation', GENERATION_WIDGET_URI, {
|
|
630
|
+
description: 'Shows what a generation produced: every variation, playable and downloadable.',
|
|
631
|
+
mimeType: RESOURCE_MIME_TYPE,
|
|
632
|
+
...WIDGET_CSP,
|
|
633
|
+
}, async () => {
|
|
634
|
+
const path = join(MODULE_DIR, 'widget', 'generation.html');
|
|
635
|
+
let text;
|
|
636
|
+
try {
|
|
637
|
+
text = readFileSync(path, 'utf8');
|
|
638
|
+
}
|
|
639
|
+
catch {
|
|
640
|
+
throw new Error(`The generation widget is missing at ${path}. It is built by \`npm run build\` ` +
|
|
641
|
+
'(`node widget/build.mjs`) and ships inside the package; a server running from source has not built it.');
|
|
642
|
+
}
|
|
643
|
+
// ⚠️ REPEATED ON THE READ RESULT, not just the listing. The spec reads csp from the `resources/read`
|
|
644
|
+
// content item and treats the `resources/list` entry as a FALLBACK, so a host that only consults the
|
|
645
|
+
// read path would otherwise see no policy and apply the secure default of blocking everything.
|
|
646
|
+
return {
|
|
647
|
+
contents: [{ uri: GENERATION_WIDGET_URI, mimeType: RESOURCE_MIME_TYPE, text, ...WIDGET_CSP }],
|
|
648
|
+
};
|
|
649
|
+
});
|
|
231
650
|
const rawRegisterTool = server.registerTool.bind(server);
|
|
232
651
|
server.registerTool = ((name, config, cb) => {
|
|
233
652
|
const shape = config.inputSchema;
|
|
@@ -295,9 +714,10 @@ export function registerTools(server, opts) {
|
|
|
295
714
|
});
|
|
296
715
|
// -- generate_image -------------------------------------------------------
|
|
297
716
|
server.registerTool('generate_image', {
|
|
717
|
+
...RENDERS_GENERATION,
|
|
298
718
|
title: 'Generate Image',
|
|
299
719
|
annotations: WRITE,
|
|
300
|
-
description: 'Generate one or more images from a text prompt (optionally image-to-image with reference images). Waits for the result and returns the image URLs. Optionally pass projectId to place the generated image onto that project in the same call, controlled by an optional placement: a VIDEO timeline places a clip on a track, a CANVAS design places a layer on a slide (defaulting to the slide the user is focused on). Omit projectId to save a standalone library output. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
720
|
+
description: 'Generate one or more images from a text prompt (optionally image-to-image with reference images). Waits for the result and returns the image URLs. Optionally pass projectId to place the generated image onto that project in the same call, controlled by an optional placement: a VIDEO timeline places a clip on a track, a CANVAS design places a layer on a slide (defaulting to the slide the user is focused on). Omit projectId to save a standalone library output. The result LINKS each output so the user sees it inline; to SEE it yourself (judge a face, check legibility, pick between variations) call get_media with the outputId. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
301
721
|
inputSchema: {
|
|
302
722
|
modelId: z.enum(models.image).describe(IMAGE_MODEL_GUIDANCE),
|
|
303
723
|
prompt: z
|
|
@@ -344,7 +764,7 @@ export function registerTools(server, opts) {
|
|
|
344
764
|
if (args.getCost)
|
|
345
765
|
return costResult(await client.estimateCost(request));
|
|
346
766
|
const gen = await client.generateAndWait(request, { timeoutMs: SMART_WAIT_MS });
|
|
347
|
-
return completedResult(gen);
|
|
767
|
+
return completedResult(gen, await attachmentsFor(gen), [], client.baseUrl);
|
|
348
768
|
}
|
|
349
769
|
catch (err) {
|
|
350
770
|
// A SUBMITTED generation is running and charged. Whether the wait timed out or a
|
|
@@ -352,12 +772,13 @@ export function registerTools(server, opts) {
|
|
|
352
772
|
// dropping it invites a retry that generates and charges a second time.
|
|
353
773
|
const pending = pendingOutputId(err);
|
|
354
774
|
if (pending)
|
|
355
|
-
return pendingResult(pending);
|
|
775
|
+
return pendingResult(pending, pollAfterSecondsFor('image'), pendingShapeFrom(args, 'image'));
|
|
356
776
|
return errorResult(err);
|
|
357
777
|
}
|
|
358
778
|
});
|
|
359
779
|
// -- generate_board -------------------------------------------------------
|
|
360
780
|
server.registerTool('generate_board', {
|
|
781
|
+
...RENDERS_GENERATION,
|
|
361
782
|
title: 'Generate Reference Board',
|
|
362
783
|
annotations: WRITE,
|
|
363
784
|
description: 'Generate a Reference Board: a dense multi-panel reference sheet (3:4, 4K) built from a source image and/or a written description, used to keep a subject on-model across later generations (feed the board back in as a referenceImage). Provide referenceImages and/or a prompt (at least one is required). Waits up to ~50s; boards render slowly (minutes), so it usually returns an outputId to poll with get_generation_status. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
@@ -399,7 +820,7 @@ export function registerTools(server, opts) {
|
|
|
399
820
|
if (args.getCost)
|
|
400
821
|
return costResult(await client.estimateBoardCost(request));
|
|
401
822
|
const gen = await client.generateBoardAndWait(request, { timeoutMs: SMART_WAIT_MS });
|
|
402
|
-
return completedResult(gen);
|
|
823
|
+
return completedResult(gen, await attachmentsFor(gen), [], client.baseUrl);
|
|
403
824
|
}
|
|
404
825
|
catch (err) {
|
|
405
826
|
// A SUBMITTED generation is running and charged. Whether the wait timed out or a
|
|
@@ -407,15 +828,16 @@ export function registerTools(server, opts) {
|
|
|
407
828
|
// dropping it invites a retry that generates and charges a second time.
|
|
408
829
|
const pending = pendingOutputId(err);
|
|
409
830
|
if (pending)
|
|
410
|
-
return pendingResult(pending);
|
|
831
|
+
return pendingResult(pending, pollAfterSecondsFor('image'), pendingShapeFrom(args, 'image'));
|
|
411
832
|
return errorResult(err);
|
|
412
833
|
}
|
|
413
834
|
});
|
|
414
835
|
// -- generate_video -------------------------------------------------------
|
|
415
836
|
server.registerTool('generate_video', {
|
|
837
|
+
...RENDERS_GENERATION,
|
|
416
838
|
title: 'Generate Video',
|
|
417
839
|
annotations: WRITE,
|
|
418
|
-
description: 'Generate a video from a text prompt (optionally from a start/end frame or reference images/videos/audio). Waits up to ~50s; if the render is still running it returns an outputId to poll with get_generation_status. Seedance 2.0 has two input modes selected by which references you pass: a startFrame (and optional endFrame) runs start/end-frame mode; referenceImages / referenceVideos / referenceAudio (without a startFrame) run references mode. Optionally pass projectId to place the generated video onto that project in the same call, controlled by an optional placement: a VIDEO timeline places a clip on a track, a CANVAS design places a layer on a slide (defaulting to the slide the user is focused on). Omit projectId to save a standalone library output. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
840
|
+
description: 'Generate a video from a text prompt (optionally from a start/end frame or reference images/videos/audio). Waits up to ~50s; if the render is still running it returns an outputId to poll with get_generation_status. Seedance 2.0 has two input modes selected by which references you pass: a startFrame (and optional endFrame) runs start/end-frame mode; referenceImages / referenceVideos / referenceAudio (without a startFrame) run references mode. Optionally pass projectId to place the generated video onto that project in the same call, controlled by an optional placement: a VIDEO timeline places a clip on a track, a CANVAS design places a layer on a slide (defaulting to the slide the user is focused on). Omit projectId to save a standalone library output. The result LINKS each output so the user sees it inline; to SEE it yourself (judge a face, check legibility, pick between variations) call get_media with the outputId. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
419
841
|
inputSchema: {
|
|
420
842
|
modelId: z.enum(models.video).describe(VIDEO_MODEL_GUIDANCE),
|
|
421
843
|
prompt: z
|
|
@@ -500,7 +922,7 @@ export function registerTools(server, opts) {
|
|
|
500
922
|
if (args.getCost)
|
|
501
923
|
return costResult(await client.estimateCost(request));
|
|
502
924
|
const gen = await client.generateAndWait(request, { timeoutMs: SMART_WAIT_MS });
|
|
503
|
-
return completedResult(gen);
|
|
925
|
+
return completedResult(gen, await attachmentsFor(gen), [], client.baseUrl);
|
|
504
926
|
}
|
|
505
927
|
catch (err) {
|
|
506
928
|
// A SUBMITTED generation is running and charged. Whether the wait timed out or a
|
|
@@ -508,7 +930,7 @@ export function registerTools(server, opts) {
|
|
|
508
930
|
// dropping it invites a retry that generates and charges a second time.
|
|
509
931
|
const pending = pendingOutputId(err);
|
|
510
932
|
if (pending)
|
|
511
|
-
return pendingResult(pending);
|
|
933
|
+
return pendingResult(pending, pollAfterSecondsFor('video'), pendingShapeFrom(args, 'video'));
|
|
512
934
|
return errorResult(err);
|
|
513
935
|
}
|
|
514
936
|
});
|
|
@@ -516,7 +938,7 @@ export function registerTools(server, opts) {
|
|
|
516
938
|
server.registerTool('generate_audio', {
|
|
517
939
|
title: 'Generate Audio',
|
|
518
940
|
annotations: WRITE,
|
|
519
|
-
description: 'Generate audio with ElevenLabs: speech (TTS), music, or a sound effect. Returns the audio URL directly (synchronous, no polling). Optionally pass projectId to place the generated audio onto that editor project\'s timeline in the same call, controlled by an optional placement; omit projectId to save a standalone library output. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
941
|
+
description: 'Generate audio with ElevenLabs: speech (TTS), music, or a sound effect. Returns the audio URL directly (synchronous, no polling). Optionally pass projectId to place the generated audio onto that editor project\'s timeline in the same call, controlled by an optional placement; omit projectId to save a standalone library output. The result LINKS each output so the user sees it inline; to SEE it yourself (judge a face, check legibility, pick between variations) call get_media with the outputId. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
520
942
|
inputSchema: {
|
|
521
943
|
modelId: z.enum(models.audio).describe(AUDIO_MODEL_GUIDANCE),
|
|
522
944
|
prompt: z.string().optional().describe('For music / sfx: what to generate.'),
|
|
@@ -617,6 +1039,7 @@ export function registerTools(server, opts) {
|
|
|
617
1039
|
});
|
|
618
1040
|
// -- upscale --------------------------------------------------------------
|
|
619
1041
|
server.registerTool('upscale', {
|
|
1042
|
+
...RENDERS_GENERATION,
|
|
620
1043
|
title: 'Upscale',
|
|
621
1044
|
annotations: WRITE,
|
|
622
1045
|
description: 'Upscale an existing image or video to a higher resolution. Provide the source media URL and a model-supported factor. Waits for the result; if the job is still running it returns an outputId to poll with get_generation_status. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
@@ -644,7 +1067,7 @@ export function registerTools(server, opts) {
|
|
|
644
1067
|
if (args.getCost)
|
|
645
1068
|
return costResult(await client.estimateCost(request));
|
|
646
1069
|
const gen = await client.generateAndWait(request, { timeoutMs: SMART_WAIT_MS });
|
|
647
|
-
return completedResult(gen);
|
|
1070
|
+
return completedResult(gen, await attachmentsFor(gen), [], client.baseUrl);
|
|
648
1071
|
}
|
|
649
1072
|
catch (err) {
|
|
650
1073
|
// A SUBMITTED generation is running and charged. Whether the wait timed out or a
|
|
@@ -652,12 +1075,13 @@ export function registerTools(server, opts) {
|
|
|
652
1075
|
// dropping it invites a retry that generates and charges a second time.
|
|
653
1076
|
const pending = pendingOutputId(err);
|
|
654
1077
|
if (pending)
|
|
655
|
-
return pendingResult(pending);
|
|
1078
|
+
return pendingResult(pending, pollAfterSecondsFor('image'), pendingShapeFrom(args, 'image'));
|
|
656
1079
|
return errorResult(err);
|
|
657
1080
|
}
|
|
658
1081
|
});
|
|
659
1082
|
// -- generate_lip_sync ----------------------------------------------------
|
|
660
1083
|
server.registerTool('generate_lip_sync', {
|
|
1084
|
+
...RENDERS_GENERATION,
|
|
661
1085
|
title: 'Generate Lip Sync',
|
|
662
1086
|
annotations: WRITE,
|
|
663
1087
|
description: 'Animate a portrait image so the subject speaks. Provide imageUrl (the face) plus a voice source: either audioUrl (an existing speech clip) or script + voiceId (we synthesize the speech). Optional motionPrompt nudges expression/motion. Waits up to ~50s; if still rendering it returns an outputId to poll with get_generation_status. SPENDS CREDITS: pass getCost to preview the price first, which runs nothing and charges nothing.',
|
|
@@ -705,7 +1129,7 @@ export function registerTools(server, opts) {
|
|
|
705
1129
|
if (args.getCost)
|
|
706
1130
|
return costResult(await client.estimateCost(request));
|
|
707
1131
|
const gen = await client.generateAndWait(request, { timeoutMs: SMART_WAIT_MS });
|
|
708
|
-
return completedResult(gen);
|
|
1132
|
+
return completedResult(gen, await attachmentsFor(gen), [], client.baseUrl);
|
|
709
1133
|
}
|
|
710
1134
|
catch (err) {
|
|
711
1135
|
// A SUBMITTED generation is running and charged. Whether the wait timed out or a
|
|
@@ -713,13 +1137,20 @@ export function registerTools(server, opts) {
|
|
|
713
1137
|
// dropping it invites a retry that generates and charges a second time.
|
|
714
1138
|
const pending = pendingOutputId(err);
|
|
715
1139
|
if (pending)
|
|
716
|
-
return pendingResult(pending);
|
|
1140
|
+
return pendingResult(pending, pollAfterSecondsFor('video'), pendingShapeFrom(args, 'video'));
|
|
717
1141
|
return errorResult(err);
|
|
718
1142
|
}
|
|
719
1143
|
});
|
|
720
1144
|
// -- transcribe -----------------------------------------------------------
|
|
721
1145
|
server.registerTool('transcribe', {
|
|
722
1146
|
title: 'Transcribe Audio',
|
|
1147
|
+
/*
|
|
1148
|
+
⭐ ANNOTATED, NOT JUST ARGUED FOR. The comment below has said "NOT read-only" since this tool
|
|
1149
|
+
shipped, and nobody ever wrote the annotation, so the tool went out carrying NEITHER hint. Claude
|
|
1150
|
+
files an unannotated tool under "Other", which is how 2 of 88 ended up unclassified: a comment
|
|
1151
|
+
describing an enforcement nobody built reads exactly like one that was built.
|
|
1152
|
+
*/
|
|
1153
|
+
annotations: WRITE,
|
|
723
1154
|
// NOT read-only, despite only returning text. readOnlyHint is a host's signal that a
|
|
724
1155
|
// tool is safe to call without asking the user, and this one is metered per minute of
|
|
725
1156
|
// audio: annotated READ, an agent could transcribe a two-hour file repeatedly,
|
|
@@ -1460,7 +1891,9 @@ export function registerTools(server, opts) {
|
|
|
1460
1891
|
// Image blocks are an MCP-layer concern: fetch the resolver-chosen still
|
|
1461
1892
|
// (imageUrl) for each item that has one (images + video posters). audio /
|
|
1462
1893
|
// transcript / posterless items stay text-only. See get-context §9.5.
|
|
1463
|
-
|
|
1894
|
+
// ⚠️ Ten items at 840 KB each is 8 MB against a 1 MB ceiling, and this used to fetch them all in
|
|
1895
|
+
// parallel with no bound. One shared budget, spent in order.
|
|
1896
|
+
const images = await inlineImagesWithinBudget(result.items.map((it) => (it.ok ? it.imageUrl : null)));
|
|
1464
1897
|
return mediaBatchResult(result, images);
|
|
1465
1898
|
}
|
|
1466
1899
|
catch (err) {
|
|
@@ -1700,6 +2133,7 @@ export function registerTools(server, opts) {
|
|
|
1700
2133
|
});
|
|
1701
2134
|
// -- get_generation_status ------------------------------------------------
|
|
1702
2135
|
server.registerTool('get_generation_status', {
|
|
2136
|
+
...RENDERS_GENERATION,
|
|
1703
2137
|
title: 'Get Generation Status',
|
|
1704
2138
|
annotations: READ,
|
|
1705
2139
|
description: "Check one or more in-progress generations (outputIds from generate_image / generate_video / upscale / generate_lip_sync / generate_board) and get their final URLs. BY DEFAULT THIS BLOCKS until they finish, up to ~50s per call, because that is almost always what you want after starting a render; if one is still running it comes back with the current status and a poll_after_seconds hint, so call again. Pass wait:false for an instant snapshot with no blocking. Accepts 1-8 outputIds in one call.",
|
|
@@ -1738,7 +2172,12 @@ export function registerTools(server, opts) {
|
|
|
1738
2172
|
}
|
|
1739
2173
|
}
|
|
1740
2174
|
}));
|
|
1741
|
-
|
|
2175
|
+
// ⭐ Only the single-generation case is attached: a batch of ten would embed ten sets of bytes into
|
|
2176
|
+
// one result. Polling ONE generation is the case a person is watching, and the one worth rendering.
|
|
2177
|
+
const attachments = gens.length === 1 && gens[0]
|
|
2178
|
+
? { [gens[0].outputId]: await attachmentsFor(gens[0]) }
|
|
2179
|
+
: {};
|
|
2180
|
+
return generationBatchResult(gens, attachments, client.baseUrl);
|
|
1742
2181
|
}
|
|
1743
2182
|
catch (err) {
|
|
1744
2183
|
return errorResult(err);
|
|
@@ -2496,19 +2935,44 @@ export function registerTools(server, opts) {
|
|
|
2496
2935
|
inputSchema: {
|
|
2497
2936
|
projectId: z.string().optional().describe('Scope to a specific project (editor/canvas). Omit for the user\'s most-recent-active surface anywhere. Required for render when no session is live.'),
|
|
2498
2937
|
capture: z.boolean().optional().describe("Also return a screenshot of the user's live viewport (their SCREEN), captured at read time. Default false returns structured context only. Request it only when the task depends on seeing the live, as-shown state including unsaved UI. To see the composed OUTPUT rather than the screen, use render instead."),
|
|
2499
|
-
render: z.boolean().optional().describe('Also
|
|
2500
|
-
mode: z.enum(['
|
|
2501
|
-
frame: z.number().int().min(0).optional().describe('
|
|
2502
|
-
slideId: z.string().optional().describe('
|
|
2503
|
-
slideIndex: z.number().int().min(1).optional().describe('
|
|
2504
|
-
fromFrame: z.number().int().min(0).optional().describe('
|
|
2505
|
-
toFrame: z.number().int().min(0).optional().describe('
|
|
2506
|
-
count: z.number().int().min(1).optional().describe('
|
|
2507
|
-
width: z.number().int().min(48).max(1440).optional().describe('
|
|
2938
|
+
render: z.boolean().optional().describe('Also render your work so you can visually verify edits. Ephemeral, stored nowhere, counts against no quota, works without a live tab. render=true alone renders the current focus point as one image; add count with fromFrame/toFrame for several across a range; mode=video returns a short playable clip of that range. Use this to check your work, not export_project, which produces a file the user KEEPS. To watch a RAW source clip instead of your composition, use get_media with a video item.'),
|
|
2939
|
+
mode: z.enum(['image', 'video']).optional().describe("What MEDIUM to render (default 'image'). 'image' returns composed frames INLINE: one by default, or several across a range when you pass count with fromFrame/toFrame, to judge motion, flow and cut placement. 'video' returns the range actually playing, as a short low-res composed clip, for timing a cut or a beat that separate frames cannot show; it is a JOB, returning a renderId to poll with get_preview, because it has to be rendered."),
|
|
2940
|
+
frame: z.number().int().min(0).optional().describe("mode='image' (editor): which single timeline frame to render. Omit to render the current playhead frame."),
|
|
2941
|
+
slideId: z.string().optional().describe("mode='image' (canvas): the id of the slide to render. Omit to render the focused slide."),
|
|
2942
|
+
slideIndex: z.number().int().min(1).optional().describe("mode='image' (canvas): the 1-based slide index to render (alternative to slideId)."),
|
|
2943
|
+
fromFrame: z.number().int().min(0).optional().describe('Start timeline frame of the range, for several frames or a video. Omit to start at the beginning.'),
|
|
2944
|
+
toFrame: z.number().int().min(0).optional().describe('End timeline frame of the range. Omit to run to the end.'),
|
|
2945
|
+
count: z.number().int().min(1).optional().describe("mode='image': how many frames to return across the range. Omit for one frame at the focus point, or a proportional default when a range is given."),
|
|
2946
|
+
width: z.number().int().min(48).max(1440).optional().describe("mode='image': render at an explicit DISPLAY width in pixels, to judge legibility at the size the output will actually be seen (a course tile, a thumbnail, a feed card) rather than at full resolution, where small type always looks fine. Height follows the composition aspect ratio and is not settable. Clamped; the size produced is reported back on rendered."),
|
|
2508
2947
|
},
|
|
2509
2948
|
}, async (args, extra) => {
|
|
2510
2949
|
try {
|
|
2511
2950
|
const client = await getClient(extra);
|
|
2951
|
+
/*
|
|
2952
|
+
⭐⭐⭐ **`video` IS THE THIRD RUNG OF THIS LADDER, NOT A SEPARATE TOOL.** It used to be
|
|
2953
|
+
`create_preview`, and the split was drawn on HOW the render is delivered (a job, not inline)
|
|
2954
|
+
rather than on WHAT the caller is asking for. Both answers to "let me look at my own work,
|
|
2955
|
+
ephemerally, without producing a deliverable" now live behind one question.
|
|
2956
|
+
|
|
2957
|
+
⛔ THE EVIDENCE THE OLD BOUNDARY WAS WRONG WAS IN THE DESCRIPTIONS. `get_context` ended with
|
|
2958
|
+
"for a composed VIDEO of a range use create_preview" and `create_preview` ended with "to see a
|
|
2959
|
+
single frame or a few frames use get_context render". Two tools each telling the agent when to
|
|
2960
|
+
use the other is routing work the schema should be doing. The CLI had already reached this
|
|
2961
|
+
conclusion: it exposes `context preview`, a sibling of `context`, while export lives under
|
|
2962
|
+
`project`.
|
|
2963
|
+
|
|
2964
|
+
⚠️ THE TRANSPORT IS UNCHANGED. This is a facade over the same client call the old tool made, so
|
|
2965
|
+
nothing moved server-side and the SDK needed no new field.
|
|
2966
|
+
*/
|
|
2967
|
+
if (args.mode === 'video') {
|
|
2968
|
+
const job = await client.createPreview({
|
|
2969
|
+
projectId: args.projectId ?? '',
|
|
2970
|
+
fromFrame: args.fromFrame,
|
|
2971
|
+
toFrame: args.toFrame,
|
|
2972
|
+
});
|
|
2973
|
+
return text(`Preview render started (frames ${job.fromFrame}-${job.toFrame}, ~${job.durationSeconds}s).\n` +
|
|
2974
|
+
`Poll get_preview with renderId="${job.renderId}" and bucketName="${job.bucketName}" until status is "done", then fetch the returned url.`);
|
|
2975
|
+
}
|
|
2512
2976
|
const result = await client.getContext({
|
|
2513
2977
|
projectId: args.projectId,
|
|
2514
2978
|
capture: args.capture,
|
|
@@ -2530,37 +2994,13 @@ export function registerTools(server, opts) {
|
|
|
2530
2994
|
return errorResult(err);
|
|
2531
2995
|
}
|
|
2532
2996
|
});
|
|
2533
|
-
server.registerTool('create_preview', {
|
|
2534
|
-
title: 'Create Preview',
|
|
2535
|
-
// NOT read-only. It does not charge the caller's credits (unlike transcribe), but it
|
|
2536
|
-
// STARTS A RENDER JOB: it returns a renderId you then poll, which is state that did
|
|
2537
|
-
// not exist before the call. readOnlyHint says a tool does not modify its
|
|
2538
|
-
// environment, and dispatching a Lambda render does. get_preview, which only reads
|
|
2539
|
-
// that job, stays READ.
|
|
2540
|
-
description: "Create an async PREVIEW of your work (ephemeral, never stored, not a deliverable). Currently a short low-res COMPOSED VIDEO of an editor range, so you can assess motion, cuts, transitions, and pacing that a still cannot show. This is a JOB: it returns a renderId + bucketName; poll get_preview with those until it is done, then fetch the returned url. To see a single frame or a few frames instead (cheaper, instant), use get_context render. Requires the context:read scope.",
|
|
2541
|
-
inputSchema: {
|
|
2542
|
-
projectId: z.string().describe('The editor project to preview.'),
|
|
2543
|
-
fromFrame: z.number().int().min(0).optional().describe('Start timeline frame of the range. Omit to start at the beginning.'),
|
|
2544
|
-
toFrame: z.number().int().min(0).optional().describe('End timeline frame. Omit to run to the end (capped to a short preview length).'),
|
|
2545
|
-
},
|
|
2546
|
-
}, async (args, extra) => {
|
|
2547
|
-
try {
|
|
2548
|
-
const client = await getClient(extra);
|
|
2549
|
-
const job = await client.createPreview({ projectId: args.projectId, fromFrame: args.fromFrame, toFrame: args.toFrame });
|
|
2550
|
-
return text(`Preview render started (frames ${job.fromFrame}-${job.toFrame}, ~${job.durationSeconds}s).\n` +
|
|
2551
|
-
`Poll get_preview with renderId="${job.renderId}" and bucketName="${job.bucketName}" until status is "done", then fetch the returned url.`);
|
|
2552
|
-
}
|
|
2553
|
-
catch (err) {
|
|
2554
|
-
return errorResult(err);
|
|
2555
|
-
}
|
|
2556
|
-
});
|
|
2557
2997
|
server.registerTool('get_preview', {
|
|
2558
2998
|
title: 'Get Preview',
|
|
2559
2999
|
annotations: READ,
|
|
2560
|
-
description: 'Poll a preview started with
|
|
3000
|
+
description: 'Poll a preview started by get_context with mode="video". While rendering, returns the progress; when done, returns a short-lived url to the ephemeral preview output (plus the estimated cost). Requires the context:read scope.',
|
|
2561
3001
|
inputSchema: {
|
|
2562
|
-
renderId: z.string().describe('The renderId returned by
|
|
2563
|
-
bucketName: z.string().describe('The bucketName returned by
|
|
3002
|
+
renderId: z.string().describe('The renderId returned by get_context with mode="video".'),
|
|
3003
|
+
bucketName: z.string().describe('The bucketName returned by get_context with mode="video".'),
|
|
2564
3004
|
},
|
|
2565
3005
|
}, async (args, extra) => {
|
|
2566
3006
|
try {
|