@kolbo/mcp 1.111.0 → 1.112.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/apps/index.js +3 -1
- package/src/apps/widgets/generation.js +3 -1
- package/src/index.js +2 -0
- package/src/toolAnnotations.js +6 -2
- package/src/tools/_shared.js +17 -4
- package/src/tools/generate.js +7 -6
- package/src/tools/video_inspection.js +41 -0
package/README.md
CHANGED
|
@@ -147,7 +147,7 @@ Without the optional skill, the config block alone already exposes every tool
|
|
|
147
147
|
| `generate_lipsync` | Source image/video + audio → lipsynced video (Sync-3 adds active-speaker selection, emotion, model mode, temperature) |
|
|
148
148
|
| `generate_creative_director` | One brief → N coordinated scenes (image or video) |
|
|
149
149
|
| `generate_music` | Text (+ optional lyrics) → song. Style, title, negative tags, length, and Suno fine-controls (style weight, weirdness, audio weight, persona / singing voice) |
|
|
150
|
-
| `generate_speech` | Text + voice → spoken audio. Full expressive/style control: Google/Gemini named voice-direction presets (`style_instructions_preset_id`: warm/dramatic/whisper/…) or free-form `style_instructions`, preset styles + emotions (DeepDub / MiniMax / Cartesia), speed, accent/`language`, and per-provider voice settings (ElevenLabs stability/similarity/style/speaker boost/seed, `enforce_language`, `output_format` mp3/wav, `text_normalization`; DeepDub accent/variance/tempo, MiniMax pitch/volume/intensity/timbre), plus `audio_effect` (phone/radio/megaphone/small_room/hall/distant) + `audio_effect_strength` for any voice at no extra credits. Status returns the same fields for reuse. |
|
|
150
|
+
| `generate_speech` | Text + voice → spoken audio. `eleven_model` (eleven_v4 / eleven_v3 / eleven_multilingual_v2) picks which ElevenLabs model speaks an ElevenLabs voice (stock or own clone; omit = the voice's default; same price; `turbo` needs v4). Full expressive/style control: Google/Gemini named voice-direction presets (`style_instructions_preset_id`: warm/dramatic/whisper/…) or free-form `style_instructions`, preset styles + emotions (DeepDub / MiniMax / Cartesia), speed, accent/`language`, and per-provider voice settings (ElevenLabs stability/similarity/style/speaker boost/seed, `enforce_language`, `output_format` mp3/wav, `text_normalization`; DeepDub accent/variance/tempo, MiniMax pitch/volume/intensity/timbre), plus `audio_effect` (phone/radio/megaphone/small_room/hall/distant) + `audio_effect_strength` for any voice at no extra credits. Status returns the same fields for reuse. |
|
|
151
151
|
| `generate_sound` | Text → sound effect. Duration, prompt influence, and per-provider controls (Stable Audio guidance, Kie loop/tempo/key, Seed-Audio voice/speed/volume/pitch + reference audio/image) |
|
|
152
152
|
| `generate_3d` | Text or reference images → 3D model (GLB/FBX/OBJ/USDZ) |
|
|
153
153
|
| `transcribe_audio` | Audio/video URL or file → text + SRT subtitles. Language, speaker diarization, audio-event tagging, and SRT formatting (words/line, lines/subtitle, caption stretch) |
|
package/package.json
CHANGED
package/src/apps/index.js
CHANGED
|
@@ -328,7 +328,9 @@ async function modelCatalog(client) {
|
|
|
328
328
|
? m.supported_aspect_ratios
|
|
329
329
|
: (Array.isArray(m.supportedAspectRatios) ? m.supportedAspectRatios : []);
|
|
330
330
|
const aspectsByType = m.supported_aspect_ratios_by_type || m.supportedAspectRatiosByType || null;
|
|
331
|
-
|
|
331
|
+
// images_per_request: fixed outputs per call (Midjourney = 4, whatever num_images says).
|
|
332
|
+
const ipr = Number(m.images_per_request || m.imagesPerRequest) || 1;
|
|
333
|
+
const info = { icon, eta, id: m.identifier || null, name: m.name || null, types, aspects, aspectsByType, ipr };
|
|
332
334
|
all.push(info);
|
|
333
335
|
// Display names collide across variants ("Nano Banana 2" names both the
|
|
334
336
|
// t2i model and its editing sibling) — on collision keep the model with
|
|
@@ -233,8 +233,9 @@ function voiceLabel(sc) { return sc.voice_name || sc.voice || (sc.settings || {}
|
|
|
233
233
|
// generate_speech controls. What the API reports it APPLIED wins: \`speech\`
|
|
234
234
|
// (status / completed payload; a key there is authoritative even when null),
|
|
235
235
|
// then the raw status result's own fields, then what the caller submitted.
|
|
236
|
-
var SPEECH_KEYS = ['turbo', 'speaking_speed', 'stability', 'similarity_boost', 'enforce_language',
|
|
236
|
+
var SPEECH_KEYS = ['eleven_model', 'turbo', 'speaking_speed', 'stability', 'similarity_boost', 'enforce_language',
|
|
237
237
|
'output_format', 'audio_effect', 'audio_effect_strength', 'seed'];
|
|
238
|
+
var SPEECH_MODELS = { eleven_v4: 'V4', eleven_v3: 'V3', eleven_multilingual_v2: 'V2' };
|
|
238
239
|
var SPEECH_FORMATS = { mp3_44100_128: 'MP3 128', mp3_44100_192: 'MP3 192', wav_44100: 'WAV' };
|
|
239
240
|
function withSpeech(settings, a) {
|
|
240
241
|
SPEECH_KEYS.forEach(function (k) { if (a && a[k] != null) settings[k] = a[k]; });
|
|
@@ -251,6 +252,7 @@ function num(v) { return String(Math.round(Number(v) * 100) / 100); }
|
|
|
251
252
|
function speechChipsHTML(sc) {
|
|
252
253
|
if (sc.tool !== 'generate_speech' && !sc.speech) return '';
|
|
253
254
|
var h = '', v;
|
|
255
|
+
v = speechVal(sc, 'eleven_model'); if (v) h += chip('Model ' + esc(SPEECH_MODELS[v] || v));
|
|
254
256
|
if (speechVal(sc, 'turbo') === true) h += chip('Turbo');
|
|
255
257
|
v = speechVal(sc, 'speaking_speed'); if (v != null) h += chip('Speed ' + esc(num(v)) + '×');
|
|
256
258
|
v = speechVal(sc, 'stability'); if (v != null) h += chip('Stability ' + esc(num(v)));
|
package/src/index.js
CHANGED
|
@@ -83,6 +83,7 @@ const { registerMusicLibraryTools } = require('./tools/music_library');
|
|
|
83
83
|
const { registerStockLibraryTools } = require('./tools/stock_library');
|
|
84
84
|
const { registerAudioStemTools } = require('./tools/audio_stems');
|
|
85
85
|
const { registerAnalyzeTools } = require('./tools/analyze');
|
|
86
|
+
const { registerVideoInspectionTools } = require('./tools/video_inspection');
|
|
86
87
|
const { registerBlenderTools } = require('./tools/blender');
|
|
87
88
|
const { registerAdobeTools } = require('./tools/adobe');
|
|
88
89
|
const { registerResolveTools } = require('./tools/resolve');
|
|
@@ -200,6 +201,7 @@ function createServer(opts = {}) {
|
|
|
200
201
|
registerTrendTools(server, client);
|
|
201
202
|
registerMorphiousStyleTools(server, client);
|
|
202
203
|
registerAnalyzeTools(server, client, toolOptions);
|
|
204
|
+
registerVideoInspectionTools(server, client);
|
|
203
205
|
registerMediaTools(server, client, toolOptions);
|
|
204
206
|
registerPresetTools(server, client, toolOptions);
|
|
205
207
|
registerArtifactTools(server, client, toolOptions);
|
package/src/toolAnnotations.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
const READ_ONLY = [
|
|
12
|
+
'get_video_inspection',
|
|
12
13
|
'get_flow_schema', 'list_flow_sessions', 'get_flow_session', 'validate_flow_session',
|
|
13
14
|
'estimate_flow_run', 'get_flow_run', 'list_flow_runs',
|
|
14
15
|
'list_morphious_styles',
|
|
@@ -40,7 +41,9 @@ const READ_ONLY = [
|
|
|
40
41
|
'create_music_composition_plan',
|
|
41
42
|
];
|
|
42
43
|
|
|
43
|
-
const OPEN_WORLD_READ_ONLY = [
|
|
44
|
+
const OPEN_WORLD_READ_ONLY = [
|
|
45
|
+
// Reads public source media into private ephemeral scratch evidence, no library writes.
|
|
46
|
+
'prepare_video_inspection', 'inspect_video',
|
|
44
47
|
'blender_list_sessions', 'blender_get_scene', 'blender_search_docs',
|
|
45
48
|
'blender_get_command_status',
|
|
46
49
|
'adobe_list_sessions', 'adobe_get_project', 'adobe_get_timeline', 'adobe_get_command_status',
|
|
@@ -100,7 +103,8 @@ const DESTRUCTIVE_WRITE = [
|
|
|
100
103
|
'chat_send_message', 'generate_character_sheet',
|
|
101
104
|
'acquire_clean_music_track', 'import_music_track_to_library',
|
|
102
105
|
'separate_audio_stems', 'clean_dialogue_leftovers', 'separate_ambience',
|
|
103
|
-
'analyze_video',
|
|
106
|
+
'analyze_video',
|
|
107
|
+
'analyze_video_evidence', 'transcribe_video_evidence', 'cancel_video_inspection',
|
|
104
108
|
|
|
105
109
|
// Deletes and whole-value replacement updates are conservatively destructive.
|
|
106
110
|
'delete_voice', 'update_visual_dna', 'delete_visual_dna', 'delete_visual_dna_folder',
|
package/src/tools/_shared.js
CHANGED
|
@@ -54,7 +54,8 @@ const LOCAL_FILE_ROUTING =
|
|
|
54
54
|
// where the model would read it, see no filesystem, and tell the user Kolbo
|
|
55
55
|
// cannot take their file at all. attachFileInputHints() below appends the route
|
|
56
56
|
// that actually works for the current transport, so the refusal never happens.
|
|
57
|
-
const FILE_INPUT_TOOLS = [
|
|
57
|
+
const FILE_INPUT_TOOLS = [
|
|
58
|
+
'prepare_video_inspection',
|
|
58
59
|
'generate_image', 'generate_image_edit', 'generate_creative_director',
|
|
59
60
|
'generate_video', 'generate_video_from_image', 'generate_video_from_video',
|
|
60
61
|
'generate_elements', 'generate_first_last_frame', 'generate_lipsync',
|
|
@@ -901,6 +902,17 @@ function moodboardIds(settings) {
|
|
|
901
902
|
* poll_tool widget-side status tool (default 'get_generation_status')
|
|
902
903
|
* status_args args for poll_tool (default { generation_id, wait: true })
|
|
903
904
|
*/
|
|
905
|
+
// How many outputs the card should expect. A model with a fixed
|
|
906
|
+
// images_per_request (Midjourney = 4) returns that many per generation
|
|
907
|
+
// regardless of num_images, so 2 prompts are 8 images, not "2 images".
|
|
908
|
+
async function outputCount(p) {
|
|
909
|
+
const info = p.model ? await modelInfo(p.client, p.model).catch(() => null) : null;
|
|
910
|
+
const ipr = info && info.ipr > 1 ? info.ipr : 0;
|
|
911
|
+
if (!ipr) return p.count || 1;
|
|
912
|
+
const gens = Array.isArray(p.generation_ids) && p.generation_ids.length ? p.generation_ids.length : 1;
|
|
913
|
+
return ipr * gens;
|
|
914
|
+
}
|
|
915
|
+
|
|
904
916
|
async function uiGenerating(p) {
|
|
905
917
|
// No ETAs anywhere — just a spinner until the poll flips to completed.
|
|
906
918
|
const chip = await modelChipFields(p.client, p.model, p.settings && p.settings.mode);
|
|
@@ -919,7 +931,7 @@ async function uiGenerating(p) {
|
|
|
919
931
|
...chip,
|
|
920
932
|
...(p.voice ? { voice_name: p.voice.name, voice_thumbnail: p.voice.thumbnail } : {}),
|
|
921
933
|
prompt: p.prompt,
|
|
922
|
-
count: p
|
|
934
|
+
count: await outputCount(p),
|
|
923
935
|
settings,
|
|
924
936
|
visual_dnas: await resolveVisualDnas(p.client, settings.visual_dna_ids),
|
|
925
937
|
moodboards: await resolveMoodboards(p.client, moodboardIds(settings)),
|
|
@@ -1023,7 +1035,7 @@ async function uiCompleted(p, textPayload, extraContent) {
|
|
|
1023
1035
|
tool: p.tool,
|
|
1024
1036
|
...chip,
|
|
1025
1037
|
prompt: p.prompt,
|
|
1026
|
-
count: p
|
|
1038
|
+
count: await outputCount(p),
|
|
1027
1039
|
// Omitted entirely when the caller has none. A live generation card merges
|
|
1028
1040
|
// an incoming status payload over its own state, so an empty-but-present
|
|
1029
1041
|
// `settings` wiped the resolution / aspect / DNA chips off the finished
|
|
@@ -1066,7 +1078,8 @@ async function uiCompleted(p, textPayload, extraContent) {
|
|
|
1066
1078
|
// voice's face. Exactly the "never show a raw id on a card" rule, broken on
|
|
1067
1079
|
// the one path the user actually ends up looking at.
|
|
1068
1080
|
...(p.voice ? { voice_name: p.voice.name, voice_thumbnail: p.voice.thumbnail } : {}),
|
|
1069
|
-
// Speech settings the API reports it applied (generate.js speechFromResult
|
|
1081
|
+
// Speech settings the API reports it applied (generate.js speechFromResult,
|
|
1082
|
+
// incl. eleven_model); passed through whole.
|
|
1070
1083
|
// the card prefers them over the submitted `settings`.
|
|
1071
1084
|
...(p.speech ? { speech: p.speech } : {}),
|
|
1072
1085
|
// The RAW generation state, when the caller has one. `phase` above is
|
package/src/tools/generate.js
CHANGED
|
@@ -163,7 +163,7 @@ function speechFromResult(r) {
|
|
|
163
163
|
stability: vs.stability, similarity_boost: vs.similarity_boost, style: vs.style, use_speaker_boost: vs.use_speaker_boost,
|
|
164
164
|
speaking_speed: r.speaking_speed, enforce_language: r.enforce_language, output_format: r.output_format,
|
|
165
165
|
text_normalization: r.text_normalization, audio_effect: r.audio_effect,
|
|
166
|
-
audio_effect_strength: r.audio_effect_strength, seed: r.seed,
|
|
166
|
+
audio_effect_strength: r.audio_effect_strength, seed: r.seed, eleven_model: r.eleven_model,
|
|
167
167
|
};
|
|
168
168
|
for (const k of Object.keys(out)) if (out[k] === undefined) out[k] = null;
|
|
169
169
|
return out;
|
|
@@ -316,7 +316,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
316
316
|
model: z.string().optional().describe('Model identifier — REQUIRED in practice: pick a specific model, do NOT omit (omitting = Smart Select auto-pick, which we avoid). Strong current defaults: "nano-banana-2" (versatile, text rendering, multilingual) or "gpt-image-2" (photoreal, infographics). Call list_models type="text_to_img" to see all options and pick per the user\'s intent.'),
|
|
317
317
|
aspect_ratio: z.string().optional().describe(aspectRatioDescribe('1:1')),
|
|
318
318
|
enhance_prompt: z.boolean().optional().describe('Enhance the prompt for better results. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
|
|
319
|
-
num_images: z.number().optional().describe('Number of images to generate in one call. Default: 1.
|
|
319
|
+
num_images: z.number().optional().describe('Number of images to generate in one call. Default: 1. Models with a fixed `images_per_request` in list_models IGNORE this: Midjourney ALWAYS returns 4 images per prompt (one call = 4 variations, billed as one request). So for "4 options" send ONE Midjourney prompt, never 4 calls or 4 entries in `prompts`; 2 prompts = 8 images.'),
|
|
320
320
|
reference_images: z.array(z.string()).optional().describe('STYLE/COMPOSITION inspiration only — does NOT embed reference pixels. Array of image URLs used to guide the look-and-feel of a brand-new generation. The model interprets the references and regenerates approximations conditioned on them. It will NOT copy pixels from these images into the output. **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.** To embed a specific logo, icon, watermark, or asset pixel-accurately, use generate_image_edit with the asset in source_images. To EDIT an existing image, also use generate_image_edit.'),
|
|
321
321
|
visual_dna_ids: z.array(z.string()).optional().describe('Visual DNA profile IDs (from create_visual_dna / list_visual_dnas) for character / style / product / scene consistency. **Cap: pass at most `max_visual_dna` IDs from list_models — if the field is null/0 or `supports_visual_dna: false`, the model rejects DNA entirely (silently ignored in some paths).** How DNA works: the server fetches the DNA\'s reference images AND always injects its `description` field into the prompt as plaintext (by design — independent of enhance_prompt). Practical implication: do NOT also write physical descriptors of the same subject in your own prompt — they will compete with the DNA description text. For pixel-accurate face anchoring of a specific person, prefer passing the DNA\'s reference image directly via source_images on generate_image_edit and OMIT visual_dna_ids. visual_dna_ids is best for style / scene / product DNAs and for soft consistency across a set.'),
|
|
322
322
|
moodboard_id: z.string().optional().describe('Moodboard ID (from list_moodboards / get_moodboard) whose master_prompt and style_guide should be applied to this generation.'),
|
|
@@ -1100,8 +1100,9 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1100
1100
|
{
|
|
1101
1101
|
text: z.string().describe('The text to convert to speech'),
|
|
1102
1102
|
voice: z.string().optional().describe('Voice ID or display name — MUST come from a `list_voices` result, never constructed. Google/Gemini ids in particular are not validated provider-side: an id that is not in the catalog is silently mapped to another voice (or a default one) and the audio comes back in a voice nobody asked for. Do not pattern-match a locale onto an id you saw for another language. Default: "Rachel"'),
|
|
1103
|
-
model: z.string().optional().describe('Model identifier. Use list_models type="text_to_speech" to see options. Default: eleven_v4 (ElevenLabs V4). ElevenLabs V4/V3 voices follow inline audio tags in the text: delivery ([whispers], [shouting], [sarcastic]), reactions ([laughs], [sighs], [clears throat]), pauses ([pause], [long pause]) and, on V4, free-form directions like [Low, steady voice, restrained urgency]. Tags are always English, even in Hebrew text. SSML such as <break> is NOT supported.'),
|
|
1103
|
+
model: z.string().optional().describe('Model identifier. Use list_models type="text_to_speech" to see options. Default: eleven_v4 (ElevenLabs V4). To choose which ElevenLabs model speaks a given ElevenLabs voice (stock or a cloned voice), use `eleven_model` instead. ElevenLabs V4/V3 voices follow inline audio tags in the text: delivery ([whispers], [shouting], [sarcastic]), reactions ([laughs], [sighs], [clears throat]), pauses ([pause], [long pause]) and, on V4, free-form directions like [Low, steady voice, restrained urgency]. Tags are always English, even in Hebrew text. SSML such as <break> is NOT supported.'),
|
|
1104
1104
|
turbo: z.boolean().optional().describe('ElevenLabs V4 voices ONLY. Turbo: faster generation at half the credits, slightly lower quality. Rejected (TURBO_UNSUPPORTED) for any voice not on ElevenLabs V4. Default false.'),
|
|
1105
|
+
eleven_model: z.enum(['eleven_v4', 'eleven_v3', 'eleven_multilingual_v2']).optional().describe("ElevenLabs voices ONLY (stock voices and the user's own cloned/character ElevenLabs voices). Which ElevenLabs model speaks: eleven_v4 (default for new voices, most expressive, audio tags, Stability/Similarity/Turbo), eleven_v3 (audio tags), eleven_multilingual_v2 (no audio tags; supports `style`). Omit = the voice's own default. Same price for all. Rejected with 400 ELEVEN_MODEL_UNSUPPORTED for non-ElevenLabs voices. `turbo` requires eleven_v4."),
|
|
1105
1106
|
language: z.string().optional().describe('Language / accent code (e.g., "en-US", "he-IL", "es-ES"). For Google/Gemini voices this is the Accent control (does not translate the text). Default: "en-US"'),
|
|
1106
1107
|
// ── Expressive style / emotion (provider-specific) ──
|
|
1107
1108
|
// Google/Gemini: prefer a named preset id from the Kolbo UI picker; free-form
|
|
@@ -1145,7 +1146,7 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1145
1146
|
project_id: projectIdField,
|
|
1146
1147
|
session_id: sessionIdField
|
|
1147
1148
|
},
|
|
1148
|
-
async ({ text, voice, model, turbo, language, style_instructions_preset_id, style_instructions, style_instructions_label, selected_style, emotion, speaking_speed, stability, similarity_boost, style, use_speaker_boost, enforce_language, output_format, text_normalization, audio_effect, audio_effect_strength, variance, tempo, promptBoost, seed, accentControl, voiceTitle, minimax_pitch, minimax_vol, minimax_intensity, minimax_timbre, project_id, session_id }) => {
|
|
1149
|
+
async ({ text, voice, model, eleven_model, turbo, language, style_instructions_preset_id, style_instructions, style_instructions_label, selected_style, emotion, speaking_speed, stability, similarity_boost, style, use_speaker_boost, enforce_language, output_format, text_normalization, audio_effect, audio_effect_strength, variance, tempo, promptBoost, seed, accentControl, voiceTitle, minimax_pitch, minimax_vol, minimax_intensity, minimax_timbre, project_id, session_id }) => {
|
|
1149
1150
|
model = await canonicalModelId(client, model, 'text_to_speech'); // lenient id resolution ("z-image" → "z-image/turbo")
|
|
1150
1151
|
// Resolve the requested voice against the REAL catalog (cached) so the card
|
|
1151
1152
|
// can show its display name + portrait instead of a raw id, and so an id
|
|
@@ -1158,9 +1159,9 @@ function registerGenerateTools(server, client, options = {}) {
|
|
|
1158
1159
|
? `Voice "${voice}" is not in the Kolbo voice catalog. Call list_voices and pass a voice_id it returns — an unrecognised id is NOT rejected, it is silently mapped to a different voice, so the audio will not be the voice you named.`
|
|
1159
1160
|
: null;
|
|
1160
1161
|
// ElevenLabs / effect controls carried on the card (undefined keys drop out of the JSON).
|
|
1161
|
-
const ttsControls = { turbo, stability, similarity_boost, style, use_speaker_boost, enforce_language, output_format, text_normalization, audio_effect, audio_effect_strength, seed };
|
|
1162
|
+
const ttsControls = { eleven_model, turbo, stability, similarity_boost, style, use_speaker_boost, enforce_language, output_format, text_normalization, audio_effect, audio_effect_strength, seed };
|
|
1162
1163
|
const gen = await client.post('/v1/generate/speech', {
|
|
1163
|
-
text, voice, model, turbo, language,
|
|
1164
|
+
text, voice, model, eleven_model, turbo, language,
|
|
1164
1165
|
style_instructions_preset_id, style_instructions, style_instructions_label,
|
|
1165
1166
|
selected_style, emotion, speaking_speed,
|
|
1166
1167
|
stability, similarity_boost, style, use_speaker_boost,
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
const { z } = require('zod');
|
|
3
|
+
const project = z.string().optional().describe('Authorized Kolbo project for billing; Act supplies its conversation project.');
|
|
4
|
+
const id = z.string().uuid().describe('Owner-scoped inspection ID returned by prepare_video_inspection.');
|
|
5
|
+
const text = value => ({ content: [{ type: 'text', text: JSON.stringify(value) }] });
|
|
6
|
+
|
|
7
|
+
// Additive tools. Source download and CPU-only measurements are free; provider calls
|
|
8
|
+
// stay explicit and billed. Never put image base64 into the language model's text.
|
|
9
|
+
function registerVideoInspectionTools(server, client) {
|
|
10
|
+
server.tool('prepare_video_inspection',
|
|
11
|
+
'Prepare a private source snapshot for adaptive video/audio investigation, including long recordings up to 2 GB. Streams the source once; returns promptly with preparing status. Poll get_video_inspection until ready. Accepts public direct media file URLs, not YouTube pages. Snapshot expires after two hours. No provider analysis or credits. Reuse the returned ID for frames, cuts and audio; refresh only when the URL content changed.',
|
|
12
|
+
{ video_url: z.string().url(), refresh: z.boolean().optional(), project_id: project },
|
|
13
|
+
async ({ video_url, refresh }) => text(await client.post('/v1/analyze/video-inspections', { video_url, refresh })));
|
|
14
|
+
server.tool('get_video_inspection', 'Check source preparation, duration, streams, dimensions, source version and limits. Do not poll unchanged status in a tight loop.',
|
|
15
|
+
{ inspection_id: id, project_id: project }, async ({ inspection_id }) => text(await client.get(`/v1/analyze/video-inspections/${inspection_id}`)));
|
|
16
|
+
server.tool('cancel_video_inspection', 'Stop preparation/extraction and expire your private snapshot. A paid provider request already accepted may still complete and be charged. Source media in the library is unchanged.',
|
|
17
|
+
{ inspection_id: id, project_id: project }, async ({ inspection_id }) => text(await client.post(`/v1/analyze/video-inspections/${inspection_id}/cancel`, {})));
|
|
18
|
+
server.tool('inspect_video',
|
|
19
|
+
'Extract bounded evidence from a prepared recording. frames returns timestamped contact-sheet IMAGE blocks (up to12 frames,6 per sheet); frame returns one larger IMAGE. Choose explicit timestamps or evenly sampled range/frame_count. cuts measures candidate changes on a640px/8fps proxy and must be verified against nearby frames. audio/clip extracts a private interval for transcribe_video_evidence or analyze_video_evidence. audio_levels measures volume/silence only. Each cut/audio/clip/levels interval is at most120 seconds. Sparse frames are observations, never proof of continuous review. Repeated identical requests reuse evidence. No provider credits.',
|
|
20
|
+
{ inspection_id: id, kind: z.enum(['frames', 'frame', 'cuts', 'audio', 'clip', 'audio_levels']).optional(),
|
|
21
|
+
timestamps: z.array(z.number().nonnegative()).min(1).max(12).optional(),
|
|
22
|
+
start_seconds: z.number().nonnegative().optional(), end_seconds: z.number().positive().optional(),
|
|
23
|
+
frame_count: z.number().int().min(1).max(12).optional(), project_id: project },
|
|
24
|
+
async ({ inspection_id, project_id, ...args }) => {
|
|
25
|
+
const result = await client.post(`/v1/analyze/video-inspections/${inspection_id}/evidence`, args, { timeoutMs: 115000 });
|
|
26
|
+
const { images = [], ...metadata } = result;
|
|
27
|
+
return { content: [{ type: 'text', text: JSON.stringify({ ...metadata, images: images.map(({ data, ...entry }) => entry) }) },
|
|
28
|
+
...images.map(img => ({ type: 'image', mimeType: img.mime_type, data: img.data }))] };
|
|
29
|
+
});
|
|
30
|
+
server.tool('analyze_video_evidence',
|
|
31
|
+
'Run paid video, music, speech or sound understanding on a previously extracted interval only. Video focus requires clip evidence; audio focus accepts audio or clip. Useful for motion/continuity, instruments/rhythm/mood, speech context and sound design. For word timestamps/SRT use transcribe_video_evidence. Returned analysis timestamps are clip-relative: add source_offset_seconds. Real model usage is billed, and identical successful requests reuse saved analysis without new charges. An uncertain previous paid request is never automatically resubmitted.',
|
|
32
|
+
{ inspection_id: id, evidence_id: z.string().regex(/^[a-f0-9]{24}$/),
|
|
33
|
+
focus: z.enum(['video', 'speech', 'music', 'sound']).optional(), prompt: z.string().max(2000).optional(),
|
|
34
|
+
quality: z.enum(['standard', 'hq']).optional(), project_id: project },
|
|
35
|
+
async ({ inspection_id, ...args }) => text(await client.post(`/v1/analyze/video-inspections/${inspection_id}/analyze`, args, { timeoutMs: 590000 })));
|
|
36
|
+
server.tool('transcribe_video_evidence',
|
|
37
|
+
'Transcribe an extracted audio interval using speech recognition with speaker labels, word timings and SRT. First inspect_video kind=audio for an interval up to120 seconds. Auto-detects language. Words include source_start/source_end in original recording seconds; SRT is clip-relative. Billed per rounded minute at current subtitle transcription pricing. Identical evidence/project requests reuse the transcript. For long recordings, transcribe selected windows or cover explicit consecutive windows and preserve their source offsets; never claim untranscribed audio was reviewed.',
|
|
38
|
+
{ inspection_id: id, evidence_id: z.string().regex(/^[a-f0-9]{24}$/), project_id: project },
|
|
39
|
+
async ({ inspection_id, ...args }) => text(await client.post(`/v1/analyze/video-inspections/${inspection_id}/analyze`, { ...args, focus: 'transcription' }, { timeoutMs: 150000 })));
|
|
40
|
+
}
|
|
41
|
+
module.exports = { registerVideoInspectionTools };
|