@kolbo/mcp 1.85.0 → 1.86.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kolbo/mcp",
-  "version": "1.85.0",
+  "version": "1.86.0",
   "description": "Kolbo AI MCP Server - Generate images, videos, music, speech, and sound effects from Claude Code",
   "main": "src/index.js",
   "bin": {

package/skill/GENERATED.md

@@ -1,6 +1,6 @@
 # AUTO-GENERATED — do not edit
 
-This tree is mirrored from kolbo-code@261fab2, the single source of truth.
+This tree is mirrored from kolbo-code@20a6572, the single source of truth.
 Canonical source: packages/opencode/skills/kolbo/
 Distribution: .github/workflows/sync-skill-to-plugin.yml
 

package/skill/SKILL.md

@@ -121,6 +121,7 @@ Each `references/models/*.md` mirrors the matching skill prompt in `kolbo-api/sr
 | `generate_speech` | TTS for narration, voiceover and standalone audio. **NOT for scene dialogue** — Seedance 2/2.5 performs quoted lines itself. |
 | `generate_sound` | Sound effects. |
 | `generate_3d` | 3D models from text / single image / multi-view. Returns GLB/FBX/OBJ/USDZ. |
+| `analyze_video` | Kolbo's official video understanding (agentic Gemini — navigates the timeline itself, so long videos are cheap and timestamp / counting / "when does X happen" questions are answered directly). Public video URL or YouTube URL + optional `prompt`; sync, token-billed. See `workflows/transcription.md`. |
 | `separate_audio_stems` | Split a soundtrack into Dialogue / Music / Effects / without-dialogue (M&E). The route for removing or isolating speech, instrumental beds, and stems for dubbing. 5cr, inline. See `workflows/audio-stems.md`. |
 | `clean_dialogue_leftovers` | Strip voices still faintly audible in an M&E layer. 17cr — only when the user reports the leak, it trades fidelity. |
 | `separate_ambience` | Pull room tone out of the Effects bed as its own lane. 17cr. |

package/skill/references/workflows/transcription.md

@@ -8,13 +8,24 @@ You have three routes. The right one depends on the file profile — pick before
 
 ```
 Image (jpg/png/webp)?                         → Read directly (native vision, up to 10 per pass)
-File >100MB OR >15 min OR dialogue-dense?     → HYBRID (transcribe + ffmpeg frames + Read + your synthesis)
+Any QUESTION about a video (what/when/how many/summarize/describe)? → analyze_video (agentic — the default for video)
 User wants the transcript/SRT as deliverable? → transcribe_audio, return the URLs
 Precise answer about one specific frame?      → ffmpeg that frame → Read
-Otherwise (short/medium video, mixed content) → upload_media → chat_send_message (Gemini native)
+File is only reachable locally and >100MB?    → split with ffmpeg, or HYBRID below
 ```
 
-## Why `upload_media` → chat is **not** always the default
+## `analyze_video` — Kolbo's official video understanding (use this first)
+
+Agentic Gemini: instead of sampling the video at a fixed frame rate, the model navigates the timeline itself — loading frames, audio, and the transcript only where the question needs them. That removes the two old failure modes below (long-form decay, transcription-dense laziness): a 90-minute lecture is answered from the parts that matter, at a fraction of the tokens.
+
+- `video_url` (public https, e.g. the URL returned by `upload_media` / `list_media`) **or** `youtube_url`.
+- `prompt`: the question. Ask directly — "At what timestamp does the logo appear?", "How many people speak, and who says X?", "List every product shown with its time". Omit for a full description + verbatim transcript.
+- `quality`: `standard` (default) or `hq` (short clips where precision matters; ~2.5x the token price).
+- Sync — returns `analysis`, `model`, `usage`, `credits_used`. Billed by real token usage; long videos are still cheap under agentic navigation.
+- Local file → `create_upload_ticket` / `upload_media` / `media_upload_widget` first, then pass the URL.
+- It does not return SRT files. For subtitles or word timings use `transcribe_audio`.
+
+## Why `upload_media` → chat is **not** the default for video questions (use `analyze_video`)
 
 Gemini-via-chat processes frames + motion + audio in one pass and is the simplest route when it works. But it has three known failure surfaces — recognize them and pivot to the hybrid path:
 

package/src/tools/generate.js

@@ -10,6 +10,7 @@ const { resolveToBuffer, pollOrTimedOut, creditFields, projectIdField, sessionId
 const { ownedUrl } = require('./owned-url');
 const { UI, uiResult, canonicalModelId, assertModelSupportsType, modelInfo, voiceInfo, resolveCatalogAspectRatio } = require('../apps');
 const { modelTypeForEditOperation, assertExecutableEditModel } = require('./editModelCatalog');
+const { withLocalRehost } = require('./local-rehost');
 
 // ─── Cinematic Dimensions schema (shared by generate_image + generate_image_edit) ───
 // Kolbo's "Cinema mode": eight independent photographic dimensions, each an OPTIONAL
@@ -201,6 +202,9 @@ const promptsField = (what) => z.array(z.string()).max(MAX_BATCH_PROMPTS).option
 );
 
 function registerGenerateTools(server, client, options = {}) {
+  // Every JSON POST from these tools rehosts local file paths into the media
+  // library first (see local-rehost.js) — the API only understands URLs.
+  client = withLocalRehost(client, { allowLocalFiles: !options.remote });
   // Only enabled by hosts that explicitly opt in (the remote HTTP connector).
   // stdio hosts (Kolbo Code, Claude Desktop, Cursor) leave this false, so their
   // tool output is unchanged: a text block with the image URL.
@@ -313,7 +317,7 @@ function registerGenerateTools(server, client, options = {}) {
       prompt: z.string().optional().describe('Description of the edit to apply (e.g., "remove the background", "change the sky to sunset"). Required unless `prompts` is provided.'),
       prompts: promptsField('edits of the SAME source images'),
       model: z.string().optional().describe('Model identifier — REQUIRED in practice: pick a specific model, do NOT omit (omitting = Smart Select auto-pick, which we avoid). Many text-to-image ids double as editors: the server auto-routes a base id to its editing variant when source_images is present (e.g. "gpt-image-2" → gpt-image-2/edit, "nano-banana-2" → nano-banana-2-image-editing) — passing the bare id is fine, no need to hunt for the "/edit" suffix yourself. BUT this only works for models that actually have a registered edit variant. For prompt-driven photoreal photo edits (object removal, keep-this-person/remove-the-rest, crowd cleanup, inpainting) the ONLY auto-pick defaults are "nano-banana-2" or "gpt-image-2" (use GPT Image 2 when the image needs readable text). Do NOT auto-pick Flux 2 / flux-2/edit / Flux Klein — those are generate-from-scratch / style models; use them only if the user names Flux. If unsure, confirm the model appears in `list_models type="image_editing"` and choose by the strengths summary — Flux edit variants are named-only.'),
-      source_images: z.array(z.string()).describe('PIXEL-ACCURATE compositing. Array of source image URLs whose pixel content is composited into the output. **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.** Three modes the model auto-detects from input shape: (1) Single image → edit/transform that image. (2) Multiple images, one base + others → composite the others into the base. (3) Multiple images with no clear base → generate a new scene that pixel-accurately embeds the supplied images at positions described in the prompt. Mode 3 is the canonical pattern for thumbnails / branded compositions where exact-pixel logo + face fidelity matter. Refer to source images in the prompt by ordinal position ("FIRST source image", "SECOND source image") or use @image1/@image2 tags. Add "composite AS-IS, do not redraw or restyle" to lock pixels.'),
+      source_images: z.array(z.string()).describe('PIXEL-ACCURATE compositing. Array of source images (URLs or absolute local paths) whose pixel content is composited into the output. **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.** Three modes the model auto-detects from input shape: (1) Single image → edit/transform that image. (2) Multiple images, one base + others → composite the others into the base. (3) Multiple images with no clear base → generate a new scene that pixel-accurately embeds the supplied images at positions described in the prompt. Mode 3 is the canonical pattern for thumbnails / branded compositions where exact-pixel logo + face fidelity matter. Refer to source images in the prompt by ordinal position ("FIRST source image", "SECOND source image") or use @image1/@image2 tags. Add "composite AS-IS, do not redraw or restyle" to lock pixels.'),
       reference_images: z.array(z.string()).optional().describe('STYLE/COMPOSITION inspiration, alongside `source_images` on the same call — does NOT embed reference pixels. Use when the edit should follow a look sampled from other images ("re-light this shot like these references"). The pixels that must survive the edit go in `source_images`; these only steer the look. **Cap: `source_images` + `reference_images` together must not exceed `max_reference_images` from list_models for the chosen model.**'),
       aspect_ratio: z.string().optional().describe(aspectRatioDescribe('1:1')),
       enhance_prompt: z.boolean().optional().describe('Enhance the prompt for better results. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
@@ -409,7 +413,7 @@ function registerGenerateTools(server, client, options = {}) {
       workflow_type: z.string().optional().describe('"image" (default) or "video"'),
       duration: z.number().optional().describe('Duration in seconds per scene (video mode only). Must be a value in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration`. E.g., 5 or 10.'),
       enhance_prompt: z.boolean().optional().describe('Enhance prompts per scene. Default: false — only pass true if the user explicitly asks to enhance/improve the prompts.'),
-      reference_images: z.array(z.string()).optional().describe('Array of reference image URLs to guide style/composition of every scene. **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model.**'),
+      reference_images: z.array(z.string()).optional().describe('Array of reference images (URLs or absolute local paths) to guide style/composition of every scene. **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model.**'),
       visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply consistently across every scene. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model.** This is the ideal way to keep a character or product looking the same in all scenes of a campaign.'),
       moodboard_id: z.string().optional().describe('A single moodboard ID whose master_prompt and style_guide should shape every scene.'),
       moodboard_ids: z.array(z.string()).optional().describe('Multiple moodboard IDs when blending styles. Prefer `moodboard_id` for single moodboards.'),
@@ -578,7 +582,7 @@ function registerGenerateTools(server, client, options = {}) {
       aspect_ratio: z.string().optional().describe(aspectRatioDescribe('16:9')),
       duration: z.number().optional().describe('Duration in seconds. Must be a value in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration` (whichever the model exposes). Default: 5'),
       enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
-      reference_images: z.array(z.string()).optional().describe('Array of image URLs used as visual references (style / composition / subject). **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.**'),
+      reference_images: z.array(z.string()).optional().describe('Array of images (URLs or absolute local paths) used as visual references (style / composition / subject). **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.**'),
       resolution: z.string().optional().describe('Video resolution tier (vertical pixels): "720p" / "1080p" / "1440p" / "2160p". Some models use labels like "512P"/"1024P"/"768P"/"1080P". Model-dependent — call list_models and read supported_resolutions. Read resolution_multipliers to predict cost.'),
       preset_id: z.string().optional().describe('Preset ID from list_presets type="video" to apply a saved motion/style preset to this generation.'),
       visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply for character/style consistency. Every DNA passed here MUST also be tagged in the prompt as @ExactDNAName. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model; if `supports_visual_dna: false`, DNA is silently ignored.**'),
@@ -655,10 +659,10 @@ function registerGenerateTools(server, client, options = {}) {
     'generate_video_from_image',
     'Animate an existing still image into a video using Kolbo AI. The image comes from `image_url`; `prompt` describes the motion (not the subject — the subject is already in the image). For generating a video from scratch, use generate_video. FOR A MULTI-SHOT FILM with recurring characters or locations, prefer ONE generate_elements call carrying the whole DNA-anchored sequence over a series of separate image-to-video clips. ANIMATING SEVERAL SHOTS OF THE SAME SEQUENCE? Pass them ALL in `items` in ONE call (one combined widget) — never a series of separate calls, which buries the chat under one widget per clip. Across calls (a sequence longer than the batch cap), make the first call without `session_id`, take the `session_id` from its result, and pass that same `session_id` on every following call — otherwise each clip becomes its own session and the user gets a stack of near-identical single-clip sessions in the Kolbo sidebar. Returns the final video URL(s) when complete.',
     {
-      image_url: z.string().optional().describe('URL of the source image to animate. Required unless `items` is provided.'),
+      image_url: z.string().optional().describe('URL or absolute local path of the source image to animate. Required unless `items` is provided.'),
       prompt: z.string().optional().describe('Text description of the desired MOTION (e.g., "camera slowly pans right while the character walks forward"). Required unless `items` is provided.'),
       items: z.array(z.object({
-        image_url: z.string().describe('URL of the source image to animate for THIS clip.'),
+        image_url: z.string().describe('URL or absolute local path of the source image to animate for THIS clip.'),
         prompt: z.string().describe('Text description of the desired MOTION for THIS clip.'),
       })).max(MAX_BATCH_PROMPTS).optional().describe(
         `BATCH MODE — several DIFFERENT stills (2–${MAX_BATCH_PROMPTS}) animated concurrently in ONE call and rendered together in ONE combined widget. Unlike the \`prompts\` array on generate_image / generate_video, each entry pairs its OWN \`image_url\` with its OWN motion \`prompt\` — the image is what varies, and that is the point. **Hard cap: ${MAX_BATCH_PROMPTS} items per call — more than that is REJECTED with an error (never silently truncated), so split a longer sequence across several calls of at most ${MAX_BATCH_PROMPTS}.** Whenever the user wants several stills animated (a shot sequence, a storyboard, an animatic), ALWAYS pass them all here instead of making several separate calls — separate calls clutter the chat with stacked widgets. All items share the same model / duration / resolution / aspect_ratio / sound_enabled / project_id / session_id. When set, \`image_url\` and \`prompt\` are ignored.`
@@ -1892,7 +1896,7 @@ function registerGenerateTools(server, client, options = {}) {
     'Generate a 3D model from a text prompt, a single reference image, or multiple reference images (for multi-view reconstruction). Returns model URLs in multiple formats (GLB, FBX, OBJ, USDZ). Modes: "text" (prompt-only), "single" (one image), "multi" (multiple images for better quality). The mode is auto-detected from the inputs if not specified.',
     {
       prompt: z.string().optional().describe('Text description of the 3D object to generate (used in text mode and also as a hint in image modes)'),
-      reference_images: z.array(z.string()).optional().describe('Array of public image URLs. 1 image → single mode, 2+ → multi mode.'),
+      reference_images: z.array(z.string()).optional().describe('Array of image URLs or absolute local paths. 1 image → single mode, 2+ → multi mode.'),
       mode: z.string().optional().describe('Explicitly set mode: "text" | "single" | "multi". Auto-detected from reference_images if omitted.'),
       texture_prompt: z.string().optional().describe('Optional prompt to guide texture generation'),
       model: z.string().optional().describe('Model identifier. Use list_models type="three_d" to see all 3D options, or filter by sub-type: "3d_text_to_model", "3d_image_to_model", "3d_multi_image_to_model", "3d_world".'),
@@ -1955,7 +1959,7 @@ function registerGenerateTools(server, client, options = {}) {
     'edit_image',
     'Apply a targeted AI edit to an existing image. Covers mechanical enhancements (upscale, remove background, skin retouching) AND creative operations (expand/outpaint, reframe, inpaint, erase, face swap, background replace, camera angle, multi-shot grid, split/upscale). ⚠️ To EXPAND an image / add space / widen it to a new aspect ratio while keeping the existing artwork intact, use operation="zoom_out" — NOT "reframe" (reframe re-generates the whole picture). ⚠️ For open-ended PROMPT-DRIVEN content edits — "make it night", restyling, adding/removing objects — use `generate_image_edit` instead; it runs on stronger dedicated editing models and produces better results.',
     {
-      image_url: z.string().describe('URL of the primary source image to edit.'),
+      image_url: z.string().describe('URL or absolute local path of the primary source image to edit.'),
 
       operation: z.enum([
         'upscale', 'clarity_upscale',
@@ -2032,10 +2036,10 @@ function registerGenerateTools(server, client, options = {}) {
         .describe('Text instruction guiding the edit. Required for "background_replace". Used with "inpaint", "zoom_out", "camera_angle", and the deprecated "magic_edit". For "zoom_out" it describes what fills the NEW space only — the original image is left as-is.'),
 
       mask_image_url: z.string().optional()
-        .describe('URL of a mask image (black & white; white = affected area). Required for "inpaint" and "erase". For "face_swap", this is the face reference image.'),
+        .describe('URL or absolute local path of a mask image (black & white; white = affected area). Required for "inpaint" and "erase". For "face_swap", this is the face reference image.'),
 
       additional_images: z.array(z.string()).optional()
-        .describe('Extra reference image URLs (up to 8). For "inpaint": reference images that guide style/content. For "multi_shot": the set of scene reference shots. For "magic_edit": additional source images for composite edits.'),
+        .describe('Extra reference images — URLs or absolute local paths (up to 8). For "inpaint": reference images that guide style/content. For "multi_shot": the set of scene reference shots. For "magic_edit": additional source images for composite edits.'),
 
       // ── camera_angle ───────────────────────────────────────
       generate_all_angles: z.boolean().optional()

package/src/tools/local-rehost.js

@@ -0,0 +1,74 @@
+'use strict';
+
+// Every JSON-body generation tool (edit_image, generate_image_edit,
+// generate_video_from_image, generate_3d, ...) advertised "URL or absolute
+// local path" but shipped the path string straight to the API, which only
+// understands URLs. Only the multipart tools (elements, first/last frame,
+// lipsync, v2v, transcribe) actually read the file. Instead of teaching every
+// handler about files, the client handed to the generate tools rehosts any
+// local path found anywhere in a POST body into the media library and swaps
+// in the CDN URL. URLs and plain text are never touched.
+
+const fs = require('fs');
+const path = require('path');
+const FormData = require('form-data');
+const { resolveToBuffer } = require('./_shared');
+
+const MAX_UPLOAD_BYTES = 200 * 1024 * 1024;
+// Free-text fields are skipped so a prompt is never stat()ed.
+const TEXT_KEYS = new Set(['prompt', 'prompts', 'text', 'lyrics', 'style', 'title', 'negative_tags', 'system_prompt', 'srt_content', 'style_instructions', 'customization', 'vocabulary']);
+
+const isUrl = (s) => /^https?:\/\//i.test(s);
+// Absolute POSIX / Windows / UNC path that ends in a file extension.
+const looksLikeLocalFile = (s) =>
+  typeof s === 'string' && s.length < 1024 && !isUrl(s) && !/[\n\r]/.test(s) &&
+  (path.isAbsolute(s) || path.win32.isAbsolute(s)) && /\.[a-z0-9]{2,5}$/i.test(s);
+
+async function uploadLocal(client, source, projectId, opts) {
+  const file = await resolveToBuffer(source, 'image', { maxBytes: MAX_UPLOAD_BYTES, allowLocalFiles: opts.allowLocalFiles });
+  const form = new FormData();
+  form.append('file', file.buffer, { filename: file.filename, contentType: file.contentType });
+  if (projectId) form.append('project_id', projectId);
+  const uploaded = await client.postMultipart('/v1/media/upload', form);
+  const url = uploaded?.media?.url || uploaded?.url;
+  if (!url) throw new Error(`Upload of ${source} returned no URL`);
+  return url;
+}
+
+/**
+ * Deep-walk a request body; replace every local file path with a CDN URL.
+ * @param {object} client - Kolbo HTTP client (needs postMultipart)
+ * @param {*} body
+ * @param {{allowLocalFiles?: boolean}} [opts] - false on remote connectors:
+ *   a local path then throws the routing hint instead of reaching the API.
+ */
+async function rehostLocalPaths(client, body, opts = {}) {
+  if (!body || typeof body !== 'object') return body;
+  const projectId = body.project_id;
+  const cache = new Map();
+  const walk = async (value, key) => {
+    if (typeof value === 'string') {
+      if (TEXT_KEYS.has(key) || !looksLikeLocalFile(value)) return value;
+      if (opts.allowLocalFiles !== false && !(fs.existsSync(value) && fs.statSync(value).isFile())) return value;
+      if (!cache.has(value)) cache.set(value, uploadLocal(client, value, projectId, opts));
+      return cache.get(value);
+    }
+    if (Array.isArray(value)) return Promise.all(value.map((v) => walk(v, key)));
+    if (value && typeof value === 'object' && !Buffer.isBuffer(value)) {
+      const out = {};
+      for (const [k, v] of Object.entries(value)) out[k] = await walk(v, k);
+      return out;
+    }
+    return value;
+  };
+  return walk(body);
+}
+
+/** Wrap a client so `post()` rehosts local paths first; everything else is inherited. */
+function withLocalRehost(client, opts = {}) {
+  return Object.create(client, {
+    post: { value: async (route, body, ...rest) => client.post(route, await rehostLocalPaths(client, body, opts), ...rest) },
+  });
+}
+
+module.exports = { rehostLocalPaths, withLocalRehost, looksLikeLocalFile };