@aexol/spectral 0.9.231 → 0.9.234

package/dist/extensions/video-generation/index.js

@@ -0,0 +1,299 @@
+import { Type } from "typebox";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { extname, join, resolve } from "node:path";
+import { generateVideos, getVideoModel, getVideoModels } from "../../sdk/ai/index.js";
+import { base64FromImageData } from "../../sdk/ai/utils/image-source.js";
+import { uploadImage } from "../../backend/generated-images.js";
+// Backend routing (backendUrl + machineJwt) is resolved the same way the
+// image tool gets it, so video generation routes through the backend proxy
+// (POST /video/generation) using the backend's own OPENROUTER_API_KEY and
+// falls back to direct OpenRouter when the proxy is unavailable or the CLI
+// is not logged in.
+import { resolveBackendRouting } from "../../backend/machine-routing.js";
+const frameImageSchema = Type.Union([
+    Type.String({
+        description: "Path to a frame image file (relative to the project working directory or absolute).",
+    }),
+    Type.Object({
+        data: Type.String({ description: "Frame image as raw base64 or data URL." }),
+        mimeType: Type.Optional(Type.String({ default: "image/png", description: "Frame image MIME type." })),
+    }, { description: "Frame image provided inline as base64 data." }),
+    Type.Object({
+        path: Type.String({ description: "Path to a frame image file (relative to the project working directory or absolute)." }),
+        mimeType: Type.Optional(Type.String({ description: "Frame image MIME type. Defaults to the type detected from the file extension." })),
+    }, { description: "Frame image provided as a file path." }),
+]);
+const videoGenerateSchema = Type.Object({
+    prompt: Type.String({ description: "Text prompt describing the video to generate." }),
+    model: Type.Optional(Type.String({ default: "google/veo-3.1", description: "OpenRouter video model id." })),
+    duration: Type.Optional(Type.Number({ description: "Clip duration in seconds (model-dependent, e.g. 4-10 for Veo)." })),
+    resolution: Type.Optional(Type.String({ description: "Resolution tier, e.g. 480p, 720p, 768p, 1080p." })),
+    aspect_ratio: Type.Optional(Type.String({ description: "Aspect ratio, e.g. 16:9, 9:16, 1:1." })),
+    size: Type.Optional(Type.String({ description: "OpenRouter size shorthand, e.g. 1280x720." })),
+    generate_audio: Type.Optional(Type.Boolean({ default: true, description: "Whether to generate audio with the video (model-dependent)." })),
+    seed: Type.Optional(Type.Number()),
+    frame_images: Type.Optional(Type.Array(frameImageSchema, { description: "Optional frame images for image-to-video generation. Each element may be a file path string, an object { data: <base64 or data URL>, mimeType?: string }, or an object { path: <file path>, mimeType?: string }." })),
+});
+function videoModel(modelId) {
+    const exact = getVideoModel("openrouter", modelId);
+    if (exact)
+        return exact;
+    return getVideoModels("openrouter").find((model) => model.id === modelId);
+}
+const GENERATED_VIDEOS_DIR = ".generated_videos";
+const VIDEO_MIME_TYPE = "video/mp4";
+function mimeTypeForPath(filePath) {
+    switch (extname(filePath).toLowerCase()) {
+        case ".png":
+            return "image/png";
+        case ".jpg":
+        case ".jpeg":
+            return "image/jpeg";
+        case ".webp":
+            return "image/webp";
+        case ".gif":
+            return "image/gif";
+        default:
+            return "image/png";
+    }
+}
+function readFrameImageFromPath(filePath, mimeType) {
+    let data;
+    try {
+        data = readFileSync(filePath).toString("base64");
+    }
+    catch (error) {
+        const code = error?.code;
+        if (code === "ENOENT") {
+            throw new Error(`Frame image file does not exist: ${filePath}`);
+        }
+        if (code === "EACCES" || code === "EPERM") {
+            throw new Error(`Permission denied while reading frame image: ${filePath}`);
+        }
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to read frame image: ${filePath} (${detail})`);
+    }
+    return {
+        type: "image",
+        data,
+        mimeType: mimeType ?? mimeTypeForPath(filePath),
+    };
+}
+function normalizeFrameImages(frames, cwd) {
+    if (!frames)
+        return [];
+    return frames.map((frame, index) => {
+        if (typeof frame === "string") {
+            return readFrameImageFromPath(resolve(cwd, frame), undefined);
+        }
+        if (typeof frame !== "object" || frame === null) {
+            throw new Error(`Invalid frame image at index ${index}: expected a file path string or an object with "data" or "path".`);
+        }
+        const reference = frame;
+        if (typeof reference.data === "string" && typeof reference.path === "string") {
+            throw new Error(`Invalid frame image at index ${index}: provide only one of "data" (inline base64) or "path" (file path), not both.`);
+        }
+        if (typeof reference.data === "string") {
+            return {
+                type: "image",
+                data: reference.data,
+                mimeType: reference.mimeType ?? "image/png",
+            };
+        }
+        if (typeof reference.path === "string") {
+            return readFrameImageFromPath(resolve(cwd, reference.path), reference.mimeType);
+        }
+        throw new Error(`Invalid frame image at index ${index}: expected a file path string or an object with "data" or "path".`);
+    });
+}
+function slugFromPrompt(prompt) {
+    const slug = prompt
+        .normalize("NFKD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 40);
+    return slug || "video";
+}
+function timestamp() {
+    return new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14);
+}
+/**
+ * Save a locally-downloaded video (direct OpenRouter path) under
+ * `.generated_videos/`. Proxy-path videos have no inline bytes here — the
+ * backend stores them and reports the hosted url instead.
+ */
+function saveGeneratedVideo(video, prompt, cwd) {
+    if (!video.data)
+        return undefined;
+    const dir = join(cwd, GENERATED_VIDEOS_DIR);
+    mkdirSync(dir, { recursive: true });
+    const filename = `${slugFromPrompt(prompt)}-${timestamp()}.mp4`;
+    const filepath = join(dir, filename);
+    writeFileSync(filepath, Buffer.from(base64FromImageData(video.data), "base64"));
+    return filepath;
+}
+function bytesFromVideoData(data) {
+    return new Uint8Array(Buffer.from(base64FromImageData(data), "base64"));
+}
+/**
+ * Make the generated video reachable through a backend URL so the session can
+ * carry a url instead of megabytes of base64. Only applies to videos that
+ * came back as inline bytes (direct OpenRouter fallback path) — proxy-path
+ * videos are already stored server-side. Best effort by design: an upload
+ * failure must not fail video generation; the caller falls back to the local
+ * file + inline bytes and annotates the result text.
+ */
+async function hostGeneratedVideo(video, routing, signal) {
+    if (video.url && video.videoId)
+        return { hosted: { url: video.url, videoId: video.videoId } };
+    if (!routing)
+        return { error: "not routed through the backend (video kept local)" };
+    if (!video.data)
+        return { error: "video has no inline bytes (kept local)" };
+    try {
+        const result = await uploadImage({
+            backendUrl: routing.backendUrl,
+            token: routing.machineJwt,
+            bytes: bytesFromVideoData(video.data),
+            contentType: video.mimeType || VIDEO_MIME_TYPE,
+            kind: "video",
+            signal,
+        });
+        return { result };
+    }
+    catch (error) {
+        return { error: error instanceof Error ? error.message : String(error) };
+    }
+}
+function isHosted(outcome) {
+    return Boolean(outcome.hosted ?? outcome.result);
+}
+function hostedUrl(outcome) {
+    return outcome.hosted?.url ?? outcome.result.url;
+}
+function hostedId(outcome) {
+    return outcome.hosted?.videoId ?? outcome.result.imageId;
+}
+function resultText(model, credits, savedPath, outcome) {
+    const lines = ["✓ Generated 1 video", `Model: ${model}`];
+    lines.push(`Cost: ${credits ?? 0} credits`);
+    if (savedPath)
+        lines.push(`Saved: ${savedPath}`);
+    else if (!isHosted(outcome))
+        lines.push("Saved: (no local copy — see errors below)");
+    if (isHosted(outcome)) {
+        lines.push(`Hosted: ${hostedUrl(outcome)}`);
+    }
+    if (!isHosted(outcome) && outcome.error) {
+        // Explicit annotation: the video bytes were NOT uploaded, so the video
+        // exists only as the local file (path above); remote URL unavailable.
+        lines.push(`⚠️ The video could not be uploaded to cloud storage and exists only locally; remote URL unavailable.`);
+        lines.push(`- ${outcome.error}`);
+    }
+    return lines.join("\n");
+}
+export default async function videoGenerationExtension(ext) {
+    const tool = {
+        name: "video_generate",
+        label: "Generate Video",
+        description: "Generate videos through OpenRouter's Videos API and save them to .generated_videos/ in the project. " +
+            "Returns the saved file path and hosted URL so the user can view the generated video. " +
+            "Supports text-to-video and image-to-video via optional frame images. " +
+            "Models cannot read video — the result carries a text summary; the video block is for the user's UI.",
+        promptSnippet: "`video_generate { prompt, model?, duration?, aspect_ratio?, resolution?, frame_images? }` — generate video content (mp4, saved to .generated_videos/)",
+        parameters: videoGenerateSchema,
+        async execute(_toolCallId, params, signal, onUpdate, ctx) {
+            const input = params;
+            const modelId = input.model ?? "google/veo-3.1";
+            const model = videoModel(modelId);
+            if (!model) {
+                return {
+                    content: [{ type: "text", text: `❌ Unknown OpenRouter video model: ${modelId}` }],
+                    details: { isError: true, model: modelId },
+                };
+            }
+            const context = {
+                input: [
+                    { type: "text", text: input.prompt },
+                    ...normalizeFrameImages(input.frame_images, ctx.cwd),
+                ],
+            };
+            // Route through the backend video proxy (POST /video/generation)
+            // using the machine JWT, mirroring image generation. Falls back to
+            // direct OpenRouter (submit + poll + download with the local
+            // OPENROUTER_API_KEY) when routing cannot be resolved or the proxy
+            // is unavailable.
+            const routing = await resolveBackendRouting();
+            const response = await generateVideos(model, context, {
+                ...input,
+                signal,
+                ...(routing ? { backendUrl: routing.backendUrl, apiKey: routing.machineJwt } : {}),
+                // Surface long-poll progress to the UI as partial tool results.
+                onProgress: (message) => {
+                    try {
+                        onUpdate?.({
+                            content: [{ type: "text", text: message }],
+                            details: { progress: message, model: model.id },
+                        });
+                    }
+                    catch {
+                        // Progress callbacks must never break generation.
+                    }
+                },
+            });
+            if (response.stopReason !== "stop") {
+                const message = response.errorMessage ?? "Video generation failed";
+                return {
+                    content: [{ type: "text", text: `❌ ${message}` }],
+                    details: { isError: true, model: model.id, stopReason: response.stopReason, errorMessage: message },
+                };
+            }
+            const video = response.output.find((item) => item.type === "video");
+            if (!video) {
+                return {
+                    content: [{ type: "text", text: "❌ Video generation returned no video" }],
+                    details: { isError: true, model: model.id },
+                };
+            }
+            // Local copy first — for videos that actually came back as bytes
+            // (direct OpenRouter fallback). Proxy-path videos have no `data`
+            // here, so nothing is written and the hosted url is reported.
+            const savedPath = saveGeneratedVideo(video, input.prompt, ctx.cwd);
+            const outcome = await hostGeneratedVideo(video, routing, signal);
+            // Swap base64 for the hosted URL when possible: the session keeps
+            // `{ url, videoId }` and avoids megabytes of base64. Videos that
+            // failed to upload keep their inline bytes (fallback, annotated in
+            // the result text).
+            const block = isHosted(outcome)
+                ? {
+                    type: "video",
+                    url: hostedUrl(outcome),
+                    videoId: hostedId(outcome),
+                    mimeType: video.mimeType,
+                }
+                : video;
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: resultText(response.model, response.usage?.credits, savedPath, outcome),
+                    },
+                    block,
+                ],
+                details: {
+                    isError: false,
+                    model: response.model,
+                    count: 1,
+                    usage: response.usage,
+                    savedPath,
+                    videoId: isHosted(outcome) ? hostedId(outcome) : undefined,
+                    url: isHosted(outcome) ? hostedUrl(outcome) : undefined,
+                },
+            };
+        },
+    };
+    ext.registerTool(tool);
+    process.stderr.write("[video-generation] Registered video_generate tool.\n");
+}

package/dist/generated/zeus/const.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"const.d.ts","sourceRoot":"","sources":["../../../src/generated/zeus/const.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAC,GAAG,CA0zB5C,CAAA;AAED,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAC,GAAG,CAumC1C,CAAA;AAED,eAAO,MAAM,GAAG;;;;CAIf,CAAA"}
+{"version":3,"file":"const.d.ts","sourceRoot":"","sources":["../../../src/generated/zeus/const.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAC,GAAG,CAm0B5C,CAAA;AAED,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAC,GAAG,CA4nC1C,CAAA;AAED,eAAO,MAAM,GAAG;;;;CAIf,CAAA"}

package/dist/generated/zeus/const.js

@@ -41,12 +41,17 @@ export const AllTypesProps = {
     RiskLevel: "enum",
     LanguageTag: "enum",
     PresetSource: "enum",
+    FollowUpMode: "enum",
+    PresetFollowUpMessageInput: {
+        mode: "FollowUpMode"
+    },
     CreatePresetInput: {
         category: "PresetCategory",
         costTier: "CostTier",
         riskLevel: "RiskLevel",
         source: "PresetSource",
         languageTags: "LanguageTag",
+        followUpMessages: "PresetFollowUpMessageInput",
         scope: "PresetScope"
     },
     UpdatePresetInput: {
@@ -54,6 +59,7 @@ export const AllTypesProps = {
         costTier: "CostTier",
         riskLevel: "RiskLevel",
         languageTags: "LanguageTag",
+        followUpMessages: "PresetFollowUpMessageInput",
         scope: "PresetScope"
     },
     AgentMode: "enum",
@@ -399,6 +405,9 @@ export const AllTypesProps = {
         adminCreateImageBaseModel: {
             input: "BaseModelCreateInput"
         },
+        adminCreateVideoBaseModel: {
+            input: "BaseModelCreateInput"
+        },
         submitLead: {
             input: "SubmitLeadInput"
         },
@@ -760,6 +769,10 @@ export const ReturnTypes = {
         createdAt: "DateTime",
         updatedAt: "DateTime"
     },
+    PresetFollowUpMessage: {
+        content: "String",
+        mode: "FollowUpMode"
+    },
     Preset: {
         id: "ID",
         teamId: "ID",
@@ -778,6 +791,7 @@ export const ReturnTypes = {
         goal: "String",
         intervalMinutes: "Int",
         languageTags: "LanguageTag",
+        followUpMessages: "PresetFollowUpMessage",
         scope: "PresetScope",
         createdAt: "DateTime",
         updatedAt: "DateTime"
@@ -1005,8 +1019,10 @@ export const ReturnTypes = {
         adminBuiltInModels: "UserModel",
         adminBaseModels: "BaseModel",
         adminImageBaseModels: "BaseModel",
+        adminVideoBaseModels: "BaseModel",
         adminOpenRouterCatalog: "OpenRouterModel",
         adminOpenRouterImageCatalog: "OpenRouterModel",
+        adminOpenRouterVideoCatalog: "OpenRouterVideoModel",
         adminOpenRouterModelProviders: "OpenRouterModelProvider",
         adminBillingPlans: "AdminBillingPlan",
         adminBillingPlanCount: "Int",
@@ -1141,6 +1157,7 @@ export const ReturnTypes = {
         adminUpdateBaseModel: "BaseModel",
         adminCreateBaseModel: "BaseModel",
         adminCreateImageBaseModel: "BaseModel",
+        adminCreateVideoBaseModel: "BaseModel",
         submitLead: "Lead",
         adminCreateLead: "Lead",
         adminUpdateLead: "Lead",
@@ -1530,6 +1547,7 @@ export const ReturnTypes = {
         creditCacheReadPer1M: "Int",
         creditCacheWritePer1M: "Int",
         creditPerImage: "Float",
+        creditPerVideoSecond: "Float",
         capabilities: "JSON",
         family: "String",
         enabled: "Boolean",
@@ -1561,6 +1579,18 @@ export const ReturnTypes = {
         outputModalities: "String",
         supportedParameters: "String"
     },
+    OpenRouterVideoModel: {
+        id: "String",
+        name: "String",
+        description: "String",
+        supportedDurations: "Int",
+        supportedResolutions: "String",
+        supportedAspectRatios: "String",
+        supportedSizes: "String",
+        supportedFrameImages: "String",
+        generateAudio: "Boolean",
+        pricingSkus: "JSON"
+    },
     ReleaseAsset: {
         name: "String",
         url: "String"