@mevdragon/vidfarm-devcli 0.2.1 → 0.2.3

package/templates/template_0000/src/template.ts

@@ -39,6 +39,32 @@ const NATIVE_UI_TEXT_ZONE = {
   maxCenterYPercent: 0.58,
 } as const;
 const TEXT_BACKGROUND_NONE = "none" as const;
+// Documentation-only dependency map for humans and agents.
+// The current template standard has no first-class dependencies field, so
+// provider/model requirements must be declared in source and SKILL.md.
+const TEMPLATE_PROVIDER_REQUIREMENTS = {
+  image: [
+    { provider: "openai", models: ["gpt-image-1", "gpt-image-2"], strict: false },
+    { provider: "gemini", models: ["gemini-3.1-flash-image-preview", "gemini-2.5-flash-image"], strict: false },
+    { provider: "openrouter", models: ["bytedance/seedance-2.0", "bytedance-seed/seedream-4.5"], strict: false, planned: true },
+  ],
+  text: [
+    { provider: "openai", models: ["gpt-5.4"], strict: false },
+    { provider: "gemini", models: ["gemini-3.1-flash-lite", "gemini-2.5-flash-lite"], strict: false },
+    { provider: "openrouter", models: ["qwen/qwen3.6-flash"], strict: false },
+  ],
+  layout_analysis: [
+    { provider: "openai", models: ["gpt-5.4"], strict: false },
+    { provider: "gemini", models: ["gemini-3.1-flash-lite", "gemini-2.5-flash-lite"], strict: false },
+    { provider: "openrouter", models: ["qwen/qwen3.6-flash"], strict: false },
+  ],
+  video: [
+    { provider: "openai", models: ["sora-2"], strict: false, planned: true },
+    { provider: "gemini", models: ["veo-3.0-generate-001"], strict: false, planned: true },
+  ],
+} as const;
+
+void TEMPLATE_PROVIDER_REQUIREMENTS;
 
 const legacySlideInputSchema = z.union([
   z.tuple([z.string().min(3), z.string().min(1)]),
@@ -81,8 +107,21 @@ type Layout = {
   justification: string;
 };
 
+type SlideSourceType = "ai_prompt" | "image_source";
+
+type NormalizedSlideInput = {
+  sourceType: SlideSourceType;
+  sourceValue: string;
+  imagePrompt: string;
+  imagePromptAttachments: string[];
+  overlayText: string;
+  durationMs: number;
+};
+
 type SlideOutput = {
   index: number;
+  sourceType: SlideSourceType;
+  sourceValue: string;
   imagePrompt: string;
   imagePromptAttachments: string[];
   overlayText: string;
@@ -213,38 +252,61 @@ export const template0000Definition = defineTemplate({
       const payload = createSlideshowInputSchema.parse(input);
       ctx.logger.progress(0.04, "Starting template_0000 slideshow image stage");
       const provider = parseImageProvider(ctx.templateConfig.defaultProvider);
+      const configuredTextModel =
+        typeof ctx.templateConfig.textModel === "string"
+          ? ctx.templateConfig.textModel
+          : null;
+      const configuredImageModel =
+        typeof ctx.templateConfig.imageModel === "string"
+          ? ctx.templateConfig.imageModel
+          : null;
       const textModel = String(
-        ctx.templateConfig.textModel ?? defaultTextModelForProvider(provider),
+        configuredTextModel ?? defaultTextModelForProvider(provider),
       );
       const imageModel = String(
-        ctx.templateConfig.imageModel ?? defaultImageModelForProvider(provider),
+        configuredImageModel ?? defaultImageModelForProvider(provider),
       );
       const textStyle = resolveTextStyleSpec(ctx.templateConfig);
 
       const slides: SlideOutput[] = [];
       for (const [index, rawSlide] of payload.slides.entries()) {
-        const { imagePrompt, imagePromptAttachments, overlayText, durationMs } =
-          normalizeSlideInput(rawSlide);
-        ctx.logger.progress(
-          0.08 + (index / payload.slides.length) * 0.42,
-          `Generating slide ${index + 1} image`,
-        );
-        const image = await generatePortraitCandidateImage(ctx, {
-          provider,
-          imageModel,
+        const {
+          sourceType,
+          sourceValue,
           imagePrompt,
           imagePromptAttachments,
           overlayText,
-        });
-        await ctx.billing.record({
-          type: "ai_generation",
-          costUsd: 0.04,
-          metadata: {
-            stage: "image_generation",
-            slideIndex: index,
-            model: imageModel,
-          },
-        });
+          durationMs,
+        } = normalizeSlideInput(rawSlide);
+        ctx.logger.progress(
+          0.08 + (index / payload.slides.length) * 0.42,
+          sourceType === "image_source"
+            ? `Loading slide ${index + 1} source image`
+            : `Generating slide ${index + 1} image`,
+        );
+        const image =
+          sourceType === "image_source"
+            ? await loadExistingSlideImage(sourceValue)
+            : await generatePortraitCandidateImage(ctx, {
+                provider,
+                imageModel,
+                configuredProvider: provider,
+                configuredImageModel,
+                imagePrompt,
+                imagePromptAttachments,
+                overlayText,
+              });
+        if (sourceType === "ai_prompt") {
+          await ctx.billing.record({
+            type: "ai_generation",
+            costUsd: 0.04,
+            metadata: {
+              stage: "image_generation",
+              slideIndex: index,
+              model: imageModel,
+            },
+          });
+        }
 
         ctx.logger.progress(
           0.13 + (index / payload.slides.length) * 0.3,
@@ -312,6 +374,8 @@ export const template0000Definition = defineTemplate({
 
         slides.push({
           index,
+          sourceType,
+          sourceValue,
           imagePrompt,
           imagePromptAttachments,
           overlayText,
@@ -328,6 +392,8 @@ export const template0000Definition = defineTemplate({
       const metaDetails = await generateMetaDetails(ctx, {
         provider,
         textModel,
+        configuredProvider: provider,
+        configuredTextModel,
         slides,
         metaDetailsPrompt: payload.meta_details_prompt,
       });
@@ -381,6 +447,8 @@ export const template0000Definition = defineTemplate({
             meta_details_prompt: payload.meta_details_prompt ?? null,
             slides: slides.map((slide) => ({
               index: slide.index,
+              sourceType: slide.sourceType,
+              sourceValue: slide.sourceValue,
               imagePrompt: slide.imagePrompt,
               imagePromptAttachments: slide.imagePromptAttachments,
               overlayText: slide.overlayText,
@@ -499,6 +567,8 @@ async function generatePortraitCandidateImage(
   input: {
     provider: ImageProvider;
     imageModel: string;
+    configuredProvider: ImageProvider;
+    configuredImageModel: string | null;
     imagePrompt: string;
     imagePromptAttachments: string[];
     overlayText: string;
@@ -553,31 +623,48 @@ async function generatePortraitCandidateImage(
     prompt: string;
     score: number;
   } | null = null;
+  let lastError: Error | null = null;
 
-  for (const prompt of attempts) {
-    const image = await ctx.providers.generateImage({
-      provider: input.provider,
-      model: input.imageModel,
-      prompt,
-      promptAttachments: input.imagePromptAttachments,
-      size: sourceImageSizeForProvider(input.provider),
-      aspectRatio: "9:16",
-      imageSize: sourceImageOutputSizeForProvider(
-        input.provider,
-        input.imageModel,
-      ),
-    });
-    const score = await portraitScore(image.bytes);
-    if (!best || score < best.score) {
-      best = { ...image, prompt, score };
-    }
-    if (score <= 0.16) {
-      break;
+  for (const provider of providerFailoverOrder(input.provider)) {
+    const imageModel = resolveImageModelForAttempt(
+      provider,
+      input.configuredProvider,
+      input.configuredImageModel,
+    );
+    try {
+      for (const prompt of attempts) {
+        const image = await ctx.providers.generateImage({
+          provider,
+          model: imageModel,
+          prompt,
+          promptAttachments: input.imagePromptAttachments,
+          size: sourceImageSizeForProvider(provider),
+          aspectRatio: "9:16",
+          imageSize: sourceImageOutputSizeForProvider(provider, imageModel),
+        });
+        const score = await portraitScore(image.bytes);
+        if (!best || score < best.score) {
+          best = { ...image, prompt, score };
+        }
+        if (score <= 0.16) {
+          return best;
+        }
+      }
+      if (best) {
+        return best;
+      }
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      ctx.logger.warn("Image generation provider failed, trying next provider", {
+        provider,
+        model: imageModel,
+        message: lastError.message,
+      });
     }
   }
 
   if (!best) {
-    throw new Error("No image candidate was generated.");
+    throw lastError ?? new Error("No image candidate was generated.");
   }
 
   return best;
@@ -611,7 +698,7 @@ function isImageProvider(value: unknown): value is ImageProvider {
 
 function sourceImageSizeForProvider(provider: ImageProvider) {
   if (provider === "openai") {
-    return "1024x1792";
+    return "1024x1536";
   }
   return "1080x1920";
 }
@@ -623,7 +710,7 @@ function defaultImageModelForProvider(provider: ImageProvider) {
   if (provider === "openrouter") {
     return "bytedance-seed/seedream-4.5";
   }
-  return "gemini-3.1-flash-image-preview";
+  return "gemini-2.5-flash-image";
 }
 
 function defaultTextModelForProvider(provider: ImageProvider) {
@@ -646,15 +733,41 @@ function sourceImageOutputSizeForProvider(
   if (provider !== "gemini") {
     return undefined;
   }
-  if (
-    model === "gemini-3.1-flash-image-preview" ||
-    model === "gemini-3-pro-image-preview"
-  ) {
+  if (model === "gemini-3-pro-image-preview") {
     return "1K" as const;
   }
   return undefined;
 }
 
+function providerFailoverOrder(primary: ImageProvider): ImageProvider[] {
+  return [
+    primary,
+    ...supportedImageProviders.filter((provider) => provider !== primary),
+  ];
+}
+
+function resolveImageModelForAttempt(
+  provider: ImageProvider,
+  configuredProvider: ImageProvider,
+  configuredImageModel: string | null,
+) {
+  if (provider === configuredProvider && configuredImageModel) {
+    return configuredImageModel;
+  }
+  return defaultImageModelForProvider(provider);
+}
+
+function resolveTextModelForAttempt(
+  provider: ImageProvider,
+  configuredProvider: ImageProvider,
+  configuredTextModel: string | null,
+) {
+  if (provider === configuredProvider && configuredTextModel) {
+    return configuredTextModel;
+  }
+  return defaultTextModelForProvider(provider);
+}
+
 function resolveTextStyleSpec(config: Record<string, unknown>): TextStyleSpec {
   const fontId = isTemplateFontId(config.captionFont)
     ? config.captionFont
@@ -759,6 +872,8 @@ const slideshowManifestSchema = z.object({
   slides: z.array(
     z.object({
       index: z.number(),
+      sourceType: z.enum(["ai_prompt", "image_source"]),
+      sourceValue: z.string(),
       imagePrompt: z.string(),
       imagePromptAttachments: z.array(z.string().url()),
       overlayText: z.string(),
@@ -779,41 +894,196 @@ const slideshowManifestSchema = z.object({
   ),
 });
 
-function normalizeSlideInput(input: z.infer<typeof slideInputSchema>) {
+function normalizeSlideInput(
+  input: z.infer<typeof slideInputSchema>,
+): NormalizedSlideInput {
   if (Array.isArray(input)) {
-    return {
-      imagePrompt: input[0],
+    return normalizeSlideFields({
+      primarySource: input[0],
       imagePromptAttachments: [],
       overlayText: input[1],
       durationMs: input[2] ?? 4000,
-    };
+    });
   }
 
-  return {
-    imagePrompt: input.image_prompt,
+  return normalizeSlideFields({
+    primarySource: input.image_prompt,
     imagePromptAttachments: input.image_prompt_attachments,
     overlayText: input.caption,
     durationMs: input.duration_ms,
+  });
+}
+
+function normalizeSlideFields(input: {
+  primarySource: string;
+  imagePromptAttachments: string[];
+  overlayText: string;
+  durationMs: number;
+}): NormalizedSlideInput {
+  const sourceValue = input.primarySource.trim();
+  const existingImageSource = resolveExistingImageSource(sourceValue);
+  return {
+    sourceType: existingImageSource ? "image_source" : "ai_prompt",
+    sourceValue: existingImageSource ?? sourceValue,
+    imagePrompt: sourceValue,
+    imagePromptAttachments: input.imagePromptAttachments,
+    overlayText: input.overlayText,
+    durationMs: input.durationMs,
   };
 }
 
+async function loadExistingSlideImage(sourceValue: string) {
+  const resolved = resolveExistingImageSource(sourceValue);
+  if (!resolved) {
+    throw new Error(`Slide source is not a supported image URL or file path: ${sourceValue}`);
+  }
+
+  const bytes =
+    isLocalFileSource(resolved)
+      ? readFileSync(resolveLocalFileSourcePath(resolved))
+      : await fetchExternalImageBytes(resolved);
+  const contentType = await detectImageContentType(bytes);
+
+  return {
+    bytes,
+    contentType,
+    revisedPrompt: null,
+    prompt: sourceValue,
+  };
+}
+
+function resolveExistingImageSource(value: string) {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    return null;
+  }
+
+  const parsedUrl = parseUrl(trimmed);
+  if (
+    parsedUrl &&
+    ["http:", "https:", "file:", "data:"].includes(parsedUrl.protocol)
+  ) {
+    return trimmed;
+  }
+
+  const localPath = resolveLocalPathIfPresent(trimmed);
+  if (localPath) {
+    return localPath;
+  }
+
+  return null;
+}
+
+function parseUrl(value: string) {
+  try {
+    return new URL(value);
+  } catch {
+    return null;
+  }
+}
+
+function resolveLocalPathIfPresent(value: string) {
+  const expandedPath = value.startsWith("~/")
+    ? path.join(process.env.HOME ?? "", value.slice(2))
+    : value;
+  const candidate = path.resolve(expandedPath);
+  return existsSync(candidate) ? candidate : null;
+}
+
+function isLocalFileSource(sourceValue: string) {
+  const parsedUrl = parseUrl(sourceValue);
+  return !parsedUrl || parsedUrl.protocol === "file:";
+}
+
+function resolveLocalFileSourcePath(sourceValue: string) {
+  const parsedUrl = parseUrl(sourceValue);
+  return parsedUrl?.protocol === "file:"
+    ? fileURLToPath(parsedUrl)
+    : path.resolve(sourceValue);
+}
+
+async function fetchExternalImageBytes(sourceValue: string) {
+  const response = await fetch(sourceValue);
+  if (!response.ok) {
+    throw new Error(`Could not fetch slide source image: ${response.status} ${response.statusText}`);
+  }
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+async function detectImageContentType(bytes: Uint8Array) {
+  try {
+    const metadata = await sharp(bytes).metadata();
+    if (metadata.format) {
+      return contentTypeForSharpFormat(metadata.format);
+    }
+  } catch (error) {
+    throw new Error(
+      `Slide source could not be decoded as an image: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+  return "image/png";
+}
+
+function contentTypeForSharpFormat(format: string) {
+  switch (format) {
+    case "jpeg":
+      return "image/jpeg";
+    case "png":
+      return "image/png";
+    case "webp":
+      return "image/webp";
+    case "gif":
+      return "image/gif";
+    case "avif":
+      return "image/avif";
+    case "tiff":
+      return "image/tiff";
+    case "svg":
+      return "image/svg+xml";
+    case "heif":
+      return "image/heif";
+    default:
+      return `image/${format}`;
+  }
+}
+
 async function generateMetaDetails(
   ctx: TemplateJobContext,
   input: {
     provider: ImageProvider;
     textModel: string;
+    configuredProvider: ImageProvider;
+    configuredTextModel: string | null;
     slides: SlideOutput[];
     metaDetailsPrompt?: string;
   },
 ): Promise<MetaDetails> {
   const prompt = buildMetaDetailsPrompt(input.slides, input.metaDetailsPrompt);
-  const response = await ctx.providers.generateText({
-    provider: input.provider,
-    model: input.textModel,
-    prompt,
-    temperature: 0.7,
-  });
-  return parseMetaDetailsResponse(response.text, input.slides);
+  let lastError: Error | null = null;
+  for (const provider of providerFailoverOrder(input.provider)) {
+    const textModel = resolveTextModelForAttempt(
+      provider,
+      input.configuredProvider,
+      input.configuredTextModel,
+    );
+    try {
+      const response = await ctx.providers.generateText({
+        provider,
+        model: textModel,
+        prompt,
+        temperature: 0.7,
+      });
+      return parseMetaDetailsResponse(response.text, input.slides);
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      ctx.logger.warn("Metadata provider failed, trying next provider", {
+        provider,
+        model: textModel,
+        message: lastError.message,
+      });
+    }
+  }
+  throw lastError ?? new Error("No metadata provider succeeded.");
 }
 
 function buildMetaDetailsPrompt(
@@ -1224,7 +1494,12 @@ async function portraitScore(input: Uint8Array) {
     thumbnailWidth,
     thumbnailHeight,
   );
-  return Math.abs(aspect - 9 / 16) + edgeContrast;
+  const edgeFlatnessPenalty = edgeBandFlatnessPenalty(
+    sample,
+    thumbnailWidth,
+    thumbnailHeight,
+  );
+  return Math.abs(aspect - 9 / 16) + edgeContrast + edgeFlatnessPenalty;
 }
 
 async function chooseLayoutFromImage(
@@ -1541,6 +1816,26 @@ function edgeBandContrast(sample: Buffer, width: number, height: number) {
   return (top + bottom) / 220;
 }
 
+function edgeBandFlatnessPenalty(sample: Buffer, width: number, height: number) {
+  const bandHeight = Math.max(8, Math.floor(height * 0.16));
+  const centerStart = Math.max(0, Math.floor(height * 0.34));
+  const centerEnd = Math.min(height, Math.ceil(height * 0.66));
+  const top = bandActivity(sample, width, height, 0, bandHeight);
+  const bottom = bandActivity(sample, width, height, height - bandHeight, height);
+  const center = bandActivity(sample, width, height, centerStart, centerEnd);
+  const edgeAverage = (top + bottom) / 2;
+  if (center <= 0) {
+    return 0;
+  }
+
+  const ratio = edgeAverage / center;
+  if (ratio >= 0.72) {
+    return 0;
+  }
+
+  return (0.72 - ratio) * 3.4;
+}
+
 function bandActivity(
   sample: Buffer,
   width: number,

package/templates/template_0000/template.config.json

@@ -2,21 +2,16 @@
   "template_id": "4c7a7e1a-7f35-4f30-9f86-9c8a63c7f2db",
   "slug_id": "template_0000",
   "project_name": "vidfarm_template_0000",
-  "github_repo": "mevdragon/vidfarm_template_0000",
-  "source_branch": "production",
+  "github_repo": "your-org/your-template-repo",
+  "source_branch": "main",
   "skill_path": "SKILL.md",
-  "template_module_path": "src/template.js",
-  "release_controls": {
-    "shared_remotion_publish": "admin_only",
-    "platform_activation": "admin_only",
-    "production_docker_promotion": "admin_only"
-  },
+  "template_module_path": "src/template.ts",
   "remotion": {
-    "region": "us-east-1",
-    "function_name": "remotion-render-4-0-355-mem2048mb-disk2048mb-180sec",
-    "bucket_name": "remotionlambda-useast1-ujg7c0h43q",
+    "region": "local",
+    "function_name": "",
+    "bucket_name": "",
     "site_name": "vidfarm-template-0000",
-    "serve_url": "https://remotionlambda-useast1-ujg7c0h43q.s3.us-east-1.amazonaws.com/sites/vidfarm-template-0000/index.html",
+    "serve_url": "",
     "composition_id": "template-0000",
     "entry_point": "src/remotion/index.tsx",
     "props_file": "composition.json",