@mevdragon/vidfarm-devcli 0.1.0 → 0.2.1

package/dist/src/cli.js

@@ -1,8 +1,11 @@
 #!/usr/bin/env node
-import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { randomUUID } from "node:crypto";
 import path from "node:path";
 import { parseArgs } from "node:util";
 import dotenv from "dotenv";
+import { STARTER_TEMPLATE_FONT_OPTIONS, STARTER_TEMPLATE_TEXT_BACKGROUND_COLOR_OPTIONS } from "./lib/template-style-options.js";
+import { analyzeTemplateDna, hasTemplatePreviewMedia, stageTemplateDnaInputs, syncTemplateDnaModule } from "./lib/template-dna.js";
 void main().catch((error) => {
     console.error(error instanceof Error ? error.stack ?? error.message : String(error));
     process.exit(1);
@@ -17,6 +20,26 @@ async function main() {
         await runDevCommand(process.argv.slice(3));
         return;
     }
+    if (command === "validate-template") {
+        await runValidateTemplateCommand(process.argv.slice(3));
+        return;
+    }
+    if (command === "import-source") {
+        await runImportSourceCommand(process.argv.slice(3));
+        return;
+    }
+    if (command === "generate-template") {
+        await runGenerateTemplateCommand(process.argv.slice(3));
+        return;
+    }
+    if (command === "analyze-viral-dna") {
+        await runAnalyzeTemplateDnaCommand("viral", process.argv.slice(3));
+        return;
+    }
+    if (command === "analyze-visual-dna") {
+        await runAnalyzeTemplateDnaCommand("visual", process.argv.slice(3));
+        return;
+    }
     throw new Error(`Unknown command: ${command}`);
 }
 async function runDevCommand(argv) {
@@ -57,6 +80,7 @@ async function runDevCommand(argv) {
     process.env.STORAGE_DRIVER = "local";
     process.env.PUBLIC_BASE_URL = `http://127.0.0.1:${port}`;
     process.env.REMOTION_MODE = "local";
+    process.env.VIDFARM_ADMIN_EMAILS = process.env.VIDFARM_ADMIN_EMAILS || parsed.values.email;
     process.env.MOCK_PROVIDER_RESPONSES =
         parsed.values["mock-ai"] ? "true" : process.env.MOCK_PROVIDER_RESPONSES ?? (hasProviderEnvKey ? "false" : "true");
     const [{ startRuntime }, { database }, { config }, { createId }, crypto] = await Promise.all([
@@ -69,13 +93,16 @@ async function runDevCommand(argv) {
     const customer = database.getCustomerByEmail(parsed.values.email) ?? database.upsertCustomer({
         id: createId("cus"),
         email: parsed.values.email,
-        name: parsed.values.name
+        name: parsed.values.name,
+        isDeveloper: true,
+        isPaidPlan: true
     });
     const rawApiKey = parsed.values["api-key"] ?? `vf_local_${Math.random().toString(36).slice(2, 12)}`;
     database.insertApiKey({
         id: createId("api"),
         customerId: customer.id,
         keyHash: crypto.hashSecret(rawApiKey + config.API_KEY_SALT),
+        rawValue: rawApiKey,
         label: "Local CLI session"
     });
     seedEnvProviderKeys({
@@ -104,7 +131,7 @@ async function runDevCommand(argv) {
         sessionPath,
         mockProviders: config.mockProviders
     });
-    const runtime = startRuntime();
+    const runtime = await startRuntime();
     for (const signal of ["SIGINT", "SIGTERM"]) {
         process.on(signal, () => {
             runtime.shutdown();
@@ -112,6 +139,226 @@ async function runDevCommand(argv) {
         });
     }
 }
+async function runValidateTemplateCommand(argv) {
+    const parsed = parseArgs({
+        args: argv,
+        options: {
+            "template-id": { type: "string" },
+            "module-path": { type: "string" },
+            "skill-path": { type: "string" }
+        }
+    });
+    const [{ templateRegistry }, { TemplateCertificationService }, { loadTemplateFromModule }] = await Promise.all([
+        import("./registry.js"),
+        import("./services/template-certification.js"),
+        import("./services/template-loader.js")
+    ]);
+    const certification = new TemplateCertificationService();
+    let template;
+    let skillPath;
+    if (parsed.values["module-path"]) {
+        const modulePath = path.resolve(process.cwd(), parsed.values["module-path"]);
+        template = await loadTemplateFromModule(modulePath);
+        skillPath = path.resolve(process.cwd(), parsed.values["skill-path"] ?? "SKILL.md");
+    }
+    else {
+        await templateRegistry.ensureInitialized();
+        const templateId = parsed.values["template-id"];
+        if (!templateId) {
+            throw new Error("Provide --template-id or --module-path.");
+        }
+        template = templateRegistry.get(templateId);
+        if (!template) {
+            throw new Error(`Unknown template: ${templateId}`);
+        }
+        skillPath = template.skillPath;
+    }
+    if (!template || !skillPath) {
+        throw new Error("Template or skill path not found for certification.");
+    }
+    const report = await certification.certify({ template, skillPath });
+    console.log(JSON.stringify(report, null, 2));
+    if (!report.passed) {
+        process.exitCode = 1;
+    }
+}
+async function runImportSourceCommand(argv) {
+    const parsed = parseArgs({
+        args: argv,
+        options: {
+            "template-id": { type: "string" },
+            "slug-id": { type: "string" },
+            "repo-url": { type: "string" },
+            branch: { type: "string", default: "production" },
+            "template-module-path": { type: "string" },
+            "skill-path": { type: "string", default: "SKILL.md" },
+            "install-command": { type: "string", default: "npm install" },
+            "build-command": { type: "string", default: "npm run build" },
+            "commit-sha": { type: "string" },
+            activate: { type: "boolean", default: true }
+        }
+    });
+    const templateId = parsed.values["template-id"];
+    const slugId = parsed.values["slug-id"];
+    const repoUrl = parsed.values["repo-url"];
+    const templateModulePath = parsed.values["template-module-path"];
+    if (!templateId || !slugId || !repoUrl || !templateModulePath) {
+        throw new Error("import-source requires --template-id, --slug-id, --repo-url, and --template-module-path.");
+    }
+    const [{ TemplateSourceService }, { templateRegistry }] = await Promise.all([
+        import("./services/template-sources.js"),
+        import("./registry.js")
+    ]);
+    const sources = new TemplateSourceService();
+    const source = sources.registerSource({
+        templateId,
+        slugId,
+        repoUrl,
+        branch: parsed.values.branch,
+        templateModulePath,
+        skillPath: parsed.values["skill-path"],
+        installCommand: parsed.values["install-command"],
+        buildCommand: parsed.values["build-command"]
+    });
+    const release = await sources.importRelease({
+        sourceId: source.id,
+        commitSha: parsed.values["commit-sha"] ?? null
+    });
+    let activated = null;
+    if (parsed.values.activate) {
+        const result = await sources.activateRelease({ releaseId: release.id });
+        templateRegistry.registerRuntimeTemplate(result.template);
+        activated = result.release;
+    }
+    console.log(JSON.stringify({
+        source,
+        release,
+        activated_release: activated
+    }, null, 2));
+}
+async function runGenerateTemplateCommand(argv) {
+    const parsed = parseArgs({
+        args: argv,
+        options: {
+            "template-id": { type: "string" },
+            "slug-id": { type: "string" },
+            "template-dir": { type: "string" },
+            "project-name": { type: "string" },
+            "github-repo": { type: "string" },
+            "site-name": { type: "string" },
+            "link-to-original": { type: "string" },
+            "source-notes-path": { type: "string" },
+            "source-preview-dir": { type: "string" },
+            "dna-env-file": { type: "string", default: ".env" },
+            "dna-model": { type: "string" },
+            "skip-dna-analysis": { type: "boolean", default: false },
+            branch: { type: "string", default: "production" },
+            force: { type: "boolean", default: false }
+        }
+    });
+    const templateId = parsed.values["template-id"] ?? randomUUID();
+    const slugId = parsed.values["slug-id"];
+    const templateDir = parsed.values["template-dir"];
+    if (!slugId || !templateDir) {
+        throw new Error("generate-template requires --slug-id and --template-dir. --template-id is optional and defaults to a new UUIDv4.");
+    }
+    const root = process.cwd();
+    const starterDir = resolveStarterTemplateDir();
+    const destinationDir = path.resolve(root, templateDir);
+    if (!existsSync(starterDir)) {
+        throw new Error(`Starter template not found: ${starterDir}`);
+    }
+    if (existsSync(destinationDir)) {
+        if (!parsed.values.force) {
+            throw new Error(`Destination already exists: ${destinationDir}`);
+        }
+        rmSync(destinationDir, { recursive: true, force: true });
+    }
+    cpSync(starterDir, destinationDir, {
+        recursive: true,
+        filter: (entry) => {
+            const name = path.basename(entry);
+            return ![".git", "node_modules", "dist", "package-lock.json"].includes(name);
+        }
+    });
+    const destinationFolderName = path.basename(destinationDir);
+    const projectName = parsed.values["project-name"] ?? toProjectName(destinationFolderName);
+    const siteName = parsed.values["site-name"] ?? projectName.replace(/_/g, "-");
+    const githubRepo = parsed.values["github-repo"] ?? `mevdragon/${projectName}`;
+    rewriteTemplateStarter(destinationDir, [
+        ["mevdragon/vidfarm_template_0000", githubRepo],
+        ["vidfarm_template_0000", projectName],
+        ["vidfarm-template-0000", siteName],
+        ["template_0000", slugId],
+        ["template-0000", slugId.replace(/_/g, "-")],
+        ["4c7a7e1a-7f35-4f30-9f86-9c8a63c7f2db", templateId],
+        ['"production"', JSON.stringify(parsed.values.branch)]
+    ]);
+    const stagedInputs = stageTemplateDnaInputs({
+        templateDir: destinationDir,
+        sourceNotesPath: parsed.values["source-notes-path"],
+        sourcePreviewDir: parsed.values["source-preview-dir"]
+    });
+    const moduleSync = syncTemplateDnaModule({
+        templateDir: destinationDir,
+        linkToOriginal: parsed.values["link-to-original"]
+    });
+    const dnaRuns = [];
+    if (!parsed.values["skip-dna-analysis"] && hasTemplatePreviewMedia(destinationDir)) {
+        dnaRuns.push(await analyzeTemplateDna({
+            mode: "viral",
+            templateDir: destinationDir,
+            envFile: parsed.values["dna-env-file"],
+            model: parsed.values["dna-model"],
+            linkToOriginal: parsed.values["link-to-original"]
+        }));
+        dnaRuns.push(await analyzeTemplateDna({
+            mode: "visual",
+            templateDir: destinationDir,
+            envFile: parsed.values["dna-env-file"],
+            model: parsed.values["dna-model"],
+            linkToOriginal: parsed.values["link-to-original"]
+        }));
+    }
+    console.log(JSON.stringify({
+        template_id: templateId,
+        slug_id: slugId,
+        template_dir: destinationDir,
+        project_name: projectName,
+        github_repo: githubRepo,
+        site_name: siteName,
+        research: stagedInputs,
+        template_dna_module: moduleSync,
+        dna_analysis_runs: dnaRuns
+    }, null, 2));
+}
+async function runAnalyzeTemplateDnaCommand(mode, argv) {
+    const parsed = parseArgs({
+        args: argv,
+        options: {
+            "template-dir": { type: "string", default: "." },
+            "env-file": { type: "string", default: ".env" },
+            model: { type: "string" },
+            "notes-path": { type: "string" },
+            "preview-dir": { type: "string" },
+            "output-path": { type: "string" },
+            "link-to-original": { type: "string" },
+            "sync-template-module": { type: "boolean", default: true }
+        }
+    });
+    const result = await analyzeTemplateDna({
+        mode,
+        templateDir: parsed.values["template-dir"],
+        envFile: parsed.values["env-file"],
+        model: parsed.values.model,
+        notesPath: parsed.values["notes-path"],
+        previewDir: parsed.values["preview-dir"],
+        outputPath: parsed.values["output-path"],
+        linkToOriginal: parsed.values["link-to-original"],
+        syncTemplateModule: parsed.values["sync-template-module"]
+    });
+    console.log(JSON.stringify(result, null, 2));
+}
 async function runSessionCommand(argv) {
     const parsed = parseArgs({
         args: argv,
@@ -124,17 +371,22 @@ async function runSessionCommand(argv) {
     console.log(JSON.stringify({
         base_url: session.baseUrl,
         email: session.email,
+        starter_style_options: {
+            fonts: STARTER_TEMPLATE_FONT_OPTIONS,
+            text_background_colors: STARTER_TEMPLATE_TEXT_BACKGROUND_COLOR_OPTIONS,
+            starter_template_file: "templates/template_0000/src/style-options.ts"
+        },
         headers: {
             "vidfarm-user-id": session.customerId,
             "vidfarm-api-key": session.apiKey,
             "content-type": "application/json"
         },
         curl_example: [
-            `curl -X POST ${session.baseUrl}/templates/demo-template/operations/generate \\`,
+            `curl -X POST ${session.baseUrl}/api/v1/templates/template_0000/operations/create_slideshow \\`,
             `  -H 'vidfarm-user-id: ${session.customerId}' \\`,
             `  -H 'vidfarm-api-key: ${session.apiKey}' \\`,
             "  -H 'content-type: application/json' \\",
-            "  -d '{\"tracer\":\"local-test\",\"payload\":{\"slides\":[[\"a cinematic founder at a desk\",\"Exact text on slide one\"]],\"secondsPerSlide\":4}}'"
+            "  -d '{\"tracer\":\"local-test\",\"payload\":{\"slides\":[[\"a cinematic founder at a desk\",\"Exact text on slide one\",2400]],\"meta_details_prompt\":\"Keep it native, casual, and curiosity-driven.\"}}'"
         ].join("\n")
     }, null, 2));
 }
@@ -185,3 +437,30 @@ function printStartupBanner(input) {
     console.log("  vidfarm session");
     console.log("");
 }
+function rewriteTemplateStarter(root, replacements) {
+    for (const entry of readdirSync(root, { withFileTypes: true })) {
+        const target = path.join(root, entry.name);
+        if (entry.name === ".git" || entry.name === "node_modules" || entry.name === "dist") {
+            continue;
+        }
+        if (entry.isDirectory()) {
+            rewriteTemplateStarter(target, replacements);
+            continue;
+        }
+        const original = readFileSync(target, "utf8");
+        const next = replacements.reduce((value, [from, to]) => value.split(from).join(to), original);
+        if (next !== original) {
+            writeFileSync(target, next);
+        }
+    }
+}
+function toProjectName(folderName) {
+    return folderName
+        .replace(/[^a-zA-Z0-9_-]+/g, "_")
+        .replace(/-+/g, "_")
+        .replace(/^_+|_+$/g, "")
+        .toLowerCase();
+}
+function resolveStarterTemplateDir() {
+    return path.resolve(import.meta.dirname, "..", "..", "templates", "template_0000");
+}

package/dist/src/config.js

@@ -11,42 +11,62 @@ const schema = z.object({
     ENCRYPTION_SECRET: z.string().default("development-encryption-secret-change-me"),
     API_KEY_SALT: z.string().default("development-api-key-salt"),
     WORKER_POLL_MS: z.coerce.number().default(1500),
-    WORKER_BATCH_SIZE: z.coerce.number().default(4),
+    WORKER_BATCH_SIZE: z.coerce.number().default(2),
+    WORKER_MAX_CONCURRENT_JOBS: z.coerce.number().default(1),
     WEBHOOK_SECRET: z.string().default("development-webhook-secret"),
     DEFAULT_JOB_DELAY_SECONDS: z.coerce.number().default(20),
+    MAX_PENDING_JOBS_GLOBAL: z.coerce.number().default(0),
+    MAX_PENDING_JOBS_PER_CUSTOMER: z.coerce.number().default(0),
     STORAGE_DRIVER: z.enum(["local", "s3"]).default("local"),
     AWS_REGION: z.string().default("us-east-1"),
     AWS_S3_BUCKET: z.string().default(""),
+    AWS_S3_PUBLIC_READ: z.string().default("false"),
     AWS_S3_ENDPOINT: z.string().optional(),
     AWS_ACCESS_KEY_ID: z.string().optional(),
     AWS_SECRET_ACCESS_KEY: z.string().optional(),
     PUBLIC_BASE_URL: z.string().optional(),
     RESEND_API_KEY: z.string().optional(),
-    RESEND_FROM_EMAIL: z.string().default("noreply@example.com"),
+    RESEND_FROM_EMAIL: z.string().default("vidfarm@fwd.zoomgtm.com"),
     OPENAI_API_KEY: z.string().optional(),
     OPENROUTER_API_KEY: z.string().optional(),
     GEMINI_API_KEY: z.string().optional(),
     PERPLEXITY_API_KEY: z.string().optional(),
+    SUPERAGENCY_KEY: z.string().optional(),
     REMOTION_REGION: z.string().default("us-east-1"),
     REMOTION_BUCKET_NAME: z.string().optional(),
     REMOTION_SITE_NAME: z.string().optional(),
     REMOTION_FUNCTION_NAME: z.string().optional(),
     REMOTION_SERVE_URL: z.string().optional(),
-    REMOTION_COMPOSITION_ID: z.string().default("demo-template"),
+    REMOTION_COMPOSITION_ID: z.string().default("template-0000"),
     REMOTION_AWS_ACCESS_KEY_ID: z.string().optional(),
     REMOTION_AWS_SECRET_ACCESS_KEY: z.string().optional(),
     REMOTION_MODE: z.enum(["auto", "mock", "local", "lambda"]).default("auto"),
-    MOCK_PROVIDER_RESPONSES: z.string().default("true")
+    MOCK_PROVIDER_RESPONSES: z.string().default("true"),
+    VIDFARM_ADMIN_EMAILS: z.string().default(""),
+    VIDFARM_DEVELOPER_EMAILS: z.string().default(""),
+    TEMPLATE_SOURCE_ROOT: z.string().default("./data/template-sources")
 });
 const parsed = schema.parse(process.env);
 const dataDir = path.resolve(parsed.VIDFARM_DATA_DIR);
 mkdirSync(dataDir, { recursive: true });
+const templateSourceRoot = path.resolve(parsed.TEMPLATE_SOURCE_ROOT);
+mkdirSync(templateSourceRoot, { recursive: true });
 const publicBaseUrl = parsed.PUBLIC_BASE_URL || `http://localhost:${parsed.PORT}`;
 export const config = {
     ...parsed,
     VIDFARM_DB_PATH: path.resolve(parsed.VIDFARM_DB_PATH),
     VIDFARM_DATA_DIR: dataDir,
+    TEMPLATE_SOURCE_ROOT: templateSourceRoot,
     PUBLIC_BASE_URL: publicBaseUrl,
     isProduction: parsed.NODE_ENV === "production",
-    mockProviders: parsed.MOCK_PROVIDER_RESPONSES !== "false"
+    mockProviders: parsed.MOCK_PROVIDER_RESPONSES !== "false",
+    s3PublicRead: parsed.AWS_S3_PUBLIC_READ === "true",
+    adminEmails: parsed.VIDFARM_ADMIN_EMAILS
+        .split(",")
+        .map((value) => value.trim().toLowerCase())
+        .filter(Boolean),
+    developerEmails: parsed.VIDFARM_DEVELOPER_EMAILS
+        .split(",")
+        .map((value) => value.trim().toLowerCase())
+        .filter(Boolean)
 };

package/dist/src/context.js

@@ -3,7 +3,7 @@ import { database } from "./db.js";
 import { createId } from "./lib/ids.js";
 export function createTemplateJobContext(input) {
     const templateConfig = database.getTemplateConfig(input.customer.id, input.template.id);
-    const prefix = `templates/${input.template.id}/users/${input.customer.id}/jobs/${input.job.id}`;
+    const prefix = input.storage.templateJobPrefix(input.template.id, input.customer.id, input.job.id);
     return {
         env: config.isProduction ? "production" : "development",
         customer: input.customer,