@mevdragon/vidfarm-devcli 0.2.7 → 0.2.9

package/dist/src/worker.js

@@ -1,10 +1,12 @@
 import { config } from "./config.js";
 import { createTemplateJobContext } from "./context.js";
 import { database } from "./db.js";
+import { createPrimitiveJobContext } from "./primitive-context.js";
+import { primitiveRegistry } from "./primitive-registry.js";
 import { templateRegistry } from "./registry.js";
 import { BillingService } from "./services/billing.js";
 import { JobsService } from "./services/jobs.js";
-import { ProviderAuthError, ProviderKeyUnavailableError, ProviderRateLimitError, ProviderService } from "./services/providers.js";
+import { ProviderAuthError, ProviderKeyUnavailableError, ProviderLeaseTimeoutError, ProviderRateLimitError, ProviderService } from "./services/providers.js";
 import { RemotionService } from "./services/remotion.js";
 import { StorageService } from "./services/storage.js";
 import { WebhookService } from "./services/webhooks.js";
@@ -36,7 +38,7 @@ export class Worker {
         try {
             const availableSlots = Math.max(0, config.WORKER_MAX_CONCURRENT_JOBS - this.activeJobs);
             if (availableSlots > 0) {
-                const runnable = database.listRunnableJobs(Math.min(config.WORKER_BATCH_SIZE, availableSlots));
+                const runnable = this.selectRunnableJobs(Math.min(config.WORKER_BATCH_SIZE, availableSlots));
                 const executions = runnable.map(async (job) => {
                     this.activeJobs += 1;
                     try {
@@ -62,39 +64,61 @@ export class Worker {
         }
         const customer = database.getCustomerById(job.customerId);
         const template = templateRegistry.get(job.templateId);
-        if (!customer || !template) {
-            this.jobs.failJob(job.id, { message: "Missing customer or template." });
+        const primitive = template ? null : primitiveRegistry.get(job.templateId);
+        if (!customer || (!template && !primitive)) {
+            this.jobs.failJob(job.id, { message: "Missing customer, template, or primitive." });
             return;
         }
-        const workflow = template.jobs[job.workflowName];
-        if (!workflow) {
-            this.jobs.failJob(job.id, { message: `Unknown workflow ${job.workflowName}` });
-            return;
+        if (template) {
+            const wallet = await this.billing.getWalletSummary(customer.id);
+            if (wallet.balanceUsd <= 0) {
+                this.jobs.failJob(job.id, {
+                    message: "Wallet balance must be greater than $0.00 USD before starting a template job.",
+                    type: "insufficient_wallet_balance"
+                });
+                await this.emitWebhookIfNeeded("job.failed", job.id);
+                return;
+            }
         }
         this.jobs.markRunning(job.id);
         await this.emitWebhookIfNeeded("job.running", job.id);
-        const ctx = createTemplateJobContext({
-            customer,
-            job,
-            template,
-            workerId: this.workerId,
-            storage: this.storage,
-            jobs: this.jobs,
-            billing: this.billing,
-            providers: this.providers,
-            remotion: this.remotion
-        });
         try {
-            const result = await workflow(ctx, job.payload);
+            const result = template
+                ? await runTemplateWorkflow({
+                    customer,
+                    job,
+                    template,
+                    workerId: this.workerId,
+                    storage: this.storage,
+                    jobs: this.jobs,
+                    billing: this.billing,
+                    providers: this.providers,
+                    remotion: this.remotion
+                })
+                : await runPrimitiveWorkflow({
+                    customer,
+                    job,
+                    primitive: primitive,
+                    workerId: this.workerId,
+                    storage: this.storage,
+                    billing: this.billing,
+                    providers: this.providers,
+                    remotion: this.remotion
+                });
             this.jobs.succeedJob(job.id, result.output ?? {}, result.progress ?? 1);
             await this.emitWebhookIfNeeded("job.succeeded", job.id);
         }
         catch (error) {
-            if (error instanceof ProviderKeyUnavailableError) {
-                this.jobs.requeueJob(job, config.DEFAULT_JOB_DELAY_SECONDS, { message: error.message, type: "provider_key_unavailable" });
+            if (error instanceof ProviderLeaseTimeoutError) {
+                this.jobs.requeueJob(job, error.retryAfterSeconds, { message: error.message, type: "provider_key_wait_timeout" });
                 await this.emitWebhookIfNeeded("job.queued", job.id);
                 return;
             }
+            if (error instanceof ProviderKeyUnavailableError) {
+                this.jobs.failJob(job.id, { message: error.message, type: "provider_key_unavailable" });
+                await this.emitWebhookIfNeeded("job.failed", job.id);
+                return;
+            }
             if (error instanceof ProviderRateLimitError) {
                 this.jobs.requeueJob(job, error.retryAfterSeconds, { message: error.message, type: "provider_rate_limited" });
                 await this.emitWebhookIfNeeded("job.queued", job.id);
@@ -131,4 +155,78 @@ export class Worker {
             })
         });
     }
+    selectRunnableJobs(limit) {
+        const candidateLimit = Math.max(limit * 10, config.WORKER_BATCH_SIZE * 5, 25);
+        const candidates = database.listRunnableJobs(candidateLimit);
+        const providerCapacity = this.buildProviderCapacityMap();
+        const selected = [];
+        for (const job of candidates) {
+            if (selected.length >= limit) {
+                break;
+            }
+            const providerHint = parseProviderHint(job.payload._providerHint);
+            if (!providerHint) {
+                selected.push(job);
+                continue;
+            }
+            const available = providerCapacity.get(this.providerCapacityKey(job.customerId, providerHint)) ?? 0;
+            if (available <= 0) {
+                continue;
+            }
+            providerCapacity.set(this.providerCapacityKey(job.customerId, providerHint), available - 1);
+            selected.push(job);
+        }
+        return selected;
+    }
+    buildProviderCapacityMap() {
+        const map = new Map();
+        for (const entry of database.listAvailableProviderKeyCounts()) {
+            map.set(this.providerCapacityKey(entry.customerId, entry.provider), entry.availableCount);
+        }
+        return map;
+    }
+    providerCapacityKey(customerId, provider) {
+        return `${customerId}:${provider}`;
+    }
+}
+async function runTemplateWorkflow(input) {
+    const workflow = input.template.jobs[input.job.workflowName];
+    if (!workflow) {
+        throw new Error(`Unknown workflow ${input.job.workflowName}`);
+    }
+    const ctx = createTemplateJobContext({
+        customer: input.customer,
+        job: input.job,
+        template: input.template,
+        workerId: input.workerId,
+        storage: input.storage,
+        jobs: input.jobs,
+        billing: input.billing,
+        providers: input.providers,
+        remotion: input.remotion
+    });
+    return workflow(ctx, input.job.payload);
+}
+async function runPrimitiveWorkflow(input) {
+    const workflow = input.primitive.jobs[input.job.workflowName];
+    if (!workflow) {
+        throw new Error(`Unknown workflow ${input.job.workflowName}`);
+    }
+    const ctx = createPrimitiveJobContext({
+        customer: input.customer,
+        job: input.job,
+        primitive: input.primitive,
+        workerId: input.workerId,
+        storage: input.storage,
+        billing: input.billing,
+        providers: input.providers,
+        remotion: input.remotion
+    });
+    return workflow(ctx, input.job.payload);
+}
+function parseProviderHint(value) {
+    if (value === "openai" || value === "gemini" || value === "openrouter" || value === "perplexity") {
+        return value;
+    }
+    return null;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.2.7",
+  "version": "0.2.9",
   "description": "Developer CLI for running the Vidfarm local template platform.",
   "type": "module",
   "bin": {
@@ -9,12 +9,20 @@
   },
   "files": [
     "dist/src",
-    "dist/templates/template_0000",
-    "templates/template_0000",
+    "public/assets",
+    "auto-create-templates/AUTO_CREATE_TEMPLATES.md",
+    "auto-create-templates/extractor-system-prompt.md",
+    "auto-create-templates/production-graph.schema.json",
+    "auto-create-templates/runbook.md",
+    "auto-create-templates/template-plan.schema.json",
+    "auto-create-templates/template-planner-prompt.md",
+    "dist/templates/vidfarm_template_0000",
+    "templates/vidfarm_template_0000",
     "README.md",
     "SKILL.developer.md",
     "GETTING_STARTED.developers.md",
-    ".env.example"
+    ".env.example",
+    "!readme.secret.md"
   ],
   "publishConfig": {
     "access": "restricted"
@@ -28,10 +36,13 @@
   },
   "scripts": {
     "dev": "tsx watch src/index.ts",
+    "dev:frontend": "node scripts/build-homepage-client.mjs --watch",
     "dev:cli": "tsx src/cli.ts dev",
+    "build:frontend": "node scripts/build-homepage-client.mjs",
+    "operator:template-artifact": "node scripts/platform-operator-template-artifact.mjs",
     "deploy:prod-inplace": "bash scripts/deploy-prod-inplace.sh",
     "rollback:prod-inplace": "bash scripts/rollback-prod-inplace.sh",
-    "build": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json",
+    "build": "npm run build:frontend && node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json",
     "start": "node dist/src/index.js",
     "check": "tsc -p tsconfig.json --noEmit",
     "cdk:synth": "npm run build && npx aws-cdk synth --app 'node dist/infra/cdk/bin/vidfarm-prod.js'",
@@ -39,12 +50,23 @@
     "prepack": "npm run build"
   },
   "dependencies": {
+    "@ai-sdk/google": "^3.0.79",
+    "@ai-sdk/openai": "^3.0.65",
+    "@ai-sdk/openai-compatible": "^2.0.48",
+    "@assistant-ui/react": "^0.14.8",
+    "@assistant-ui/react-data-stream": "^0.12.14",
+    "@aws-sdk/client-dynamodb": "^3.1056.0",
     "@aws-sdk/client-s3": "^3.787.0",
     "@aws-sdk/s3-request-presigner": "^3.787.0",
+    "@fontsource/libre-baskerville": "^5.2.10",
+    "@fontsource/montserrat": "^5.2.8",
+    "@fontsource/source-code-pro": "^5.2.7",
+    "@fontsource/tiktok-sans": "^5.2.4",
     "@hono/node-server": "^1.14.4",
     "@remotion/bundler": "4.0.355",
     "@remotion/lambda": "4.0.355",
     "@remotion/renderer": "4.0.355",
+    "ai": "^6.0.191",
     "better-sqlite3": "^12.1.1",
     "dotenv": "^16.5.0",
     "hono": "^4.8.3",
@@ -61,7 +83,9 @@
     "@types/react-dom": "^18.3.7",
     "aws-cdk-lib": "^2.196.0",
     "constructs": "^10.4.2",
+    "esbuild": "^0.28.0",
     "tsx": "^4.19.4",
-    "typescript": "^5.8.3"
+    "typescript": "^5.8.3",
+    "zustand": "^5.0.13"
   }
 }