@mevdragon/vidfarm-devcli 0.2.7 → 0.2.9

package/dist/src/registry.js

@@ -1,14 +1,10 @@
+import { existsSync, readdirSync } from "node:fs";
+import path from "node:path";
 import { database } from "./db.js";
-import { template0000Definition } from "../templates/template_0000/src/template.js";
 import { loadTemplateFromModule } from "./services/template-loader.js";
-import { TemplateCertificationService } from "./services/template-certification.js";
 class TemplateRegistry {
-    localTemplates = new Map([[
-            template0000Definition.id,
-            template0000Definition
-        ]]);
+    localTemplates = new Map();
     runtimeTemplates = new Map();
-    certification = new TemplateCertificationService();
     initialized = false;
     initPromise = null;
     async ensureInitialized() {
@@ -19,22 +15,24 @@ class TemplateRegistry {
             return this.initPromise;
         }
         this.initPromise = (async () => {
-            for (const template of this.localTemplates.values()) {
-                const skillPath = template.skillPath;
-                if (!skillPath) {
-                    throw new Error(`Local template ${template.id} is missing skillPath.`);
+            await this.loadLocalTemplates();
+            const visibleReleaseByTemplateId = new Map();
+            for (const release of database.listTemplateReleases()) {
+                if (release.status !== "approved" && release.status !== "active") {
+                    continue;
                 }
-                const report = await this.certification.certify({ template, skillPath });
-                if (!report.passed) {
-                    throw new Error(`Local template ${template.id} failed certification.`);
+                const current = visibleReleaseByTemplateId.get(release.templateId);
+                if (!current || getReleaseTimestamp(release) > getReleaseTimestamp(current)) {
+                    visibleReleaseByTemplateId.set(release.templateId, release);
                 }
             }
-            for (const release of database.getActiveTemplateReleases()) {
+            for (const release of visibleReleaseByTemplateId.values()) {
                 const template = await loadTemplateFromModule(release.modulePath);
-                this.runtimeTemplates.set(release.templateId, {
+                const runtimeTemplate = {
                     ...template,
                     skillPath: release.skillPath
-                });
+                };
+                this.runtimeTemplates.set(release.templateId, runtimeTemplate);
             }
             this.initialized = true;
             this.initPromise = null;
@@ -42,7 +40,14 @@ class TemplateRegistry {
         return this.initPromise;
     }
     list() {
-        return [...this.localTemplates.values(), ...this.runtimeTemplates.values()].sort((a, b) => a.id.localeCompare(b.id));
+        const merged = new Map();
+        for (const template of this.localTemplates.values()) {
+            merged.set(template.id, template);
+        }
+        for (const template of this.runtimeTemplates.values()) {
+            merged.set(template.id, template);
+        }
+        return [...merged.values()].sort((a, b) => a.id.localeCompare(b.id));
     }
     get(templateId) {
         return this.findTemplate(templateId, this.runtimeTemplates) ?? this.findTemplate(templateId, this.localTemplates);
@@ -50,8 +55,29 @@ class TemplateRegistry {
     registerRuntimeTemplate(template) {
         this.runtimeTemplates.set(template.id, template);
     }
+    async loadLocalTemplates() {
+        this.localTemplates.clear();
+        for (const modulePath of discoverLocalTemplateModulePaths(process.cwd())) {
+            const template = await loadTemplateFromModule(modulePath);
+            this.localTemplates.set(template.id, template);
+        }
+    }
     findTemplate(templateKey, source) {
         return source.get(templateKey) ?? [...source.values()].find((template) => template.slugId === templateKey) ?? null;
     }
 }
+function getReleaseTimestamp(release) {
+    return Date.parse(release.activatedAt ?? release.updatedAt ?? release.createdAt);
+}
+function discoverLocalTemplateModulePaths(rootDir) {
+    const templatesDir = path.resolve(rootDir, "templates");
+    if (!existsSync(templatesDir)) {
+        return [];
+    }
+    return readdirSync(templatesDir, { withFileTypes: true })
+        .filter((entry) => entry.isDirectory() && entry.name.startsWith("vidfarm_template_"))
+        .map((entry) => path.join(templatesDir, entry.name, "src", "template.ts"))
+        .filter((modulePath) => existsSync(modulePath))
+        .sort((a, b) => a.localeCompare(b));
+}
 export const templateRegistry = new TemplateRegistry();

package/dist/src/runtime.js

@@ -2,6 +2,7 @@ import { serve } from "@hono/node-server";
 import app from "./app.js";
 import { config } from "./config.js";
 import { templateRegistry } from "./registry.js";
+import { apiCallHistoryStore } from "./services/api-call-history.js";
 import { jobLogStore } from "./services/job-logs.js";
 import { Worker } from "./worker.js";
 export async function startRuntime() {
@@ -10,12 +11,16 @@ export async function startRuntime() {
     worker.start();
     const server = serve({
         fetch: app.fetch,
-        port: config.PORT
+        port: config.PORT,
+        serverOptions: {
+            maxHeaderSize: 64 * 1024
+        }
     }, (info) => {
         console.log(`[vidfarm] listening on http://localhost:${info.port}`);
     });
     const shutdown = async () => {
         worker.stop();
+        await apiCallHistoryStore.shutdown();
         await jobLogStore.shutdown();
         server.close();
     };

package/dist/src/services/api-call-history.js

@@ -0,0 +1,245 @@
+import { StorageService } from "./storage.js";
+const API_CALL_HISTORY_PREFIX = "api-call-history";
+const CHUNK_INTERVAL_MS = 60_000;
+const CHUNK_MAX_BYTES = 1024 * 1024;
+const FLUSH_INTERVAL_MS = 5_000;
+export class ApiCallHistoryStore {
+    storage = new StorageService();
+    activeChunks = new Map();
+    flushTimer = setInterval(() => {
+        void this.flushExpiredChunks();
+    }, FLUSH_INTERVAL_MS);
+    constructor() {
+        this.flushTimer.unref();
+    }
+    addEntry(entry) {
+        const normalized = {
+            ...entry,
+            createdAt: entry.createdAt ?? new Date().toISOString()
+        };
+        const minuteStartMs = floorToMinute(Date.parse(normalized.createdAt));
+        const activePrefix = this.activeChunkPrefix(normalized.customerId, minuteStartMs);
+        const current = this.findLatestChunk(activePrefix);
+        const entryBytes = serializedEntrySize(normalized);
+        const chunk = current && current.sizeBytes + entryBytes <= CHUNK_MAX_BYTES
+            ? current
+            : this.createChunk(normalized.customerId, minuteStartMs, current ? current.sequence + 1 : 0);
+        chunk.entries.push(normalized);
+        chunk.sizeBytes += entryBytes;
+        chunk.dirty = true;
+        if (chunk.sizeBytes >= CHUNK_MAX_BYTES) {
+            void this.persistChunk(chunk, true);
+        }
+    }
+    async listEntries(input) {
+        const persisted = await this.readPersistedEntries(input);
+        const buffered = this.readBufferedEntries(input);
+        const merged = dedupeAndSort([...persisted, ...buffered]).reverse();
+        const filtered = input.cursor
+            ? merged.filter((entry) => (Date.parse(entry.createdAt) < Date.parse(input.cursor.createdAt)
+                || (entry.createdAt === input.cursor.createdAt && entry.id < input.cursor.id)))
+            : merged;
+        if (input.limit === undefined) {
+            return filtered;
+        }
+        const limit = Math.max(1, Math.min(input.limit, 501));
+        return filtered.slice(0, limit);
+    }
+    async renameTracer(input) {
+        const fromTracer = input.fromTracer.trim();
+        const toTracer = input.toTracer.trim();
+        if (!fromTracer || !toTracer || fromTracer === toTracer) {
+            return { updated: 0 };
+        }
+        let updated = 0;
+        for (const chunk of this.activeChunks.values()) {
+            if (chunk.customerId !== input.customerId) {
+                continue;
+            }
+            let chunkChanged = false;
+            for (const entry of chunk.entries) {
+                if (entry.tracer === fromTracer && withinRange(entry.createdAt, input.startTime, input.endTime)) {
+                    entry.tracer = toTracer;
+                    updated += 1;
+                    chunkChanged = true;
+                }
+            }
+            if (chunkChanged) {
+                chunk.dirty = true;
+                chunk.sizeBytes = chunk.entries.reduce((sum, entry) => sum + serializedEntrySize(entry), 0);
+                await this.persistChunk(chunk, false);
+            }
+        }
+        const keys = await this.listRelevantKeys(input);
+        await Promise.all(keys.map(async (key) => {
+            const content = await this.storage.readText(key);
+            const entries = parseChunk(content);
+            let changed = false;
+            for (const entry of entries) {
+                if (entry.customerId === input.customerId
+                    && entry.tracer === fromTracer
+                    && withinRange(entry.createdAt, input.startTime, input.endTime)) {
+                    entry.tracer = toTracer;
+                    updated += 1;
+                    changed = true;
+                }
+            }
+            if (changed) {
+                const body = `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
+                await this.storage.putText(key, body, "application/x-ndjson; charset=utf-8");
+            }
+        }));
+        return { updated };
+    }
+    async shutdown() {
+        clearInterval(this.flushTimer);
+        await Promise.all([...this.activeChunks.values()].map((chunk) => this.persistChunk(chunk, true)));
+    }
+    async readPersistedEntries(input) {
+        const keys = await this.listRelevantKeys(input);
+        const entries = await Promise.all(keys.map(async (key) => parseChunk(await this.storage.readText(key))));
+        return entries
+            .flat()
+            .filter((entry) => entry.customerId === input.customerId)
+            .filter((entry) => withinRange(entry.createdAt, input.startTime, input.endTime));
+    }
+    readBufferedEntries(input) {
+        const buffered = [];
+        for (const chunk of this.activeChunks.values()) {
+            if (chunk.customerId !== input.customerId) {
+                continue;
+            }
+            for (const entry of chunk.entries.slice(chunk.lastPersistedEntryCount)) {
+                if (withinRange(entry.createdAt, input.startTime, input.endTime)) {
+                    buffered.push(entry);
+                }
+            }
+        }
+        return buffered;
+    }
+    async listRelevantKeys(input) {
+        const minuteStarts = this.minuteStartsForQuery(input.startTime, input.endTime);
+        if (minuteStarts === null) {
+            return this.storage.listKeys(`${API_CALL_HISTORY_PREFIX}/customer_id=${input.customerId}/`);
+        }
+        const nested = await Promise.all(minuteStarts.map((minuteStartMs) => (this.storage.listKeys(this.minutePrefix(input.customerId, minuteStartMs)))));
+        return nested.flat();
+    }
+    minuteStartsForQuery(startTime, endTime) {
+        if (!startTime && !endTime) {
+            return null;
+        }
+        const startMs = floorToMinute(Date.parse(startTime ?? endTime ?? new Date().toISOString()));
+        const endMs = floorToMinute(Date.parse(endTime ?? startTime ?? new Date().toISOString()));
+        const minMs = Math.min(startMs, endMs);
+        const maxMs = Math.max(startMs, endMs);
+        const minuteStarts = [];
+        for (let cursor = minMs; cursor <= maxMs; cursor += 60_000) {
+            minuteStarts.push(cursor);
+        }
+        return minuteStarts;
+    }
+    async flushExpiredChunks() {
+        const now = Date.now();
+        await Promise.all([...this.activeChunks.values()].map(async (chunk) => {
+            const intervalElapsed = now >= chunk.minuteStartMs + CHUNK_INTERVAL_MS;
+            if (intervalElapsed || chunk.sizeBytes >= CHUNK_MAX_BYTES) {
+                await this.persistChunk(chunk, true);
+                return;
+            }
+            if (chunk.dirty) {
+                await this.persistChunk(chunk, false);
+            }
+        }));
+    }
+    createChunk(customerId, minuteStartMs, sequence) {
+        const chunk = {
+            customerId,
+            minuteStartMs,
+            sequence,
+            entries: [],
+            sizeBytes: 0,
+            lastPersistedEntryCount: 0,
+            dirty: false
+        };
+        this.activeChunks.set(this.chunkKeyForMap(customerId, minuteStartMs, sequence), chunk);
+        return chunk;
+    }
+    findLatestChunk(activePrefix) {
+        const matching = [...this.activeChunks.entries()]
+            .filter(([key]) => key.startsWith(activePrefix))
+            .map(([, chunk]) => chunk)
+            .sort((a, b) => a.sequence - b.sequence);
+        return matching.at(-1) ?? null;
+    }
+    activeChunkPrefix(customerId, minuteStartMs) {
+        return `${customerId}:${minuteStartMs}:`;
+    }
+    chunkKeyForMap(customerId, minuteStartMs, sequence) {
+        return `${this.activeChunkPrefix(customerId, minuteStartMs)}${sequence}`;
+    }
+    async persistChunk(chunk, finalize) {
+        if (!chunk.entries.length) {
+            if (finalize) {
+                this.activeChunks.delete(this.chunkKeyForMap(chunk.customerId, chunk.minuteStartMs, chunk.sequence));
+            }
+            return;
+        }
+        const key = this.objectKeyForChunk(chunk);
+        const body = `${chunk.entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
+        await this.storage.putText(key, body, "application/x-ndjson; charset=utf-8");
+        chunk.lastPersistedEntryCount = chunk.entries.length;
+        chunk.dirty = false;
+        if (finalize) {
+            this.activeChunks.delete(this.chunkKeyForMap(chunk.customerId, chunk.minuteStartMs, chunk.sequence));
+        }
+    }
+    objectKeyForChunk(chunk) {
+        const startTs = Math.floor(chunk.entries[0] ? Date.parse(chunk.entries[0].createdAt) / 1000 : chunk.minuteStartMs / 1000);
+        const endTs = Math.floor(chunk.entries.at(-1) ? Date.parse(chunk.entries.at(-1).createdAt) / 1000 : chunk.minuteStartMs / 1000);
+        return `${this.minutePrefix(chunk.customerId, chunk.minuteStartMs)}chunk-${startTs}-${endTs}-${String(chunk.sequence).padStart(4, "0")}.ndjson`;
+    }
+    minutePrefix(customerId, minuteStartMs) {
+        return `${API_CALL_HISTORY_PREFIX}/customer_id=${customerId}/minute=${Math.floor(minuteStartMs / 1000)}/`;
+    }
+}
+export const apiCallHistoryStore = new ApiCallHistoryStore();
+function serializedEntrySize(entry) {
+    return Buffer.byteLength(`${JSON.stringify(entry)}\n`, "utf8");
+}
+function floorToMinute(timestampMs) {
+    return Math.floor(timestampMs / 60_000) * 60_000;
+}
+function parseChunk(content) {
+    if (!content) {
+        return [];
+    }
+    return content
+        .split("\n")
+        .map((line) => line.trim())
+        .filter(Boolean)
+        .map((line) => {
+        const parsed = JSON.parse(line);
+        return {
+            ...parsed,
+            jobId: typeof parsed.jobId === "string" && parsed.jobId.trim() ? parsed.jobId.trim() : null
+        };
+    });
+}
+function dedupeAndSort(entries) {
+    const deduped = new Map();
+    for (const entry of entries) {
+        deduped.set(entry.id, entry);
+    }
+    return [...deduped.values()].sort((a, b) => Date.parse(a.createdAt) - Date.parse(b.createdAt));
+}
+function withinRange(createdAt, startTime, endTime) {
+    const value = Date.parse(createdAt);
+    if (startTime && value < Date.parse(startTime)) {
+        return false;
+    }
+    if (endTime && value > Date.parse(endTime)) {
+        return false;
+    }
+    return true;
+}

package/dist/src/services/auth.js

@@ -14,7 +14,7 @@ export class AuthService {
             expiresAt: addSeconds(new Date(), 600)
         });
         if (config.RESEND_API_KEY) {
-            await fetch("https://api.resend.com/emails", {
+            const response = await fetch("https://api.resend.com/emails", {
                 method: "POST",
                 headers: {
                     "Content-Type": "application/json",
@@ -27,9 +27,20 @@ export class AuthService {
                     text: `Your login code is ${code}. It expires in 10 minutes.`
                 })
             });
+            if (!response.ok) {
+                const body = await response.text();
+                throw new Error(`Unable to send OTP email (${response.status}): ${body || "empty response body"}`);
+            }
+            return {
+                delivery: "email"
+            };
         }
         else {
             console.log(`[vidfarm] OTP for ${email}: ${code}`);
+            return {
+                delivery: "console",
+                code: config.isProduction ? undefined : code
+            };
         }
     }
     verifyOtp(email, code, name) {
@@ -54,10 +65,9 @@ export class AuthService {
             email,
             name: name ?? existing?.name ?? null,
             defaultWebhookUrl: existing?.defaultWebhookUrl ?? null,
-            isDeveloper: existing?.isDeveloper
-                || config.adminEmails.includes(normalizedEmail)
-                || config.developerEmails.includes(normalizedEmail),
-            isPaidPlan: existing?.isPaidPlan ?? false
+            isDeveloper: existing?.isDeveloper || config.adminEmails.includes(normalizedEmail),
+            isPaidPlan: existing?.isPaidPlan ?? false,
+            planTier: existing?.planTier ?? "starter"
         });
         const rawApiKey = `vf_${createId("key")}`;
         database.insertApiKey({
@@ -88,9 +98,12 @@ export class AuthService {
             defaultWebhookUrl: row.default_webhook_url ? String(row.default_webhook_url) : null,
             isDeveloper: Boolean(row.is_developer),
             isPaidPlan: Boolean(row.is_paid_plan),
+            planTier: normalizePlanTier(row.plan_tier),
             about: row.about ? String(row.about) : null,
             groupchatUrl: row.groupchat_url ? String(row.groupchat_url) : null,
-            flockposterApiKey: row.flockposter_api_key ? String(row.flockposter_api_key) : null
+            flockposterApiKey: row.flockposter_api_key ? String(row.flockposter_api_key) : null,
+            createdAt: String(row.created_at),
+            updatedAt: String(row.updated_at)
         };
     }
     verifyOtpForBrowserLogin(email, code, name) {
@@ -134,11 +147,16 @@ export class AuthService {
             email: normalizedEmail,
             name: input.name ?? existing?.name ?? null,
             defaultWebhookUrl: existing?.defaultWebhookUrl ?? null,
-            isDeveloper: existing?.isDeveloper
-                || config.adminEmails.includes(normalizedEmail)
-                || config.developerEmails.includes(normalizedEmail),
+            isDeveloper: existing?.isDeveloper || config.adminEmails.includes(normalizedEmail),
             isPaidPlan: true,
+            planTier: input.planTier ?? existing?.planTier ?? "starter",
             passwordHash: hashPassword(input.password)
         });
     }
 }
+function normalizePlanTier(value) {
+    if (value === "starter" || value === "agency" || value === "standard") {
+        return value;
+    }
+    return "starter";
+}