@mevdragon/vidfarm-devcli 0.5.3 → 0.6.0

package/dist/src/registry.js

@@ -0,0 +1,296 @@
+import { existsSync, readdirSync } from "node:fs";
+import path from "node:path";
+import { z } from "zod";
+import { config } from "./config.js";
+import { devLog } from "./lib/dev-log.js";
+import { loadTemplateFromModule } from "./services/template-loader.js";
+import { loadTemplateFromRuntimeRelease } from "./services/template-runtime-bundles.js";
+import { serverlessRecords } from "./services/serverless-records.js";
+class TemplateRegistry {
+    localTemplates = new Map();
+    releaseTemplates = new Map();
+    activeReleases = new Map();
+    runtimeTemplates = new Map();
+    initialized = false;
+    initPromise = null;
+    async ensureInitialized() {
+        if (this.initialized) {
+            return;
+        }
+        if (this.initPromise) {
+            return this.initPromise;
+        }
+        this.initPromise = (async () => {
+            await this.loadActiveReleaseSnapshots();
+            if (shouldLoadLocalTemplates()) {
+                await this.loadLocalTemplates();
+                devLog("template_registry.local_loaded", {
+                    count: this.localTemplates.size,
+                    template_ids: [...this.localTemplates.keys()]
+                });
+            }
+            this.initialized = true;
+            this.initPromise = null;
+            devLog("template_registry.initialized", {
+                local_count: this.localTemplates.size,
+                release_count: this.releaseTemplates.size,
+                runtime_count: this.runtimeTemplates.size
+            });
+        })();
+        return this.initPromise;
+    }
+    list() {
+        const merged = new Map();
+        for (const template of this.releaseTemplates.values()) {
+            merged.set(template.id, template);
+        }
+        for (const template of this.localTemplates.values()) {
+            merged.set(template.id, template);
+        }
+        for (const template of this.runtimeTemplates.values()) {
+            merged.set(template.id, template);
+        }
+        return [...merged.values()].sort((a, b) => a.id.localeCompare(b.id));
+    }
+    get(templateId) {
+        return this.findTemplate(templateId, this.runtimeTemplates)
+            ?? this.findTemplate(templateId, this.localTemplates)
+            ?? this.findTemplate(templateId, this.releaseTemplates);
+    }
+    async listMetadata() {
+        try {
+            const releases = await serverlessRecords.listActiveTemplateReleases?.() ?? [];
+            for (const release of releases) {
+                this.cacheReleaseSnapshot(release);
+            }
+            const releaseTemplates = releases
+                .filter((release) => release.templateSnapshot)
+                .map((release) => templateFromSnapshot(release.templateSnapshot, release));
+            return await this.mergeLocalMetadata(releaseTemplates);
+        }
+        catch (error) {
+            devLog("template_registry.metadata.list_failed", {
+                message: error instanceof Error ? error.message : String(error)
+            }, "warn");
+            await this.ensureInitialized();
+            return this.list();
+        }
+    }
+    async listMetadataPage(input = {}) {
+        try {
+            const page = await serverlessRecords.listActiveTemplateReleasePage?.(input);
+            if (!page) {
+                return { templates: await this.listMetadata(), nextCursor: null };
+            }
+            for (const release of page.items) {
+                this.cacheReleaseSnapshot(release);
+            }
+            const releaseTemplates = page.items
+                .filter((release) => release.templateSnapshot)
+                .map((release) => templateFromSnapshot(release.templateSnapshot, release));
+            return {
+                templates: await this.mergeLocalMetadata(releaseTemplates),
+                nextCursor: page.nextCursor
+            };
+        }
+        catch (error) {
+            devLog("template_registry.metadata.page_failed", {
+                message: error instanceof Error ? error.message : String(error)
+            }, "warn");
+            return { templates: await this.listMetadata(), nextCursor: null };
+        }
+    }
+    async getMetadata(templateId) {
+        const cached = this.get(templateId);
+        if (cached) {
+            return cached;
+        }
+        try {
+            const release = await serverlessRecords.getActiveTemplateReleaseByTemplateId?.(templateId);
+            if (release?.templateSnapshot) {
+                return this.cacheReleaseSnapshot(release);
+            }
+        }
+        catch (error) {
+            devLog("template_registry.metadata.get_failed", {
+                template_id: templateId,
+                message: error instanceof Error ? error.message : String(error)
+            }, "warn");
+        }
+        await this.ensureInitialized();
+        return this.get(templateId);
+    }
+    async getExecutable(templateId) {
+        const runtime = this.findTemplate(templateId, this.runtimeTemplates) ?? this.findTemplate(templateId, this.localTemplates);
+        if (runtime) {
+            return runtime;
+        }
+        let releaseTemplate = this.findTemplate(templateId, this.releaseTemplates);
+        let release = releaseTemplate ? this.activeReleases.get(releaseTemplate.id) : null;
+        if (!release) {
+            try {
+                const activeRelease = await serverlessRecords.getActiveTemplateReleaseByTemplateId?.(templateId);
+                if (activeRelease?.templateSnapshot) {
+                    releaseTemplate = this.cacheReleaseSnapshot(activeRelease);
+                    release = activeRelease;
+                }
+            }
+            catch (error) {
+                devLog("template_registry.executable.active_release_lookup_failed", {
+                    template_id: templateId,
+                    message: error instanceof Error ? error.message : String(error)
+                }, "warn");
+            }
+        }
+        if (!release) {
+            await this.ensureInitialized();
+            const initializedRuntime = this.findTemplate(templateId, this.runtimeTemplates) ?? this.findTemplate(templateId, this.localTemplates);
+            if (initializedRuntime) {
+                return initializedRuntime;
+            }
+            releaseTemplate = this.findTemplate(templateId, this.releaseTemplates);
+            release = releaseTemplate ? this.activeReleases.get(releaseTemplate.id) : null;
+        }
+        if (!release) {
+            return null;
+        }
+        const template = await loadTemplateFromRuntimeRelease(release);
+        this.runtimeTemplates.set(template.id, template);
+        return template;
+    }
+    registerRuntimeTemplate(template) {
+        this.runtimeTemplates.set(template.id, template);
+    }
+    cacheReleaseSnapshot(release) {
+        if (!release.templateSnapshot) {
+            return null;
+        }
+        const template = templateFromSnapshot(release.templateSnapshot, release);
+        this.releaseTemplates.set(template.id, template);
+        this.activeReleases.set(template.id, release);
+        return template;
+    }
+    async loadLocalTemplates() {
+        this.localTemplates.clear();
+        for (const modulePath of discoverLocalTemplateModulePaths(process.cwd())) {
+            const template = await loadTemplateFromModule(modulePath);
+            this.localTemplates.set(template.id, template);
+            devLog("template_registry.local_template.loaded", {
+                template_id: template.id,
+                slug_id: template.slugId,
+                module_path: modulePath
+            });
+        }
+    }
+    async loadActiveReleaseSnapshots() {
+        this.releaseTemplates.clear();
+        this.activeReleases.clear();
+        let releases = [];
+        try {
+            releases = await serverlessRecords.listActiveTemplateReleases?.() ?? [];
+        }
+        catch (error) {
+            devLog("template_registry.active_releases.load_failed", {
+                message: error instanceof Error ? error.message : String(error)
+            }, "warn");
+            return;
+        }
+        for (const release of releases) {
+            if (!release.templateSnapshot) {
+                continue;
+            }
+            this.cacheReleaseSnapshot(release);
+        }
+        devLog("template_registry.active_releases.loaded", {
+            count: this.releaseTemplates.size,
+            template_ids: [...this.releaseTemplates.keys()]
+        });
+    }
+    async mergeLocalMetadata(templates) {
+        if (shouldLoadLocalTemplates()) {
+            await this.ensureInitialized();
+        }
+        const merged = new Map();
+        for (const template of templates) {
+            merged.set(template.id, template);
+        }
+        for (const template of this.localTemplates.values()) {
+            merged.set(template.id, template);
+        }
+        for (const template of this.runtimeTemplates.values()) {
+            merged.set(template.id, template);
+        }
+        return [...merged.values()].sort((a, b) => a.id.localeCompare(b.id));
+    }
+    findTemplate(templateKey, source) {
+        return source.get(templateKey) ?? [...source.values()].find((template) => template.slugId === templateKey) ?? null;
+    }
+}
+function templateFromSnapshot(snapshot, release) {
+    const snapshotAbout = snapshot.about;
+    const operations = Object.fromEntries(snapshot.operations.map((operation) => [
+        operation.name,
+        {
+            description: operation.description,
+            inputSchema: z.record(z.string(), z.unknown()),
+            workflow: operation.workflow,
+            providerHint: operation.providerHint ?? undefined,
+            webhookSupport: operation.webhookSupport ?? undefined
+        }
+    ]));
+    return {
+        id: snapshot.id,
+        slugId: snapshot.slugId,
+        version: snapshot.version,
+        about: {
+            ...snapshot.about,
+            links_to_inspiration: normalizeTemplateInspirationLinks(snapshotAbout.links_to_inspiration, snapshot.about.link_to_original),
+            sound: normalizeTemplateSound(snapshotAbout.sound)
+        },
+        examples: snapshot.examples,
+        skillPath: release.skillPath,
+        configSchema: z.record(z.string(), z.unknown()),
+        operations,
+        jobs: {}
+    };
+}
+function normalizeTemplateInspirationLinks(value, linkToOriginal) {
+    if (Array.isArray(value)) {
+        return value.filter((entry) => typeof entry === "string");
+    }
+    return linkToOriginal ? [linkToOriginal] : [];
+}
+function normalizeTemplateSound(value) {
+    return {
+        sound_title: typeof value?.sound_title === "string" ? value.sound_title : "",
+        sound_url: typeof value?.sound_url === "string" ? value.sound_url : ""
+    };
+}
+function shouldLoadLocalTemplates() {
+    if (config.VIDFARM_LOAD_LOCAL_TEMPLATES === "true") {
+        return true;
+    }
+    if (config.VIDFARM_LOAD_LOCAL_TEMPLATES === "false") {
+        return false;
+    }
+    return !config.isProduction;
+}
+function discoverLocalTemplateModulePaths(rootDir) {
+    const templatesDir = path.resolve(rootDir, "templates");
+    if (!existsSync(templatesDir)) {
+        return [];
+    }
+    return readdirSync(templatesDir, { withFileTypes: true })
+        .filter((entry) => entry.isDirectory() && entry.name.startsWith("vidfarm_template_"))
+        .filter((entry) => (entry.name !== "vidfarm_template_0000"
+        || process.env.NODE_ENV !== "production"
+        || process.env.VIDFARM_DEPLOYMENT_TARGET === "serverless-staging"))
+        .map((entry) => [
+        path.join(templatesDir, entry.name, "src", "template.js"),
+        path.join(templatesDir, entry.name, "src", "template.ts")
+    ].find((modulePath) => existsSync(modulePath)))
+        .filter((modulePath) => Boolean(modulePath))
+        .sort((a, b) => a.localeCompare(b));
+}
+export const templateRegistry = new TemplateRegistry();
+//# sourceMappingURL=registry.js.map

package/dist/src/runtime.js

@@ -0,0 +1,35 @@
+import { serve } from "@hono/node-server";
+import * as Sentry from "@sentry/node";
+import app from "./app.js";
+import { config } from "./config.js";
+import { templateRegistry } from "./registry.js";
+import { apiCallHistoryStore } from "./services/api-call-history.js";
+import { jobLogStore } from "./services/job-logs.js";
+import { compositionWatch } from "./services/composition-watch.js";
+import { Worker } from "./worker.js";
+export async function startRuntime() {
+    await templateRegistry.ensureInitialized();
+    const worker = new Worker();
+    worker.start();
+    // No-op unless STORAGE_DRIVER=local (i.e. `vidfarm serve`): fans on-disk
+    // composition edits out to open editor tabs over same-origin SSE.
+    compositionWatch.start();
+    const server = serve({
+        fetch: app.fetch,
+        port: config.PORT,
+        serverOptions: {
+            maxHeaderSize: 64 * 1024
+        }
+    }, (info) => {
+        console.log(`[vidfarm] listening on http://localhost:${info.port}`);
+    });
+    const shutdown = async () => {
+        worker.stop();
+        await apiCallHistoryStore.shutdown();
+        await jobLogStore.shutdown();
+        await Sentry.flush(2_000).catch(() => undefined);
+        server.close();
+    };
+    return { server, worker, shutdown };
+}
+//# sourceMappingURL=runtime.js.map

package/dist/src/services/api-call-history.js

@@ -0,0 +1,249 @@
+import { StorageService } from "./storage.js";
+const API_CALL_HISTORY_PREFIX = "api-call-history";
+const CHUNK_INTERVAL_MS = 60_000;
+const CHUNK_MAX_BYTES = 1024 * 1024;
+const FLUSH_INTERVAL_MS = 5_000;
+export class ApiCallHistoryStore {
+    storage = new StorageService();
+    activeChunks = new Map();
+    flushTimer = setInterval(() => {
+        void this.flushExpiredChunks();
+    }, FLUSH_INTERVAL_MS);
+    constructor() {
+        this.flushTimer.unref();
+    }
+    addEntry(entry) {
+        const normalized = {
+            ...entry,
+            createdAt: entry.createdAt ?? new Date().toISOString()
+        };
+        const minuteStartMs = floorToMinute(Date.parse(normalized.createdAt));
+        const activePrefix = this.activeChunkPrefix(normalized.customerId, minuteStartMs);
+        const current = this.findLatestChunk(activePrefix);
+        const entryBytes = serializedEntrySize(normalized);
+        const chunk = current && current.sizeBytes + entryBytes <= CHUNK_MAX_BYTES
+            ? current
+            : this.createChunk(normalized.customerId, minuteStartMs, current ? current.sequence + 1 : 0);
+        chunk.entries.push(normalized);
+        chunk.sizeBytes += entryBytes;
+        chunk.dirty = true;
+        if (chunk.sizeBytes >= CHUNK_MAX_BYTES) {
+            void this.persistChunk(chunk, true);
+        }
+    }
+    async listEntries(input) {
+        const persisted = await this.readPersistedEntries(input);
+        const buffered = this.readBufferedEntries(input);
+        const merged = dedupeAndSort([...persisted, ...buffered]).reverse();
+        const filtered = input.cursor
+            ? merged.filter((entry) => (Date.parse(entry.createdAt) < Date.parse(input.cursor.createdAt)
+                || (entry.createdAt === input.cursor.createdAt && entry.id < input.cursor.id)))
+            : merged;
+        if (input.limit === undefined) {
+            return filtered;
+        }
+        const limit = Math.max(1, Math.min(input.limit, 501));
+        return filtered.slice(0, limit);
+    }
+    async listStorageKeys(input) {
+        return this.listRelevantKeys(input);
+    }
+    async renameTracer(input) {
+        const fromTracer = input.fromTracer.trim();
+        const toTracer = input.toTracer.trim();
+        if (!fromTracer || !toTracer || fromTracer === toTracer) {
+            return { updated: 0 };
+        }
+        let updated = 0;
+        for (const chunk of this.activeChunks.values()) {
+            if (chunk.customerId !== input.customerId) {
+                continue;
+            }
+            let chunkChanged = false;
+            for (const entry of chunk.entries) {
+                if (entry.tracer === fromTracer && withinRange(entry.createdAt, input.startTime, input.endTime)) {
+                    entry.tracer = toTracer;
+                    updated += 1;
+                    chunkChanged = true;
+                }
+            }
+            if (chunkChanged) {
+                chunk.dirty = true;
+                chunk.sizeBytes = chunk.entries.reduce((sum, entry) => sum + serializedEntrySize(entry), 0);
+                await this.persistChunk(chunk, false);
+            }
+        }
+        const keys = await this.listRelevantKeys(input);
+        await Promise.all(keys.map(async (key) => {
+            const content = await this.storage.readText(key);
+            const entries = parseChunk(content);
+            let changed = false;
+            for (const entry of entries) {
+                if (entry.customerId === input.customerId
+                    && entry.tracer === fromTracer
+                    && withinRange(entry.createdAt, input.startTime, input.endTime)) {
+                    entry.tracer = toTracer;
+                    updated += 1;
+                    changed = true;
+                }
+            }
+            if (changed) {
+                const body = `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
+                await this.storage.putText(key, body, "application/x-ndjson; charset=utf-8");
+            }
+        }));
+        return { updated };
+    }
+    async shutdown() {
+        clearInterval(this.flushTimer);
+        await Promise.all([...this.activeChunks.values()].map((chunk) => this.persistChunk(chunk, true)));
+    }
+    async readPersistedEntries(input) {
+        const keys = await this.listRelevantKeys(input);
+        const entries = await Promise.all(keys.map(async (key) => parseChunk(await this.storage.readText(key))));
+        return entries
+            .flat()
+            .filter((entry) => entry.customerId === input.customerId)
+            .filter((entry) => withinRange(entry.createdAt, input.startTime, input.endTime));
+    }
+    readBufferedEntries(input) {
+        const buffered = [];
+        for (const chunk of this.activeChunks.values()) {
+            if (chunk.customerId !== input.customerId) {
+                continue;
+            }
+            for (const entry of chunk.entries.slice(chunk.lastPersistedEntryCount)) {
+                if (withinRange(entry.createdAt, input.startTime, input.endTime)) {
+                    buffered.push(entry);
+                }
+            }
+        }
+        return buffered;
+    }
+    async listRelevantKeys(input) {
+        const minuteStarts = this.minuteStartsForQuery(input.startTime, input.endTime);
+        if (minuteStarts === null) {
+            return this.storage.listKeys(`${API_CALL_HISTORY_PREFIX}/customer_id=${input.customerId}/`);
+        }
+        const nested = await Promise.all(minuteStarts.map((minuteStartMs) => (this.storage.listKeys(this.minutePrefix(input.customerId, minuteStartMs)))));
+        return nested.flat();
+    }
+    minuteStartsForQuery(startTime, endTime) {
+        if (!startTime && !endTime) {
+            return null;
+        }
+        const startMs = floorToMinute(Date.parse(startTime ?? endTime ?? new Date().toISOString()));
+        const endMs = floorToMinute(Date.parse(endTime ?? startTime ?? new Date().toISOString()));
+        const minMs = Math.min(startMs, endMs);
+        const maxMs = Math.max(startMs, endMs);
+        const minuteStarts = [];
+        for (let cursor = minMs; cursor <= maxMs; cursor += 60_000) {
+            minuteStarts.push(cursor);
+        }
+        return minuteStarts;
+    }
+    async flushExpiredChunks() {
+        const now = Date.now();
+        await Promise.all([...this.activeChunks.values()].map(async (chunk) => {
+            const intervalElapsed = now >= chunk.minuteStartMs + CHUNK_INTERVAL_MS;
+            if (intervalElapsed || chunk.sizeBytes >= CHUNK_MAX_BYTES) {
+                await this.persistChunk(chunk, true);
+                return;
+            }
+            if (chunk.dirty) {
+                await this.persistChunk(chunk, false);
+            }
+        }));
+    }
+    createChunk(customerId, minuteStartMs, sequence) {
+        const chunk = {
+            customerId,
+            minuteStartMs,
+            sequence,
+            entries: [],
+            sizeBytes: 0,
+            lastPersistedEntryCount: 0,
+            dirty: false
+        };
+        this.activeChunks.set(this.chunkKeyForMap(customerId, minuteStartMs, sequence), chunk);
+        return chunk;
+    }
+    findLatestChunk(activePrefix) {
+        const matching = [...this.activeChunks.entries()]
+            .filter(([key]) => key.startsWith(activePrefix))
+            .map(([, chunk]) => chunk)
+            .sort((a, b) => a.sequence - b.sequence);
+        return matching.at(-1) ?? null;
+    }
+    activeChunkPrefix(customerId, minuteStartMs) {
+        return `${customerId}:${minuteStartMs}:`;
+    }
+    chunkKeyForMap(customerId, minuteStartMs, sequence) {
+        return `${this.activeChunkPrefix(customerId, minuteStartMs)}${sequence}`;
+    }
+    async persistChunk(chunk, finalize) {
+        if (!chunk.entries.length) {
+            if (finalize) {
+                this.activeChunks.delete(this.chunkKeyForMap(chunk.customerId, chunk.minuteStartMs, chunk.sequence));
+            }
+            return;
+        }
+        const key = this.objectKeyForChunk(chunk);
+        const body = `${chunk.entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
+        await this.storage.putText(key, body, "application/x-ndjson; charset=utf-8");
+        chunk.lastPersistedEntryCount = chunk.entries.length;
+        chunk.dirty = false;
+        if (finalize) {
+            this.activeChunks.delete(this.chunkKeyForMap(chunk.customerId, chunk.minuteStartMs, chunk.sequence));
+        }
+    }
+    objectKeyForChunk(chunk) {
+        const startTs = Math.floor(chunk.entries[0] ? Date.parse(chunk.entries[0].createdAt) / 1000 : chunk.minuteStartMs / 1000);
+        const endTs = Math.floor(chunk.entries.at(-1) ? Date.parse(chunk.entries.at(-1).createdAt) / 1000 : chunk.minuteStartMs / 1000);
+        return `${this.minutePrefix(chunk.customerId, chunk.minuteStartMs)}chunk-${startTs}-${endTs}-${String(chunk.sequence).padStart(4, "0")}.ndjson`;
+    }
+    minutePrefix(customerId, minuteStartMs) {
+        return `${API_CALL_HISTORY_PREFIX}/customer_id=${customerId}/minute=${Math.floor(minuteStartMs / 1000)}/`;
+    }
+}
+export const apiCallHistoryStore = new ApiCallHistoryStore();
+function serializedEntrySize(entry) {
+    return Buffer.byteLength(`${JSON.stringify(entry)}\n`, "utf8");
+}
+function floorToMinute(timestampMs) {
+    return Math.floor(timestampMs / 60_000) * 60_000;
+}
+function parseChunk(content) {
+    if (!content) {
+        return [];
+    }
+    return content
+        .split("\n")
+        .map((line) => line.trim())
+        .filter(Boolean)
+        .map((line) => {
+        const parsed = JSON.parse(line);
+        return {
+            ...parsed,
+            jobId: typeof parsed.jobId === "string" && parsed.jobId.trim() ? parsed.jobId.trim() : null
+        };
+    });
+}
+function dedupeAndSort(entries) {
+    const deduped = new Map();
+    for (const entry of entries) {
+        deduped.set(entry.id, entry);
+    }
+    return [...deduped.values()].sort((a, b) => Date.parse(a.createdAt) - Date.parse(b.createdAt));
+}
+function withinRange(createdAt, startTime, endTime) {
+    const value = Date.parse(createdAt);
+    if (startTime && value < Date.parse(startTime)) {
+        return false;
+    }
+    if (endTime && value > Date.parse(endTime)) {
+        return false;
+    }
+    return true;
+}
+//# sourceMappingURL=api-call-history.js.map