@mevdragon/vidfarm-devcli 0.18.1 → 0.19.1

package/dist/src/devcli/migrate-local.js

@@ -0,0 +1,140 @@
+// `vidfarm migrate-local` — bridge the legacy standalone SQLite clip library
+// (~/.vidfarm/clips.db, sqlite-vec) into the UNIFIED local backend that
+// `vidfarm serve` + the file directory read (disk-backed clipRecords + storage
+// under ~/.vidfarm/data). After migrating, every locally-hunted clip is
+// browsable at /raws and semantically searchable with the SAME 768-dim vectors
+// (identical model + text builder), so nothing is re-embedded.
+//
+// Idempotent: putClip is an upsert keyed by clip_id, and a `migrated.json` marker
+// stops the lazy auto-offer from nagging. The SQLite db is left untouched.
+import path from "node:path";
+import { existsSync, writeFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+const GREEN = "\x1b[32m";
+const DIM = "\x1b[2m";
+const BOLD = "\x1b[1m";
+const RESET = "\x1b[0m";
+/** Slugify a source filename into the folder clips group under (mirrors the
+ *  serve-box source-slug so migrated clips land in the same /raws/<slug> folder). */
+function sourceSlug(filename) {
+    return String(filename || "")
+        .toLowerCase()
+        .replace(/\.[a-z0-9]+$/i, "")
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 60);
+}
+export async function runMigrateLocalCommand(argv) {
+    const { values } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            home: { type: "string" },
+            "dry-run": { type: "boolean", default: false },
+            force: { type: "boolean", default: false }
+        }
+    });
+    const { withLocalBackend, resolveLocalHome, resolveLocalDataDir } = await import("./local-backend.js");
+    const home = resolveLocalHome(values.home);
+    const dbPath = path.join(home, "clips.db");
+    if (!existsSync(dbPath)) {
+        console.log(`No local SQLite clip library at ${DIM}${dbPath}${RESET} — nothing to migrate.`);
+        return;
+    }
+    const markerPath = path.join(resolveLocalDataDir(values.home), "migrated.json");
+    if (existsSync(markerPath) && !values.force) {
+        console.log(`Already migrated (${DIM}${markerPath}${RESET}). Re-run with ${BOLD}--force${RESET} to migrate again.`);
+        return;
+    }
+    // Read the legacy SQLite store (read-only) — the ClipStore class is retained
+    // solely for this bridge.
+    const { ClipStore } = await import("./clip-store.js");
+    const store = new ClipStore(home);
+    const clips = store.listClips();
+    store.attachEmbeddings(clips);
+    const sources = store.listSourceVideos();
+    const presets = store.listPresets().filter((p) => !p.builtin);
+    const libModel = store.getLibraryEmbeddingModel();
+    if (clips.length === 0 && sources.length === 0) {
+        store.close();
+        console.log("The local SQLite library is empty — nothing to migrate.");
+        return;
+    }
+    if (values["dry-run"]) {
+        store.close();
+        console.log(`${BOLD}migrate-local (dry run)${RESET}  ${DIM}${home}${RESET}`);
+        console.log(`  ${clips.length} clip${clips.length === 1 ? "" : "s"} · ${sources.length} source${sources.length === 1 ? "" : "s"} · ${presets.length} custom preset${presets.length === 1 ? "" : "s"}`);
+        console.log(`  embedding model: ${libModel ?? DIM + "(none — clips keyword-only)" + RESET}`);
+        console.log(`  → would write into the unified backend at ${DIM}${resolveLocalDataDir(values.home)}${RESET}`);
+        return;
+    }
+    // Boot the local backend (sets drivers + resolves the stable owner id), then
+    // reach the disk-backed services directly.
+    const backend = await withLocalBackend({ home: values.home });
+    const owner = backend.ownerId;
+    const { clipRecords } = await import("../services/clip-records.js");
+    const { storage } = await import("../services/storage.js");
+    const now = new Date().toISOString();
+    // Sources first (so /raws folder grouping + source lookups resolve).
+    let sourceCount = 0;
+    for (const s of sources) {
+        await clipRecords.putSource({
+            source_video_id: s.source_video_id,
+            owner_id: owner,
+            filename: s.filename,
+            raw_s3_uri: s.source_path ?? undefined,
+            duration_sec: s.duration_sec ?? undefined,
+            clip_count: s.clip_count ?? 0,
+            tier: s.tier ?? undefined,
+            status: "complete",
+            created_at: s.scanned_at ?? now,
+            updated_at: s.scanned_at ?? now
+        });
+        sourceCount += 1;
+    }
+    // Clips: copy the mp4/thumb bytes into the storage driver at the canonical
+    // unified keys, then upsert the record carrying its embedding verbatim.
+    let clipCount = 0;
+    let withVideo = 0;
+    for (const clip of clips) {
+        const clipId = clip.clip_id;
+        const srcMp4 = path.join(store.paths.clipsDir, `${clipId}.mp4`);
+        const clipKey = `clips/${owner}/${clipId}.mp4`;
+        if (existsSync(srcMp4)) {
+            await storage.putFile(clipKey, srcMp4, "video/mp4");
+            withVideo += 1;
+        }
+        const srcThumb = path.join(store.paths.thumbsDir, `${clipId}.jpg`);
+        let thumbKey = "";
+        if (existsSync(srcThumb)) {
+            thumbKey = `thumbs/${owner}/${clipId}.jpg`;
+            await storage.putFile(thumbKey, srcThumb, "image/jpeg");
+        }
+        const unified = {
+            ...clip,
+            owner_id: owner,
+            file_path: clipKey,
+            thumbnail_path: thumbKey,
+            folder_path: clip.folder_path && clip.folder_path.trim() ? clip.folder_path : sourceSlug(clip.source_filename),
+            // Vectors are the same 768-dim space; stamp the library model so cloud/
+            // local search stays cosine-comparable.
+            ...(clip.embedding && clip.embedding.length && !clip.embedding_model && libModel
+                ? { embedding_model: libModel }
+                : {})
+        };
+        await clipRecords.putClip(unified);
+        clipCount += 1;
+    }
+    // Custom presets (built-ins are seeded on both sides already).
+    let presetCount = 0;
+    for (const p of presets) {
+        await clipRecords.putPreset(owner, p);
+        presetCount += 1;
+    }
+    store.close();
+    writeFileSync(markerPath, JSON.stringify({ migrated_at: now, clips: clipCount, sources: sourceCount, presets: presetCount, embedding_model: libModel }, null, 2));
+    console.log(`${GREEN}✓ Migrated the local clip library into the unified backend.${RESET}`);
+    console.log(`  ${clipCount} clip${clipCount === 1 ? "" : "s"} (${withVideo} with video) · ${sourceCount} source${sourceCount === 1 ? "" : "s"} · ${presetCount} preset${presetCount === 1 ? "" : "s"}`);
+    console.log(`  ${DIM}Browse them:${RESET} vidfarm directory ls /raws   ${DIM}or${RESET}   vidfarm serve  →  /library/raws`);
+}
+//# sourceMappingURL=migrate-local.js.map

package/dist/src/devcli/sync.js

@@ -0,0 +1,311 @@
+// `vidfarm sync` — move files between the LOCAL disk backend (what `vidfarm
+// serve` + the file directory read) and the CLOUD (vidfarm.cc), with filepath
+// conflict resolution. This is how an AI agent hands local work up to the cloud
+// after a long local-first editing / clip-hunting session:
+//
+//   vidfarm sync push            local My Files → cloud   (upload local work)
+//   vidfarm sync pull            cloud My Files → local   (bring cloud assets down)
+//   vidfarm sync push /files/brand   scope to a subtree
+//   --dry-run                    list the deltas + conflicts, transfer nothing
+//   --on-conflict newest|skip|keep-both|local|cloud   (default: prompt; agents = newest)
+//
+// Scope: the /files (My Files) root — the durable, vector-searchable per-user
+// asset store. Both sides share the canonical /files path, so "same path" is the
+// collision key. Raws (/raws) sync is a documented follow-up (it needs the
+// source-ingest pipeline, not a plain byte copy).
+import { parseArgs } from "node:util";
+import { homedir } from "node:os";
+import path from "node:path";
+import { createInterface } from "node:readline";
+const GREEN = "\x1b[32m";
+const YELLOW = "\x1b[33m";
+const RED = "\x1b[31m";
+const DIM = "\x1b[2m";
+const BOLD = "\x1b[1m";
+const RESET = "\x1b[0m";
+const DEFAULT_HOST = "https://vidfarm.cc";
+function authHeaders(apiKey) {
+    const h = { accept: "application/json" };
+    if (apiKey)
+        h["vidfarm-api-key"] = apiKey;
+    return h;
+}
+// ── transport: one request/upload/delete per space ──────────────────────────
+async function req(ctx, space, input) {
+    if (space === "local") {
+        const { localApiRequest } = await import("./local-backend.js");
+        const r = await localApiRequest({ ...input, auth: { apiKey: ctx.apiKey }, home: ctx.home });
+        return { status: r.status, ok: r.ok, json: r.json };
+    }
+    const url = new URL(input.path, ctx.host);
+    for (const [k, v] of Object.entries(input.query ?? {}))
+        if (v)
+            url.searchParams.set(k, v);
+    const headers = authHeaders(ctx.apiKey);
+    let body;
+    if (input.body !== undefined) {
+        headers["content-type"] = "application/json";
+        body = JSON.stringify(input.body);
+    }
+    const res = await fetch(url, { method: input.method.toUpperCase(), headers, body });
+    const text = await res.text();
+    let json = null;
+    try {
+        json = text ? JSON.parse(text) : null;
+    }
+    catch {
+        json = null;
+    }
+    return { status: res.status, ok: res.ok, json };
+}
+/** Fetch the raw bytes of a file item from its own space. */
+async function readBytes(ctx, space, item) {
+    if (!item.viewUrl)
+        throw new Error(`No view URL for ${item.path}`);
+    if (space === "local") {
+        const { withLocalBackend } = await import("./local-backend.js");
+        const backend = await withLocalBackend({ home: ctx.home, apiKey: ctx.apiKey });
+        const u = new URL(item.viewUrl, "http://vidfarm.local");
+        const res = await backend.app.request(u.pathname + u.search, { headers: authHeaders(ctx.apiKey) });
+        if (!res.ok)
+            throw new Error(`local read ${item.path} → HTTP ${res.status}`);
+        return Buffer.from(await res.arrayBuffer());
+    }
+    const res = await fetch(item.viewUrl, { headers: authHeaders(ctx.apiKey) });
+    if (!res.ok)
+        throw new Error(`cloud read ${item.path} → HTTP ${res.status}`);
+    return Buffer.from(await res.arrayBuffer());
+}
+/** Upload bytes into a space's My Files at folderPath/fileName (multipart — the
+ *  one transport that works on both S3 and local storage). */
+async function uploadBytes(ctx, space, input) {
+    const form = new FormData();
+    const blob = new Blob([new Uint8Array(input.bytes)], { type: input.contentType || "application/octet-stream" });
+    form.set("file", blob, input.fileName);
+    if (input.folderPath)
+        form.set("folder_path", input.folderPath);
+    if (input.notes)
+        form.set("notes", input.notes);
+    const p = "/api/v1/user/me/attachments/upload";
+    if (space === "local") {
+        const { withLocalBackend } = await import("./local-backend.js");
+        const backend = await withLocalBackend({ home: ctx.home, apiKey: ctx.apiKey });
+        const res = await backend.app.request(p, { method: "POST", headers: authHeaders(ctx.apiKey), body: form });
+        if (!res.ok)
+            throw new Error(`local upload ${input.folderPath}/${input.fileName} → HTTP ${res.status}`);
+        return;
+    }
+    const res = await fetch(new URL(p, ctx.host), { method: "POST", headers: authHeaders(ctx.apiKey), body: form });
+    if (!res.ok)
+        throw new Error(`cloud upload ${input.folderPath}/${input.fileName} → HTTP ${res.status} ${(await res.text()).slice(0, 200)}`);
+}
+async function deleteAttachment(ctx, space, id) {
+    await req(ctx, space, { method: "DELETE", path: `/api/v1/user/me/attachments/${encodeURIComponent(id)}` }).catch(() => undefined);
+}
+// ── enumerate a root recursively as a flat file list ────────────────────────
+function toFileItem(raw) {
+    const meta = raw?.meta ?? {};
+    const updatedAtRaw = meta.updatedAt ?? meta.updated_at ?? raw.updatedAt ?? raw.createdAt ?? meta.createdAt;
+    const updatedAt = updatedAtRaw ? Date.parse(String(updatedAtRaw)) : undefined;
+    return {
+        path: String(raw?.path ?? ""),
+        name: String(raw?.name ?? ""),
+        folderPath: String(raw?.folderPath ?? ""),
+        id: raw?.id,
+        contentType: raw?.contentType,
+        sizeBytes: typeof raw?.sizeBytes === "number" ? raw.sizeBytes : undefined,
+        viewUrl: raw?.viewUrl,
+        updatedAt: Number.isFinite(updatedAt) ? updatedAt : undefined,
+        notes: typeof meta.notes === "string" ? meta.notes : undefined
+    };
+}
+async function enumerateFiles(ctx, space, rootPath) {
+    const out = [];
+    const queue = [rootPath];
+    const seen = new Set();
+    while (queue.length) {
+        const dir = queue.shift();
+        if (seen.has(dir))
+            continue;
+        seen.add(dir);
+        const r = await req(ctx, space, { method: "GET", path: "/api/v1/user/me/directory", query: { path: dir, limit: "500" } });
+        if (!r.ok)
+            throw new Error(`${space} directory ls ${dir} → HTTP ${r.status}${r.json?.error ? ` (${r.json.error})` : ""}`);
+        for (const f of Array.isArray(r.json?.folders) ? r.json.folders : []) {
+            if (f?.path)
+                queue.push(String(f.path));
+        }
+        for (const f of Array.isArray(r.json?.files) ? r.json.files : []) {
+            out.push(toFileItem(f));
+        }
+    }
+    return out;
+}
+async function resolveConflict(policy, src, dst, direction) {
+    const srcIsLocal = direction === "push";
+    switch (policy) {
+        case "skip":
+            return "skip";
+        case "keep-both":
+            return "keep-both";
+        case "local":
+            return srcIsLocal ? "transfer" : "skip";
+        case "cloud":
+            return srcIsLocal ? "skip" : "transfer";
+        case "newest": {
+            const s = src.updatedAt ?? 0;
+            const d = dst.updatedAt ?? 0;
+            if (s === d)
+                return "skip";
+            return s > d ? "transfer" : "skip";
+        }
+        case "prompt":
+        default: {
+            if (!process.stdin.isTTY) {
+                // Non-interactive (AI agent): newest-wins.
+                return resolveConflict("newest", src, dst, direction);
+            }
+            const answer = await prompt(`  ${YELLOW}conflict${RESET} ${src.path}\n    keep [${BOLD}s${RESET}ource=${srcIsLocal ? "local" : "cloud"}] · [${BOLD}k${RESET}eep target] · [${BOLD}b${RESET}oth] · [s${BOLD}K${RESET}ip]? `);
+            const a = answer.trim().toLowerCase();
+            if (a === "s" || a === "source")
+                return "transfer";
+            if (a === "b" || a === "both")
+                return "keep-both";
+            return "skip";
+        }
+    }
+}
+function prompt(q) {
+    const rl = createInterface({ input: process.stdin, output: process.stdout });
+    return new Promise((resolve) => rl.question(q, (a) => { rl.close(); resolve(a); }));
+}
+// Suffix a file name before its extension: "logo.png" → "logo (cloud).png".
+function suffixName(name, suffix) {
+    const dot = name.lastIndexOf(".");
+    if (dot <= 0)
+        return `${name} (${suffix})`;
+    return `${name.slice(0, dot)} (${suffix})${name.slice(dot)}`;
+}
+// ── command ─────────────────────────────────────────────────────────────────
+export async function runSyncCommand(argv) {
+    const direction = argv[0];
+    if (direction !== "push" && direction !== "pull") {
+        console.log(SYNC_HELP);
+        if (direction && direction !== "help" && direction !== "--help" && direction !== "-h")
+            process.exitCode = 1;
+        return;
+    }
+    const { values, positionals } = parseArgs({
+        args: argv.slice(1),
+        allowPositionals: true,
+        options: {
+            host: { type: "string", default: DEFAULT_HOST },
+            "api-key": { type: "string" },
+            home: { type: "string" },
+            "dry-run": { type: "boolean", default: false },
+            "on-conflict": { type: "string" }
+        }
+    });
+    const policy = normalizePolicy(values["on-conflict"]);
+    const ctx = {
+        host: String(values.host ?? DEFAULT_HOST).replace(/\/+$/, ""),
+        apiKey: values["api-key"] ?? process.env.VIDFARM_API_KEY,
+        home: values.home
+    };
+    // Scope: a /files subtree (default the whole root).
+    let rootPath = (positionals[0] ?? "/files").trim();
+    if (!rootPath.startsWith("/files")) {
+        throw new Error(`sync currently covers the /files (My Files) root — got "${rootPath}". Raws sync is a follow-up.`);
+    }
+    const srcSpace = direction === "push" ? "local" : "cloud";
+    const dstSpace = direction === "push" ? "cloud" : "local";
+    const homeLabel = ctx.home ?? process.env.VIDFARM_HOME ?? path.join(homedir(), ".vidfarm");
+    console.log(`${BOLD}sync ${direction}${RESET}  ${DIM}${srcSpace} → ${dstSpace}${RESET}  ${DIM}scope=${rootPath} · local=${homeLabel} · cloud=${ctx.host} · on-conflict=${policy}${values["dry-run"] ? " · DRY RUN" : ""}${RESET}\n`);
+    const [srcFiles, dstFiles] = await Promise.all([enumerateFiles(ctx, srcSpace, rootPath), enumerateFiles(ctx, dstSpace, rootPath)]);
+    const dstByPath = new Map(dstFiles.map((f) => [f.path, f]));
+    let transferred = 0;
+    let skipped = 0;
+    let conflicts = 0;
+    let inSync = 0;
+    for (const src of srcFiles) {
+        const dst = dstByPath.get(src.path);
+        if (!dst) {
+            // New file on the target → transfer.
+            if (values["dry-run"]) {
+                console.log(`  ${GREEN}+ new${RESET}     ${src.path}`);
+            }
+            else {
+                await transfer(ctx, srcSpace, dstSpace, src, src.name);
+                console.log(`  ${GREEN}✓ sent${RESET}    ${src.path}`);
+            }
+            transferred += 1;
+            continue;
+        }
+        // Same path both sides. Equal size → treat as already in sync (cheap, no
+        // byte compare); differing size → a real conflict.
+        if (src.sizeBytes != null && dst.sizeBytes != null && src.sizeBytes === dst.sizeBytes) {
+            inSync += 1;
+            continue;
+        }
+        conflicts += 1;
+        const resolution = await resolveConflict(policy, src, dst, direction);
+        if (resolution === "skip") {
+            console.log(`  ${DIM}– skip${RESET}    ${src.path} ${DIM}(conflict)${RESET}`);
+            skipped += 1;
+            continue;
+        }
+        if (values["dry-run"]) {
+            console.log(`  ${YELLOW}~ ${resolution === "keep-both" ? "both" : "over"}${RESET}   ${src.path}`);
+            transferred += 1;
+            continue;
+        }
+        if (resolution === "keep-both") {
+            await transfer(ctx, srcSpace, dstSpace, src, suffixName(src.name, srcSpace));
+            console.log(`  ${YELLOW}✓ both${RESET}    ${src.path} ${DIM}→ ${suffixName(src.name, srcSpace)}${RESET}`);
+        }
+        else {
+            // Overwrite: drop the target copy first so we replace rather than dup.
+            if (dst.id)
+                await deleteAttachment(ctx, dstSpace, dst.id);
+            await transfer(ctx, srcSpace, dstSpace, src, src.name);
+            console.log(`  ${GREEN}✓ over${RESET}    ${src.path}`);
+        }
+        transferred += 1;
+    }
+    console.log(`\n  ${transferred} transferred · ${inSync} already in sync · ${conflicts} conflict${conflicts === 1 ? "" : "s"} · ${skipped} skipped`);
+    if (values["dry-run"])
+        console.log(`  ${DIM}(dry run — nothing was written)${RESET}`);
+}
+async function transfer(ctx, from, to, item, fileName) {
+    const bytes = await readBytes(ctx, from, item);
+    await uploadBytes(ctx, to, {
+        fileName,
+        folderPath: item.folderPath.replace(/^files\/?/, "").replace(/^\/+|\/+$/g, ""),
+        contentType: item.contentType,
+        notes: item.notes,
+        bytes
+    });
+}
+function normalizePolicy(v) {
+    const s = String(v ?? "").trim().toLowerCase();
+    if (["newest", "skip", "keep-both", "local", "cloud"].includes(s))
+        return s;
+    if (s === "keepboth" || s === "both")
+        return "keep-both";
+    return "prompt";
+}
+const SYNC_HELP = `vidfarm sync — move My Files between the local disk backend and the cloud
+
+  sync push [path]     local → cloud   (upload local work; default scope /files)
+  sync pull [path]     cloud → local   (bring cloud assets down)
+
+  --dry-run                     list deltas + conflicts, transfer nothing
+  --on-conflict <policy>        newest | skip | keep-both | local | cloud
+                                (default: prompt; non-interactive/agent = newest)
+  --host <url>                  cloud host (default https://vidfarm.cc)
+  --api-key <key>               or VIDFARM_API_KEY
+  --home <dir>                  local backend home (or VIDFARM_HOME; default ~/.vidfarm)
+
+  Collision key = the canonical /files path. Equal size ⇒ already in sync.
+  Raws (/raws) sync is a follow-up — it needs the source-ingest pipeline.`;
+//# sourceMappingURL=sync.js.map