@vfrey/vfrey 0.0.1 → 1.0.0-beta.1

package/src/cli.ts

@@ -0,0 +1,468 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { runDesk } from "./desk/cli.js";
+import { deskAccounts, seedDesk } from "./desk/series.js";
+import { runDeskTick, TickBusyError } from "./desk/tick.js";
+import { packageVersion, runSkillCommand, staleAgentSkills, staleSkillWarning } from "./skill-install.js";
+import { runProductCommand } from "./product/commands.js";
+import { loadConfig, mockWriterRefusal } from "./config.js";
+import { unavailableMessage } from "./product/engine-choice.js";
+import { engineById, engineStatus } from "./product/engines.js";
+import { defaultProbes } from "./product/probes.js";
+import { drain } from "./drain.js";
+import { formatDoctor, runDoctor } from "./doctor.js";
+import { localVoiceProblem } from "./tts.js";
+import { commandHelp, generalHelp } from "./help.js";
+import { deskOf } from "./dashboard-link.js";
+import { dataDirFor, resolveHome } from "./home.js";
+import { runInit, runProjects, runVideos } from "./project/commands.js";
+import { touchProject } from "./project/index-file.js";
+import { runService } from "./service/commands.js";
+import { runAllProjectsWorker } from "./service/worker.js";
+import { LOCK_BUSY_EXIT, acquireLockForProcess, busyMessage } from "./service/lock.js";
+import { runOpen } from "./project/open.js";
+import { ProjectError } from "./project/marker.js";
+import { projectJson, projectNotice } from "./project/notice.js";
+import { COMPLIANCE_CHECKLIST } from "@vfrey/contracts";
+import { VARIATION_CHECKLIST } from "@vfrey/contracts";
+import { createLlmClient } from "./llm/client.js";
+import { scriptTemplate, validateScript, type ScriptFile } from "./generate/script.js";
+import { log } from "./log.js";
+import { runLogin, runLogout, runOrg, runWhoami } from "./api/commands.js";
+import { runTopics } from "./api/topics.js";
+import { runCreators } from "./api/creators.js";
+import { runMail } from "./api/mail.js";
+import { runCloud } from "./api/cloud.js";
+import { checkScene, SceneError } from "./scene/scene.js";
+import { writeNow } from "./sync-write.js";
+import { defaultRender, tick, tickJobs, type Runtime } from "./pipeline/run.js";
+import { Store } from "./queue/store.js";
+import { attachRunUrls, buildTickReport, formatTickReport } from "./tick-report.js";
+
+const COMMANDS = ["doctor", "tick", "worker", "engage", "analytics", "render", "engine", "dashboard", "dev", "push", "update", "desk", "skill", "scene", "init", "videos", "projects", "open", "service", "login", "logout", "whoami", "org", "topics", "creators", "mail", "cloud"];
+
+const command = process.argv[2] ?? "doctor";
+
+const dryFlag = process.argv.includes("--dry-run");
+
+/** The CLI's refusal style: a cross line, a Fix line, and a non-zero exit. */
+function refuse(message: string, fix: string): void {
+  process.stderr.write(`✗ ${message}\nFix: ${fix}\n`);
+  process.exitCode = 1;
+}
+
+async function main(): Promise<void> {
+  const rest = process.argv.slice(3);
+  // Internal: the render worker's entry (docker/render-worker). Not listed in help; it reads no project or ~/.vfrey state.
+  if (command === "render-job") {
+    const { runRenderJobCommand } = await import("./cloud/render-job.js");
+    process.exitCode = await runRenderJobCommand(rest, { env: process.env, stdout: (text) => writeNow(1, text), stderr: (text) => writeNow(2, text) });
+    return;
+  }
+  if (command === "--version" || command === "-v" || command === "version") {
+    process.stdout.write(`${packageVersion()}\n`);
+    return;
+  }
+  if (command === "--help" || command === "-h" || command === "help") {
+    const topic = command === "help" ? rest[0] : undefined;
+    process.stdout.write((topic && commandHelp(topic)) || generalHelp());
+    return;
+  }
+  if (COMMANDS.includes(command) && (rest.includes("--help") || rest.includes("-h"))) {
+    process.stdout.write(commandHelp(command) ?? generalHelp());
+    return;
+  }
+  if (!COMMANDS.includes(command)) {
+    refuse(`Unknown command "${command}".`, `use one of: ${COMMANDS.join(", ")}.`);
+    return;
+  }
+  // Which data folder this command works on. A command that makes or lists videos says so when it is the machine desk.
+  const home = resolveHome(process.env);
+  if (home.project) touchProject(home.machineHome, home.project);
+  const makesVideos = command === "tick" || (command === "worker" && !rest.includes("--all-projects")) || command === "render" || (command === "desk" && rest[0] === "tick");
+  if (makesVideos && !rest.includes("--script-template")) {
+    const notice = projectNotice(home);
+    if (notice) process.stderr.write(notice);
+  }
+  if (command === "login" || command === "logout" || command === "whoami" || command === "org" || command === "topics" || command === "creators" || command === "mail" || command === "cloud") {
+    const io = { env: process.env, stdout: (text: string) => writeNow(1, text), stderr: (text: string) => writeNow(2, text) };
+    process.exitCode = await (command === "login" ? runLogin(rest, io) : command === "logout" ? runLogout(rest, io) : command === "org" ? runOrg(rest, io) : command === "topics" ? runTopics(rest, io) : command === "creators" ? runCreators(rest, io) : command === "mail" ? runMail(rest, io) : command === "cloud" ? runCloud(rest, { ...io, cwd: process.cwd() }) : runWhoami(rest, io));
+    return;
+  }
+  if (command === "open") {
+    process.exitCode = await runOpen(rest, { env: process.env, cwd: process.cwd(), stdout: (text: string) => process.stdout.write(text), stderr: (text: string) => process.stderr.write(text) });
+    return;
+  }
+  if (command === "init" || command === "videos" || command === "projects") {
+    const io = { env: process.env, cwd: process.cwd(), stdout: (text: string) => process.stdout.write(text), stderr: (text: string) => process.stderr.write(text) };
+    process.exitCode = command === "init" ? runInit(rest, io) : command === "videos" ? runVideos(rest, io) : await runProjects(rest, io);
+    return;
+  }
+  if (command === "service") {
+    process.exitCode = await runService(rest, { env: process.env, cwd: process.cwd(), stdout: (text: string) => process.stdout.write(text), stderr: (text: string) => process.stderr.write(text) });
+    return;
+  }
+  if (command === "worker" && rest.includes("--all-projects")) {
+    process.exitCode = await runAllProjectsWorker({
+      machineHome: home.machineHome,
+      env: process.env,
+      dryRun: dryFlag,
+      once: rest.includes("--once"),
+      echo: Boolean(process.stderr.isTTY),
+      stderr: (text) => process.stderr.write(text),
+    });
+    return;
+  }
+  if (command !== "skill" && command !== "doctor" && !process.env.VFREY_NO_SKILL_CHECK) {
+    try {
+      for (const stale of staleAgentSkills(process.env, process.cwd())) process.stderr.write(staleSkillWarning(stale));
+    } catch {
+      /* the warning is a courtesy: never block a command on it */
+    }
+  }
+  if (command === "skill") {
+    process.exitCode = runSkillCommand(process.argv.slice(3), {
+      env: process.env,
+      cwd: process.cwd(),
+      stdout: (text) => process.stdout.write(text),
+      stderr: (text) => process.stderr.write(text),
+    });
+    return;
+  }
+  if (command === "desk") {
+    const rest = process.argv.slice(3);
+    if (rest[0] === "serve") {
+      process.exitCode = await runProductCommand("dashboard", rest.slice(1), {
+        env: process.env,
+        stdout: (text) => writeNow(1, text),
+        stderr: (text) => writeNow(2, text),
+      });
+      return;
+    }
+    const dataDir = dataDirFor(process.env);
+    const store = new Store(path.join(dataDir, "pipeline.sqlite"));
+    if (rest[0] === "tick") {
+      try {
+        const count = await runDeskTick({ store, env: process.env, argv: rest });
+        process.stdout.write(`processed ${count}\n`);
+      } catch (error) {
+        if (error instanceof TickBusyError) {
+          const busy = busyMessage(error.holder);
+          refuse(busy.message, busy.fix);
+          process.exitCode = LOCK_BUSY_EXIT;
+          return;
+        }
+        throw error;
+      } finally {
+        store.close();
+      }
+      return;
+    }
+    const code = await runDesk(rest, {
+      env: process.env,
+      stdout: (text) => process.stdout.write(text),
+      stderr: (text) => process.stderr.write(text),
+      readStdin: async () => {
+        if (process.stdin.isTTY) return "";
+        const chunks: Buffer[] = [];
+        for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
+        return Buffer.concat(chunks).toString("utf8");
+      },
+    }, store);
+    store.close();
+    process.exitCode = code;
+    return;
+  }
+  if (command === "render" || command === "scene" || command === "engine" || command === "dev" || command === "dashboard" || command === "push" || command === "update") {
+    const code = await runProductCommand(command, process.argv.slice(3), {
+      env: process.env,
+      stdout: (text) => writeNow(1, text),
+      stderr: (text) => writeNow(2, text),
+    });
+    process.exitCode = code;
+    return;
+  }
+  if (command === "doctor") {
+    doctor(process.argv.includes("--json"), process.argv.includes("--rules"));
+    return;
+  }
+  const env = process.env;
+  let about: string | undefined;
+  const aboutAt = process.argv.indexOf("--about");
+  if (aboutAt >= 0) {
+    if (command !== "tick") {
+      refuse("--about works with tick only.", 'run `npx vfrey tick --now --about "<subject>"`.');
+      return;
+    }
+    about = (process.argv[aboutAt + 1] ?? "").trim();
+    if (!about || about.startsWith("--")) {
+      refuse("--about needs a subject.", 'pass the topic in quotes, for example `npx vfrey tick --now --about "why octopuses have three hearts"`.');
+      return;
+    }
+  }
+  let script: ScriptFile | undefined;
+  if (process.argv.includes("--script-template")) {
+    if (command !== "tick") {
+      refuse("--script-template works with tick only.", "run `npx vfrey tick --script-template`.");
+      return;
+    }
+    process.stdout.write(scriptTemplate());
+    return;
+  }
+  const scriptAt = process.argv.indexOf("--script");
+  if (scriptAt >= 0) {
+    if (command !== "tick") {
+      refuse("--script works with tick only.", 'run `npx vfrey tick --now --about "<subject>" --script script.json`.');
+      return;
+    }
+    const file = process.argv[scriptAt + 1] ?? "";
+    if (!file || file.startsWith("--")) {
+      refuse("--script needs a file.", "pass the JSON file you wrote; `npx vfrey tick --script-template` prints an example.");
+      return;
+    }
+    let text: string;
+    try {
+      text = readFileSync(path.resolve(file), "utf8");
+    } catch {
+      refuse(`Cannot read the script file ${file}.`, "check the path, or write the file first; `npx vfrey tick --script-template` prints an example.");
+      return;
+    }
+    const checked = validateScript(text);
+    if (!checked.ok) {
+      refuse(`${file}: ${checked.message}`, checked.fix);
+      return;
+    }
+    script = checked.script;
+  }
+  let sceneDir: string | undefined;
+  const sceneAt = process.argv.indexOf("--scene");
+  if (sceneAt >= 0) {
+    if (command !== "tick") {
+      refuse("--scene works with tick and render only.", "run `npx vfrey tick --now --about \"<subject>\" --script script.json --scene DIR`.");
+      return;
+    }
+    const value = process.argv[sceneAt + 1] ?? "";
+    if (!value || value.startsWith("--")) {
+      refuse("--scene needs a folder.", "pass the folder that holds your index.html (or index.tsx for Remotion); `npx vfrey scene init DIR` writes a starting point.");
+      return;
+    }
+    sceneDir = path.resolve(value);
+  }
+  let engineFlag: string | undefined;
+  const engineAt = process.argv.indexOf("--engine");
+  if (engineAt >= 0) {
+    if (command !== "tick") {
+      refuse("--engine works with tick and render only.", "run `npx vfrey tick --now --engine vfrey-framey`, or `npx vfrey render --engine remotion`.");
+      return;
+    }
+    const value = process.argv[engineAt + 1] ?? "";
+    try {
+      engineFlag = engineById(value.startsWith("--") ? "" : value).id;
+    } catch {
+      refuse(`"${value}" is not an engine.`, "choose remotion, hyperframes, vfrey-framey, or comfyui; `npx vfrey engine list` shows which are ready.");
+      return;
+    }
+    if (sceneDir) {
+      try {
+        await checkScene(sceneDir, engineFlag);
+      } catch (error) {
+        if (error instanceof SceneError) {
+          refuse(error.message, error.fix);
+          return;
+        }
+        throw error;
+      }
+    }
+    const probes = await defaultProbes(process.env);
+    const chosen = engineById(engineFlag);
+    if (!dryFlag && !process.argv.includes("--skip-render") && engineStatus(chosen, probes).availability === "unavailable") {
+      const message = unavailableMessage(chosen, probes);
+      const [what, fix] = message.split(" Fix: ");
+      refuse(what ?? message, fix ?? "run `npx vfrey doctor`.");
+      return;
+    }
+  }
+  const config = loadConfig(env);
+  const dryRun = dryFlag || config.dryRun;
+  const mock = mockWriterRefusal(config, env, dryRun);
+  if (mock) {
+    refuse(mock.message, mock.fix);
+    return;
+  }
+  if (!dryRun && !process.argv.includes("--skip-render") && (config.tts.provider as string) === "local") {
+    const voice = localVoiceProblem(env);
+    if (voice) {
+      refuse(voice.message, voice.fix);
+      return;
+    }
+  }
+  const store = new Store(path.join(config.dataDir, "pipeline.sqlite"));
+  const runtime: Runtime = {
+    config,
+    env,
+    store,
+    llm: createLlmClient(config.llm),
+    fetchImpl: fetch,
+    now: () => new Date(),
+    dryRun,
+    skipRender: dryRun || process.argv.includes("--skip-render"),
+    forceNow: process.argv.includes("--now") || about !== undefined || script !== undefined,
+    about,
+    script,
+    project: home.project,
+    engine: engineFlag,
+    render: (content, outputPath, engine, scene) => defaultRender(content, outputPath, config.dataDir, engine, env, scene),
+    scene: sceneDir,
+  };
+  if (command === "tick") {
+    // A project is ticked by one process at a time: the background worker, `tick` and `desk tick` share this lock.
+    const lock = acquireLockForProcess(config.dataDir);
+    if (!lock.ok) {
+      const busy = busyMessage(lock.holder);
+      store.close();
+      refuse(busy.message, busy.fix);
+      process.exitCode = LOCK_BUSY_EXIT;
+      return;
+    }
+    if (!runtime.forceNow) {
+      const note = "Running the series cadence: a video is made only for a series that is due. Use `--now` to make a video now.\n";
+      (process.argv.includes("--json") ? process.stderr : process.stdout).write(note);
+    }
+    runtime.notes = [];
+    // Ctrl-C hands the lease back at once instead of leaving the job silent for 20 minutes.
+    let interrupted = false;
+    const interrupt = (signal: string) => {
+      // The launcher forwards the signal the terminal already sent, so a second one is expected.
+      if (interrupted) return;
+      interrupted = true;
+      log("info", "tick interrupted", { signal });
+      store.releaseOwned();
+      store.close();
+      process.exit(signal === "SIGINT" ? 130 : 143);
+    };
+    process.on("SIGINT", () => interrupt("SIGINT"));
+    process.on("SIGTERM", () => interrupt("SIGTERM"));
+    // The shared packages and Remotion print progress and JSON log lines to stdout. Here stdout belongs to
+    // the summary, so everything written while the pipeline runs goes to stderr.
+    const writeOut = process.stdout.write.bind(process.stdout);
+    process.stdout.write = ((chunk: string | Uint8Array, ...args: unknown[]) =>
+      (process.stderr.write as (...a: unknown[]) => boolean)(chunk, ...args)) as typeof process.stdout.write;
+    let ids: string[];
+    try {
+      ids = await tickJobs(runtime);
+    } finally {
+      lock.release();
+    }
+    process.stdout.write = writeOut as typeof process.stdout.write;
+    log("info", "tick complete", { processed: ids.length, dryRun });
+    const report = buildTickReport(
+      store,
+      ids,
+      ids.length === 0 ? runtime.notes : [],
+      script ? "agent" : config.llm.provider === "mock" || config.llm.provider === "none" ? "stand-in" : "model",
+      config.dataDir,
+    );
+    report.project = projectJson(home);
+    await attachRunUrls(report, deskOf(home), env, (text) => process.stderr.write(text));
+    process.stdout.write(process.argv.includes("--json") ? JSON.stringify(report, null, 2) + "\n" : formatTickReport(report, { forced: runtime.forceNow, home: config.dataDir }));
+  } else if (command === "worker") {
+    log("info", "worker started", { dryRun });
+    let stopped = false;
+    let wake: (() => void) | null = null;
+    const stop = (signal: string) => {
+      if (stopped) return;
+      stopped = true;
+      log("info", "worker stopping", { signal });
+      wake?.();
+    };
+    process.on("SIGINT", () => stop("SIGINT"));
+    process.on("SIGTERM", () => stop("SIGTERM"));
+    runtime.shouldStop = () => stopped;
+    while (!stopped) {
+      const lock = acquireLockForProcess(config.dataDir);
+      if (lock.ok) {
+        try {
+          await tick(runtime);
+        } finally {
+          lock.release();
+        }
+        const waiting = store.awaitingReviewCount();
+        if (waiting > 0) log("info", "jobs wait for review", { waiting, fix: "npx vfrey desk job approve --id <id>, or open the Review screen" });
+      } else {
+        log("info", "worker cycle skipped: another tick holds the project lock", { pid: lock.holder.pid });
+      }
+      if (stopped) break;
+      await new Promise<void>((resolve) => {
+        const timer = setTimeout(resolve, 60_000);
+        wake = () => {
+          clearTimeout(timer);
+          resolve();
+        };
+      });
+    }
+    log("info", "worker stopped", {});
+  } else if (command === "engage") {
+    runtime.shouldStop = stopOnSignal();
+    seedDesk(store, config.accounts);
+    for (const account of deskAccounts(store)) {
+      store.enqueue({ accountId: account.id, kind: "engage", stage: "engage" });
+    }
+    await drain(runtime);
+  } else if (command === "analytics") {
+    runtime.shouldStop = stopOnSignal();
+    seedDesk(store, config.accounts);
+    for (const account of deskAccounts(store)) {
+      store.enqueue({ accountId: account.id, kind: "analytics", stage: "source" });
+    }
+    await drain(runtime);
+  }
+  store.close();
+}
+
+/** A stop flag that SIGINT and SIGTERM set, for commands that drain the queue. */
+function stopOnSignal(): () => boolean {
+  let stopped = false;
+  const stop = (signal: string) => {
+    if (stopped) return;
+    stopped = true;
+    log("info", "stopping", { signal });
+  };
+  process.on("SIGINT", () => stop("SIGINT"));
+  process.on("SIGTERM", () => stop("SIGTERM"));
+  return () => stopped;
+}
+
+function doctor(json: boolean, rules: boolean): void {
+  const report = runDoctor(process.env);
+  if (json) {
+    process.stdout.write(JSON.stringify(report, null, 2) + "\n");
+    return;
+  }
+  if (!report.ok) process.exitCode = 1;
+  if (!rules) {
+    process.stdout.write(formatDoctor(report) + "\n\nRules (variation and compliance): npx vfrey doctor --rules\n");
+    return;
+  }
+  const lines = [
+    formatDoctor(report),
+    "",
+    "Variation checklist (quality gates, not API errors):",
+    ...VARIATION_CHECKLIST.map((item) => `- [${item.id}] ${item.axis}: ${item.rule} (${item.enforcedBy})`),
+    "",
+    "Compliance checklist (platform rules):",
+    ...COMPLIANCE_CHECKLIST.map((item) => `- [${item.id}] ${item.field} @ ${item.location}`),
+    "",
+    "TikTok comment reply is not implemented. The Content Posting API has no organic comment-reply method.",
+    "YouTube's February 1, 2027 Partner Program update is a terms acceptance in YouTube Studio. Accept it there.",
+  ];
+  process.stdout.write(lines.join("\n") + "\n");
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().catch((error: unknown) => {
+    if (error instanceof ProjectError) return refuse(error.reason, error.fix);
+    const message = error instanceof Error ? error.message : String(error);
+    refuse(message, "run `npx vfrey doctor` and check the line above.");
+  });
+}

package/src/cloud/bundle.ts

@@ -0,0 +1,70 @@
+import { lstat, mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { gunzipSync, gzipSync } from "node:zlib";
+import { insidePath, JobError } from "./job.js";
+
+/**
+ * A job's input folder as ONE object (spec 019, section C2). The cloud executor uploads it to a private bucket and the
+ * job container downloads it through a signed URL, so the container needs no permission on the bucket at all.
+ *
+ * The format is gzip over JSON: `{ version: 1, files: [{ path, data }] }`, `data` base64. It carries regular files only.
+ * A symlink or any other entry is refused when packing, and every path is re-checked (relative, no `..`) when unpacking.
+ */
+export const BUNDLE_VERSION = 1;
+export const BUNDLE_LIMITS = { files: 5000, totalBytes: 512 * 1024 * 1024 } as const;
+
+interface BundleFile {
+  path: string;
+  data: string;
+}
+
+async function walk(root: string, rel: string, out: string[]): Promise<void> {
+  for (const entry of await readdir(path.join(root, rel), { withFileTypes: true })) {
+    const next = rel ? `${rel}/${entry.name}` : entry.name;
+    const info = await lstat(path.join(root, next));
+    if (info.isSymbolicLink()) throw new JobError(`"${next}" is a symbolic link; a job folder holds regular files only.`, "job_symlink");
+    if (info.isDirectory()) await walk(root, next, out);
+    else if (info.isFile()) out.push(next);
+    else throw new JobError(`"${next}" is not a regular file.`, "job_invalid");
+  }
+}
+
+/** Packs a folder into one gzip buffer. */
+export async function packFolder(root: string): Promise<Buffer> {
+  const names: string[] = [];
+  await walk(root, "", names);
+  if (names.length > BUNDLE_LIMITS.files) throw new JobError(`The job folder holds ${names.length} files; the limit is ${BUNDLE_LIMITS.files}.`, "job_too_large");
+  let total = 0;
+  const files: BundleFile[] = [];
+  for (const name of names.sort()) {
+    const data = await readFile(path.join(root, name));
+    total += data.length;
+    if (total > BUNDLE_LIMITS.totalBytes) throw new JobError("The job folder is larger than 512 MB.", "job_too_large");
+    files.push({ path: name, data: data.toString("base64") });
+  }
+  return gzipSync(Buffer.from(JSON.stringify({ version: BUNDLE_VERSION, files })));
+}
+
+/** Unpacks a bundle into an (empty or new) folder. Throws a JobError for anything that is not a plain, in-folder file. */
+export async function unpackBundle(bundle: Buffer, dest: string): Promise<number> {
+  let parsed: { version?: unknown; files?: unknown };
+  try {
+    parsed = JSON.parse(gunzipSync(bundle, { maxOutputLength: BUNDLE_LIMITS.totalBytes * 2 }).toString("utf8"));
+  } catch {
+    throw new JobError("The input bundle could not be read.", "job_unreadable");
+  }
+  if (parsed.version !== BUNDLE_VERSION || !Array.isArray(parsed.files)) throw new JobError("The input bundle has the wrong version.", "job_unreadable");
+  if (parsed.files.length > BUNDLE_LIMITS.files) throw new JobError("The input bundle holds too many files.", "job_too_large");
+  await mkdir(dest, { recursive: true });
+  const seen = new Set<string>();
+  for (const file of parsed.files as BundleFile[]) {
+    if (!file || typeof file.data !== "string") throw new JobError("The input bundle has a malformed entry.", "job_unreadable");
+    const rel = insidePath(file.path, "bundle path");
+    if (rel === "." || seen.has(rel)) throw new JobError(`The input bundle has a bad or repeated path "${file.path}".`, "job_invalid");
+    seen.add(rel);
+    const target = path.join(dest, rel);
+    await mkdir(path.dirname(target), { recursive: true });
+    await writeFile(target, Buffer.from(file.data, "base64"), { flag: "wx", mode: 0o644 });
+  }
+  return seen.size;
+}