@vfrey/vfrey 0.0.1 → 1.0.0-beta.1

package/src/api/cloud.ts

@@ -0,0 +1,413 @@
+// `vfrey cloud link | push | pull | status` (spec 019 B): sync a local project with a hosted one in the signed-in org.
+// Every call goes through ApiClient; the token never leaves it, and is never sent to the storage host.
+import { createReadStream, createWriteStream, existsSync, mkdirSync, renameSync, rmSync, statSync, utimesSync } from "node:fs";
+import { createHash } from "node:crypto";
+import os from "node:os";
+import path from "node:path";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { ApiFailure, clean, type ApiClient } from "./client.js";
+import { ApiConfigError, apiHost, resolveApiUrl } from "./config.js";
+import type { ApiIo } from "./commands.js";
+import { openSession } from "./topics.js";
+import { collectRuns } from "./cloud-runs.js";
+import {
+  MAX_FILES_PER_REQUEST,
+  MAX_PUSH_BYTES,
+  classify,
+  conflictPath,
+  formatSize,
+  readState,
+  safeTarget,
+  scanProject,
+  sha256File,
+  writeState,
+  type LocalFile,
+  type Skipped,
+  type SyncState,
+} from "./cloud-files.js";
+import type { CloudFileDecl, CloudFileView, CloudPullFile } from "./types.js";
+import { resolveHome } from "../home.js";
+import { findProject } from "../project/discover.js";
+import { readMarker, writeMarker, type Project } from "../project/marker.js";
+
+export type CloudIo = ApiIo & { cwd?: string };
+
+const USAGE = [
+  "usage: npx vfrey cloud link [--json]",
+  "       npx vfrey cloud push [--videos] [--dry-run] [--json]",
+  "       npx vfrey cloud pull [--json]",
+  "       npx vfrey cloud status [--json]",
+].join("\n");
+
+function refuse(io: CloudIo, message: string, fix: string): number {
+  io.stderr(`✗ ${message}\nFix: ${fix}\n`);
+  return 1;
+}
+
+function failure(io: CloudIo, error: unknown): number {
+  if (error instanceof ApiFailure || error instanceof ApiConfigError) return refuse(io, error.message, error.fix);
+  return refuse(io, error instanceof Error ? clean(error.message) : "Something went wrong.", "run `npx vfrey doctor` and check the line above.");
+}
+
+/** The server's answers about this project and role, in the person's words. */
+function cloudFailure(error: unknown): unknown {
+  if (!(error instanceof ApiFailure)) return error;
+  if (error.status === 404) return new ApiFailure("The cloud does not know this project in your current org.", "run `npx vfrey cloud link` (it makes or finds the project in the org you are signed in to).", 404, error.code);
+  if (error.status === 403) return new ApiFailure("Your role in this org cannot do that. Pushing and pulling need a member, admin or owner; reviewers can read.", "ask an owner or admin, or switch org with `npx vfrey org use`.", 403, error.code);
+  if (error.status === 402 || error.status === 401 || error.status === undefined || error.code === "limit_reached") return error;
+  if (error.status === 409) return new ApiFailure(error.message, "run `npx vfrey cloud push` again. Only what changed is sent.", 409, error.code);
+  if (error.status === 503) return new ApiFailure("Cloud storage is not set up on this vfrey server yet.", "try again later. Nothing was changed.", 503, error.code);
+  return error;
+}
+
+function parseArgs(args: string[], bools: string[]): { flags: Set<string>; api?: string; positionals: string[]; error?: string } {
+  const flags = new Set<string>();
+  const positionals: string[] = [];
+  let api: string | undefined;
+  for (let i = 0; i < args.length; i += 1) {
+    const arg = args[i]!;
+    if (!arg.startsWith("--")) {
+      positionals.push(arg);
+      continue;
+    }
+    const eq = arg.indexOf("=");
+    const name = eq > 0 ? arg.slice(0, eq) : arg;
+    if (bools.includes(name)) flags.add(name);
+    else if (name === "--api") {
+      const value = eq > 0 ? arg.slice(eq + 1) : args[i + 1] && !args[i + 1]!.startsWith("--") ? args[(i += 1)]! : "";
+      if (!value.trim()) return { flags, positionals, error: "--api needs a value." };
+      api = value;
+    } else return { flags, positionals, error: `Unknown option "${arg}".` };
+  }
+  return { flags, positionals, ...(api ? { api } : {}) };
+}
+
+const json = (v: unknown): string => JSON.stringify(v, null, 2) + "\n";
+
+interface Ctx {
+  project: Project;
+  client: ApiClient;
+  host: string;
+  state: SyncState;
+}
+
+function requireProject(io: CloudIo): Project | number {
+  const project = findProject(io.cwd ?? process.cwd());
+  if (!project) return refuse(io, "This folder is not inside a vfrey project.", "cd into your project folder, or start one with `npx vfrey init`.");
+  return project;
+}
+
+function open(io: CloudIo, project: Project, api?: string, needLink = true): Ctx | number {
+  const base = resolveApiUrl(io.env, api ?? null);
+  const client = openSession({ env: io.env, ...(io.homedir ? { homedir: io.homedir } : {}), ...(io.platform ? { platform: io.platform } : {}), ...(io.run ? { run: io.run } : {}), ...(io.fetch ? { fetch: io.fetch } : {}), ...(io.timeoutMs ? { timeoutMs: io.timeoutMs } : {}) }, api);
+  if (needLink && !project.cloudProjectId) return refuse(io, `${project.name} is not linked to a cloud project yet.`, "run `npx vfrey cloud link`.");
+  return { project, client, host: apiHost(base!), state: readState(project.root, project.cloudProjectId ?? "") };
+}
+
+const now = (io: CloudIo): Date => new Date(io.now ? io.now() : Date.now());
+
+function skippedNote(skipped: Skipped[]): string {
+  if (skipped.length === 0) return "";
+  const shown = skipped.slice(0, 5).map((s) => `  ${clean(s.path)} (${clean(s.reason)})`);
+  return `Not synced: ${skipped.length} file${skipped.length === 1 ? "" : "s"}\n${shown.join("\n")}${skipped.length > 5 ? `\n  and ${skipped.length - 5} more` : ""}\n`;
+}
+
+// ---------- link ----------
+
+async function link(io: CloudIo, args: string[]): Promise<number> {
+  const p = parseArgs(args, ["--json"]);
+  if (p.error || p.positionals.length) return refuse(io, p.error ?? `Unexpected "${p.positionals[0]}".`, USAGE);
+  const project = requireProject(io);
+  if (typeof project === "number") return project;
+  const ctx = open(io, project, p.api, false);
+  if (typeof ctx === "number") return ctx;
+  const { project: cloud } = await ctx.client.cloudLink({ localProjectId: project.id, name: project.name });
+  if (!cloud || typeof cloud.id !== "string") throw new ApiFailure("The vfrey server sent an answer this version does not understand.", "update vfrey with `npx vfrey update`.");
+  const changed = project.cloudProjectId !== cloud.id;
+  if (changed) writeMarker(project.root, { ...readMarker(project.root), cloudProjectId: cloud.id });
+  if (p.flags.has("--json")) io.stdout(json({ linked: true, cloudProjectId: cloud.id, name: cloud.name, created: Boolean(cloud.created), host: ctx.host }));
+  else io.stdout(`✓ ${changed ? "Linked" : "Already linked:"} ${clean(project.name)} to cloud project ${clean(cloud.id)} on ${ctx.host}${cloud.created ? " (new)" : ""}.\nNext: \`npx vfrey cloud push\` sends scenes, scripts and run details. Add --videos for the finished videos.\n`);
+  return 0;
+}
+
+// ---------- push ----------
+
+const decl = (f: LocalFile): CloudFileDecl => ({ path: f.path, size: f.size, sha256: f.sha256, modifiedAt: f.mtime.toISOString() });
+
+function batches(files: LocalFile[]): LocalFile[][] {
+  const out: LocalFile[][] = [];
+  let cur: LocalFile[] = [];
+  let bytes = 0;
+  for (const f of files) {
+    if (cur.length && (cur.length >= MAX_FILES_PER_REQUEST || bytes + f.size > MAX_PUSH_BYTES)) {
+      out.push(cur);
+      cur = [];
+      bytes = 0;
+    }
+    cur.push(f);
+    bytes += f.size;
+  }
+  if (cur.length) out.push(cur);
+  return out;
+}
+
+async function push(io: CloudIo, args: string[]): Promise<number> {
+  const p = parseArgs(args, ["--json", "--videos", "--dry-run"]);
+  if (p.error || p.positionals.length) return refuse(io, p.error ?? `Unexpected "${p.positionals[0]}".`, USAGE);
+  const project = requireProject(io);
+  if (typeof project === "number") return project;
+  const ctx = open(io, project, p.api);
+  if (typeof ctx === "number") return ctx;
+  const id = project.cloudProjectId!;
+  const dry = p.flags.has("--dry-run");
+  const withVideos = p.flags.has("--videos");
+
+  const { files: local, skipped } = await scanProject(project.root, { videos: withVideos });
+  const cloud = new Map((await ctx.client.cloudFiles(id)).files.map((f) => [f.path, f]));
+  const toPush: LocalFile[] = [];
+  const conflicts: string[] = [];
+  const unchanged: string[] = [];
+  for (const f of local) {
+    const verdict = classify(f.sha256, cloud.get(f.path)?.sha256, ctx.state.files[f.path]);
+    if (verdict === "same") unchanged.push(f.path);
+    else if (verdict === "conflict" || verdict === "pull") conflicts.push(f.path);
+    else toPush.push(f);
+  }
+  const bytes = toPush.reduce((n, f) => n + f.size, 0);
+  const videoPaths = new Set<string>([...local.filter((f) => f.kind === "video" && !conflicts.includes(f.path)).map((f) => f.path), ...[...cloud.values()].filter((f) => f.kind === "video").map((f) => f.path)]);
+  const runs = collectRuns(resolveHome(io.env, io.homedir ?? os.homedir(), project.root).dataDir, project.root, videoPaths);
+
+  if (dry) {
+    const report = { dryRun: true, wouldPush: toPush.map((f) => f.path), bytes, unchanged: unchanged.length, conflicts, skipped, runs: runs.length };
+    if (p.flags.has("--json")) io.stdout(json(report));
+    else {
+      io.stdout(`Dry run for ${clean(project.name)}: nothing was uploaded.\nWould push ${toPush.length} file${toPush.length === 1 ? "" : "s"} (${formatSize(bytes)}); ${unchanged.length} unchanged; ${runs.length} run${runs.length === 1 ? "" : "s"} of metadata.\n`);
+      for (const f of toPush.slice(0, 20)) io.stdout(`  + ${clean(f.path)} (${formatSize(f.size)})\n`);
+      if (toPush.length > 20) io.stdout(`  and ${toPush.length - 20} more\n`);
+      if (!withVideos) io.stdout("Videos stay on this machine unless you add --videos.\n");
+      io.stdout(skippedNote(skipped));
+      if (conflicts.length) io.stdout(`${conflicts.length} file${conflicts.length === 1 ? " differs" : "s differ"} in the cloud and would not be pushed: ${conflicts.slice(0, 5).map((x) => clean(x)).join(", ")}. Run \`npx vfrey cloud pull\` first.\n`);
+    }
+    return conflicts.length ? 1 : 0;
+  }
+
+  const pushed: string[] = [];
+  const failed: Array<{ path: string; reason: string }> = [];
+  for (const batch of batches(toPush)) {
+    const plan = await ctx.client.cloudPlan(id, batch.map(decl));
+    const byPath = new Map(batch.map((f) => [f.path, f]));
+    const sent: LocalFile[] = [];
+    for (const u of plan.uploads) {
+      const f = byPath.get(u.path);
+      if (!f) throw new ApiFailure("The server asked for a file that was not in this push.", "run the command again; if it repeats, update vfrey with `npx vfrey update`.");
+      const st = statSync(f.abs);
+      if (st.size !== f.size || st.mtime.getTime() !== f.mtime.getTime()) {
+        failed.push({ path: f.path, reason: "changed while pushing; run push again" });
+        continue;
+      }
+      await ctx.client.transfer("PUT", u.url, { headers: u.headers ?? {}, body: Readable.toWeb(createReadStream(f.abs)) as unknown as BodyInit, length: f.size });
+      sent.push(f);
+    }
+    for (const path_ of plan.unchanged) {
+      const f = byPath.get(path_);
+      if (f) ctx.state.files[f.path] = f.sha256;
+    }
+    if (sent.length) {
+      const answer = await ctx.client.cloudConfirm(id, sent.map(decl));
+      for (const path_ of answer.confirmed) {
+        pushed.push(path_);
+        ctx.state.files[path_] = byPath.get(path_)!.sha256;
+      }
+      failed.push(...answer.failed);
+    }
+    writeState(project.root, ctx.state);
+  }
+  for (const path_ of unchanged) ctx.state.files[path_] = local.find((f) => f.path === path_)!.sha256;
+  writeState(project.root, ctx.state);
+
+  let runCount = 0;
+  for (let i = 0; i < runs.length; i += 100) {
+    runCount += (await ctx.client.cloudPushRuns(id, runs.slice(i, i + 100))).runs;
+  }
+
+  const ok = failed.length === 0 && conflicts.length === 0;
+  if (p.flags.has("--json")) io.stdout(json({ pushed, bytes, unchanged: unchanged.length, conflicts, failed, skipped, runs: runCount }));
+  else {
+    io.stdout(`${ok ? "✓" : "!"} Pushed ${pushed.length} file${pushed.length === 1 ? "" : "s"} (${formatSize(toPush.filter((f) => pushed.includes(f.path)).reduce((n, f) => n + f.size, 0))}); ${unchanged.length} unchanged; ${runCount} run${runCount === 1 ? "" : "s"} synced.\n`);
+    if (!withVideos) io.stdout("Videos stay on this machine unless you add --videos.\n");
+    io.stdout(skippedNote(skipped));
+  }
+  if (failed.length) io.stderr(`✗ ${failed.length} file${failed.length === 1 ? " was" : "s were"} not recorded: ${failed.slice(0, 5).map((f) => `${clean(f.path)} (${clean(f.reason)})`).join(", ")}.\nFix: run \`npx vfrey cloud push\` again. Only what changed is sent.\n`);
+  if (conflicts.length) io.stderr(`✗ ${conflicts.length} file${conflicts.length === 1 ? " is" : "s are"} different in the cloud (changed there since your last sync), so ${conflicts.length === 1 ? "it was" : "they were"} not pushed: ${conflicts.slice(0, 5).map((x) => clean(x)).join(", ")}.\nFix: run \`npx vfrey cloud pull\`. The newest copy wins and the other is kept as <file>.conflict-<time>.\n`);
+  return ok ? 0 : 1;
+}
+
+// ---------- pull ----------
+
+async function download(client: ApiClient, file: CloudPullFile, dest: string): Promise<void> {
+  const response = await client.transfer("GET", file.url, { timeoutMs: 600_000 });
+  if (!response.body) throw new ApiFailure(`Storage sent no data for ${clean(file.path)}.`, "run `npx vfrey cloud pull` again.");
+  mkdirSync(path.dirname(dest), { recursive: true });
+  const tmp = `${dest}.vfrey-tmp`;
+  const hash = createHash("sha256");
+  let size = 0;
+  try {
+    await pipeline(
+      Readable.fromWeb(response.body as never),
+      new Transform({
+        transform(chunk: Buffer, _enc, cb) {
+          hash.update(chunk);
+          size += chunk.length;
+          cb(null, chunk);
+        },
+      }),
+      createWriteStream(tmp),
+    );
+    if (size !== file.size || hash.digest("hex") !== file.sha256) {
+      throw new ApiFailure(`${clean(file.path)} arrived damaged (it does not match its hash).`, "run `npx vfrey cloud pull` again.");
+    }
+    renameSync(tmp, dest);
+  } catch (error) {
+    rmSync(tmp, { force: true });
+    throw error;
+  }
+  const at = new Date(file.modifiedAt);
+  if (!Number.isNaN(at.getTime())) utimesSync(dest, at, at);
+}
+
+interface PullResult {
+  written: string[];
+  conflicts: Array<{ path: string; kept: string; winner: "cloud" | "local" }>;
+  refused: Array<{ path: string; reason: string }>;
+}
+
+async function pull(io: CloudIo, args: string[]): Promise<number> {
+  const p = parseArgs(args, ["--json"]);
+  if (p.error || p.positionals.length) return refuse(io, p.error ?? `Unexpected "${p.positionals[0]}".`, USAGE);
+  const project = requireProject(io);
+  if (typeof project === "number") return project;
+  const ctx = open(io, project, p.api);
+  if (typeof ctx === "number") return ctx;
+  const id = project.cloudProjectId!;
+  const result: PullResult = { written: [], conflicts: [], refused: [] };
+  let bytes = 0;
+  let cursor = ctx.state.cursor;
+  for (let page = 0; page < 1000; page += 1) {
+    const answer = await ctx.client.cloudPull(id, { since: cursor });
+    if (!answer || !Array.isArray(answer.files)) throw new ApiFailure("The vfrey server sent an answer this version does not understand.", "update vfrey with `npx vfrey update`.");
+    for (const f of answer.files) {
+      const target = safeTarget(project.root, f.path);
+      if (!target.ok) {
+        result.refused.push({ path: String(f.path).slice(0, 80), reason: target.reason });
+        continue;
+      }
+      const have = existsSync(target.abs);
+      const base = ctx.state.files[f.path];
+      if (have) {
+        const st = statSync(target.abs);
+        if (!st.isFile()) {
+          result.refused.push({ path: f.path, reason: "not a regular file here" });
+          continue;
+        }
+        const here = await sha256File(target.abs);
+        if (here === f.sha256) {
+          ctx.state.files[f.path] = f.sha256;
+          continue;
+        }
+        const verdict = classify(here, f.sha256, base);
+        if (verdict === "conflict") {
+          const cloudNewer = new Date(f.modifiedAt).getTime() >= st.mtimeMs;
+          const kept = conflictPath(target.abs, now(io));
+          if (cloudNewer) {
+            renameSync(target.abs, kept);
+            await download(ctx.client, f, target.abs);
+          } else {
+            await download(ctx.client, f, kept);
+          }
+          result.conflicts.push({ path: f.path, kept: path.relative(project.root, kept).split(path.sep).join("/"), winner: cloudNewer ? "cloud" : "local" });
+          ctx.state.files[f.path] = f.sha256;
+          bytes += f.size;
+          continue;
+        }
+        // Only the cloud changed since this machine last synced: take it.
+      }
+      await download(ctx.client, f, target.abs);
+      ctx.state.files[f.path] = f.sha256;
+      result.written.push(f.path);
+      bytes += f.size;
+    }
+    const next = answer.cursor ?? cursor;
+    if (answer.hasMore && next === cursor && answer.files.length === 0) throw new ApiFailure("The server's pull cursor did not move.", "try again in a moment.");
+    cursor = next;
+    ctx.state.cursor = cursor;
+    writeState(project.root, ctx.state);
+    if (!answer.hasMore) break;
+  }
+  if (p.flags.has("--json")) io.stdout(json({ pulled: result.written, bytes, conflicts: result.conflicts, refused: result.refused }));
+  else {
+    const total = result.written.length + result.conflicts.length;
+    io.stdout(`✓ Pulled ${total} file${total === 1 ? "" : "s"} (${formatSize(bytes)}) from ${ctx.host}.${total === 0 ? " Everything here is already up to date." : ""}\n`);
+    for (const f of result.written.slice(0, 20)) io.stdout(`  ${clean(f)}\n`);
+    for (const c of result.conflicts) io.stdout(`  ${clean(c.path)}: changed in both places. The ${c.winner === "cloud" ? "cloud" : "local"} copy is newer and wins; the other is kept as ${clean(c.kept)}.\n`);
+  }
+  if (result.refused.length) {
+    io.stderr(`✗ ${result.refused.length} file${result.refused.length === 1 ? " was" : "s were"} not written: ${result.refused.slice(0, 5).map((r) => `${clean(r.path)} (${clean(r.reason)})`).join(", ")}.\nFix: these paths are not safe to write into a project; nothing outside scenes/, scripts/ and videos/ is ever written.\n`);
+    return 1;
+  }
+  return 0;
+}
+
+// ---------- status ----------
+
+async function status(io: CloudIo, args: string[]): Promise<number> {
+  const p = parseArgs(args, ["--json"]);
+  if (p.error || p.positionals.length) return refuse(io, p.error ?? `Unexpected "${p.positionals[0]}".`, USAGE);
+  const project = requireProject(io);
+  if (typeof project === "number") return project;
+  if (!project.cloudProjectId) {
+    if (p.flags.has("--json")) io.stdout(json({ linked: false, name: project.name }));
+    else io.stdout(`${clean(project.name)} is not linked to a cloud project.\nLink it: \`npx vfrey cloud link\`.\n`);
+    return 0;
+  }
+  const ctx = open(io, project, p.api);
+  if (typeof ctx === "number") return ctx;
+  const { files: local } = await scanProject(project.root, { videos: false });
+  const cloudAll = (await ctx.client.cloudFiles(project.cloudProjectId)).files;
+  const cloud = new Map<string, CloudFileView>(cloudAll.filter((f) => f.kind !== "video" && f.kind !== "transcript").map((f) => [f.path, f]));
+  const counts = { same: 0, push: [] as string[], pull: [] as string[], conflict: [] as string[] };
+  const seen = new Set<string>();
+  for (const f of local) {
+    seen.add(f.path);
+    const v = classify(f.sha256, cloud.get(f.path)?.sha256, ctx.state.files[f.path]);
+    if (v === "same") counts.same += 1;
+    else if (v === "new" || v === "push") counts.push.push(f.path);
+    else if (v === "pull") counts.pull.push(f.path);
+    else counts.conflict.push(f.path);
+  }
+  for (const c of cloud.keys()) if (!seen.has(c)) counts.pull.push(c);
+  const videos = cloudAll.filter((f) => f.kind === "video").length;
+  const report = { linked: true, name: project.name, cloudProjectId: project.cloudProjectId, host: ctx.host, inSync: counts.same, toPush: counts.push, toPull: counts.pull, conflicts: counts.conflict, videosInCloud: videos, lastPullCursor: ctx.state.cursor };
+  if (p.flags.has("--json")) io.stdout(json(report));
+  else {
+    const line = (label: string, list: string[]) => (list.length ? `${label}: ${list.length} (${list.slice(0, 3).map((x) => clean(x)).join(", ")}${list.length > 3 ? ", ..." : ""})\n` : "");
+    io.stdout(`${clean(project.name)} is linked to ${clean(project.cloudProjectId)} on ${ctx.host}.\nIn sync: ${counts.same} file${counts.same === 1 ? "" : "s"}.\n${line("To push", counts.push)}${line("To pull", counts.pull)}${line("Changed in both places", counts.conflict)}Videos in the cloud: ${videos}.\n`);
+    if (counts.push.length === 0 && counts.pull.length === 0 && counts.conflict.length === 0) io.stdout("Scenes and scripts match the cloud.\n");
+    else io.stdout(`Next: ${counts.conflict.length || counts.pull.length ? "`npx vfrey cloud pull`" : ""}${(counts.conflict.length || counts.pull.length) && counts.push.length ? " then " : ""}${counts.push.length ? "`npx vfrey cloud push`" : ""}.\n`);
+  }
+  return 0;
+}
+
+export async function runCloud(args: string[], io: CloudIo): Promise<number> {
+  const sub = args[0];
+  const rest = args.slice(1);
+  try {
+    if (sub === "link") return await link(io, rest);
+    if (sub === "push") return await push(io, rest);
+    if (sub === "pull") return await pull(io, rest);
+    if (sub === "status") return await status(io, rest);
+    return refuse(io, sub ? `Unknown cloud command "${sub}".` : "Say what to do in the cloud.", USAGE);
+  } catch (error) {
+    return failure(io, cloudFailure(error));
+  }
+}