@asteroidai/cli 1.9.2 → 1.9.11

package/src/sync/sync.test.ts

@@ -0,0 +1,463 @@
+import { describe, expect, it } from "vitest";
+
+import { UsageError } from "../args";
+import type { BuiltRequest } from "../request";
+import { formatExecutionPull, pullExecution } from "./execution-pull";
+import { sha256 } from "./paths";
+import {
+  type FileTree,
+  type LocalFiles,
+  MANIFEST,
+  type Manifest,
+  type Sync,
+} from "./types";
+import { pullWorkflow } from "./workflow-pull";
+import { pushWorkflow } from "./workflow-push";
+
+const WF = "11111111-aaaa-bbbb-cccc-000000000000";
+const HEAD = "22222222-aaaa-bbbb-cccc-000000000000";
+
+function memoryFiles(initial: Record<string, string> = {}) {
+  const store = new Map(
+    Object.entries(initial).map(([path, text]) => [path, Buffer.from(text)]),
+  );
+  const files: LocalFiles = {
+    read: async (path) => store.get(path),
+    write: async (path, bytes) => void store.set(path, Buffer.from(bytes)),
+    remove: async (path) => void store.delete(path),
+    list: async (dir) =>
+      [...store.keys()]
+        .filter((path) => path.startsWith(`${dir}/`))
+        .map((path) => path.slice(dir.length + 1))
+        .sort(),
+  };
+  const text = (path: string) => store.get(path)?.toString("utf8");
+  return { files, store, text };
+}
+
+function fakeRemote(
+  routes: Record<string, (request: BuiltRequest) => unknown>,
+) {
+  const calls: BuiltRequest[] = [];
+  const downloads: string[] = [];
+  const remote = {
+    async json(request: BuiltRequest) {
+      calls.push(request);
+      const handler = routes[`${request.method} ${request.path}`];
+      if (!handler)
+        throw new Error(`unexpected ${request.method} ${request.path}`);
+      return handler(request);
+    },
+    async download(url: string) {
+      downloads.push(url);
+      return Buffer.from(`bytes of ${url}`);
+    },
+  };
+  return { remote, calls, downloads };
+}
+
+function tree(overrides: Partial<FileTree> = {}): FileTree {
+  return {
+    rev: 7,
+    version: 3,
+    versionId: HEAD,
+    workflowName: "Fill Form",
+    files: [
+      {
+        path: "settings.yaml",
+        kind: "structural",
+        size: 14,
+        content: "start: Login\n",
+      },
+      {
+        path: "memory.md",
+        kind: "agent_file",
+        size: 5,
+        contentBase64: Buffer.from("notes").toString("base64"),
+      },
+      { path: "big.pdf", kind: "agent_file", size: 9000000, fileId: "f-big" },
+    ],
+    ...overrides,
+  };
+}
+
+const headFiles = `GET /agents/v2/workflows/${WF}/head/files`;
+const signedBig = `GET /agents/v2/workflows/${WF}/versions/${HEAD}/files/f-big/signed-url`;
+
+describe("pullWorkflow", () => {
+  function pulled(initial: Record<string, string> = {}) {
+    const local = memoryFiles(initial);
+    const { remote, calls, downloads } = fakeRemote({
+      [headFiles]: () => tree(),
+      [signedBig]: () => ({ url: "https://storage/big" }),
+    });
+    const sync: Sync = { remote, files: local.files };
+    return { sync, local, calls, downloads };
+  }
+
+  it("writes the head under <workflow-name>/head with every file's bytes", async () => {
+    const { sync, local, downloads } = pulled();
+
+    const result = await pullWorkflow(sync, { workflowId: WF, root: "out" });
+
+    expect(result.dir).toBe("out/Fill_Form/head");
+    expect(local.text("out/Fill_Form/head/settings.yaml")).toBe(
+      "start: Login\n",
+    );
+    expect(local.text("out/Fill_Form/head/memory.md")).toBe("notes");
+    expect(local.text("out/Fill_Form/head/big.pdf")).toBe(
+      "bytes of https://storage/big",
+    );
+    expect(downloads).toEqual(["https://storage/big"]);
+  });
+
+  it("asks for every file's contents", async () => {
+    const { sync, calls } = pulled();
+
+    await pullWorkflow(sync, { workflowId: WF, root: "" });
+
+    expect(calls[0].query.get("contents")).toBe("all");
+  });
+
+  it("records the rev and each file's hash for a later push", async () => {
+    const { sync, local } = pulled();
+
+    await pullWorkflow(sync, { workflowId: WF, root: "" });
+
+    const manifest = JSON.parse(
+      local.text(`Fill_Form/head/${MANIFEST}`)!,
+    ) as Manifest;
+    expect(manifest).toMatchObject({
+      workflowId: WF,
+      versionId: HEAD,
+      head: true,
+      rev: 7,
+    });
+    expect(manifest.files["memory.md"]).toBe(sha256(Buffer.from("notes")));
+  });
+
+  it("removes local files the workflow no longer has", async () => {
+    const { sync, local } = pulled({
+      "Fill_Form/head/nodes/Old/instructions.md": "gone",
+    });
+
+    await pullWorkflow(sync, { workflowId: WF, root: "" });
+
+    expect(local.store.has("Fill_Form/head/nodes/Old/instructions.md")).toBe(
+      false,
+    );
+  });
+
+  it("skips files over the size limit instead of downloading them", async () => {
+    const { sync, local, downloads } = pulled();
+
+    const result = await pullWorkflow(sync, {
+      workflowId: WF,
+      root: "",
+      maxFileBytes: 1000,
+    });
+
+    expect(downloads).toEqual([]);
+    expect(local.store.has("Fill_Form/head/big.pdf")).toBe(false);
+    expect(result.skipped).toEqual([
+      "big.pdf (9000000 bytes, over the size limit)",
+    ]);
+  });
+
+  it("writes a pinned version under .versions/", async () => {
+    const local = memoryFiles();
+    const { remote } = fakeRemote({
+      [`GET /agents/v2/workflows/${WF}/versions/${HEAD}/files`]: () =>
+        tree({ files: [] }),
+    });
+
+    const result = await pullWorkflow(
+      { remote, files: local.files },
+      { workflowId: WF, versionId: HEAD, root: "" },
+    );
+
+    expect(result.dir).toBe("Fill_Form/.versions/v3_22222222");
+  });
+
+  it("refuses an entry whose path leaves the folder", async () => {
+    const local = memoryFiles();
+    const { remote } = fakeRemote({
+      [headFiles]: () =>
+        tree({
+          files: [
+            { path: "../escape", kind: "structural", size: 1, content: "x" },
+          ],
+        }),
+    });
+
+    const result = await pullWorkflow(
+      { remote, files: local.files },
+      { workflowId: WF, root: "" },
+    );
+
+    expect(result.skipped).toEqual(["../escape (unsafe path)"]);
+    expect([...local.store.keys()]).toEqual([`Fill_Form/head/${MANIFEST}`]);
+  });
+});
+
+describe("pushWorkflow", () => {
+  const dir = "Fill_Form/head";
+
+  function pulledHead(edits: Record<string, string | null>) {
+    const base: Record<string, string> = {
+      "settings.yaml": "start: Login\n",
+      "memory.md": "notes",
+      "old.md": "old",
+    };
+    const manifest: Manifest = {
+      workflowId: WF,
+      versionId: HEAD,
+      head: true,
+      rev: 7,
+      files: Object.fromEntries(
+        Object.entries(base).map(([p, t]) => [p, sha256(Buffer.from(t))]),
+      ),
+    };
+    const merged = { ...base, ...edits };
+    const initial: Record<string, string> = {
+      [`${dir}/${MANIFEST}`]: JSON.stringify(manifest),
+    };
+    for (const [path, text] of Object.entries(merged)) {
+      if (text !== null) initial[`${dir}/${path}`] = text;
+    }
+    return memoryFiles(initial);
+  }
+
+  it("sends changed and new files as writes and removed files as deletes", async () => {
+    const local = pulledHead({
+      "memory.md": "more notes",
+      "new.md": "hi",
+      "old.md": null,
+    });
+    let patched: unknown;
+    const { remote } = fakeRemote({
+      [`PATCH /agents/v2/workflows/${WF}/head/files`]: (request) => {
+        patched = request.json;
+        return {};
+      },
+      [headFiles]: () => tree({ rev: 8, files: [] }),
+    });
+
+    const result = await pushWorkflow({ remote, files: local.files }, dir);
+
+    expect(patched).toEqual({
+      baseRev: 7,
+      writes: [
+        {
+          path: "memory.md",
+          contentBase64: Buffer.from("more notes").toString("base64"),
+        },
+        { path: "new.md", contentBase64: Buffer.from("hi").toString("base64") },
+      ],
+      deletes: ["old.md"],
+    });
+    expect(result).toMatchObject({
+      written: ["memory.md", "new.md"],
+      deleted: ["old.md"],
+      rev: 8,
+    });
+  });
+
+  it("takes the server's rendering of structural files and the new rev", async () => {
+    const local = pulledHead({ "settings.yaml": "start:   Login\n" });
+    const { remote } = fakeRemote({
+      [`PATCH /agents/v2/workflows/${WF}/head/files`]: () => ({}),
+      [headFiles]: () =>
+        tree({
+          rev: 8,
+          files: [
+            {
+              path: "settings.yaml",
+              kind: "structural",
+              size: 13,
+              content: "start: Login\n",
+            },
+            { path: "memory.md", kind: "agent_file", size: 5 },
+            { path: "old.md", kind: "agent_file", size: 3 },
+          ],
+        }),
+    });
+
+    await pushWorkflow({ remote, files: local.files }, dir);
+
+    expect(local.text(`${dir}/settings.yaml`)).toBe("start: Login\n");
+    const manifest = JSON.parse(local.text(`${dir}/${MANIFEST}`)!) as Manifest;
+    expect(manifest.rev).toBe(8);
+    expect(manifest.files["settings.yaml"]).toBe(
+      sha256(Buffer.from("start: Login\n")),
+    );
+  });
+
+  it("sends nothing when the folder matches the pull", async () => {
+    const local = pulledHead({});
+    const { remote, calls } = fakeRemote({});
+
+    const result = await pushWorkflow({ remote, files: local.files }, dir);
+
+    expect(calls).toEqual([]);
+    expect(result).toMatchObject({ written: [], deleted: [], rev: 7 });
+  });
+
+  it("tells the user to pull again when the head moved", async () => {
+    const local = pulledHead({ "memory.md": "edit" });
+    const { remote } = fakeRemote({
+      [`PATCH /agents/v2/workflows/${WF}/head/files`]: () => {
+        throw Object.assign(new Error("stale"), { status: 409 });
+      },
+    });
+
+    const push = pushWorkflow({ remote, files: local.files }, dir);
+
+    await expect(push).rejects.toThrow(UsageError);
+    await expect(push).rejects.toThrow("changed since your pull");
+  });
+
+  it("refuses a folder with no pull in it", async () => {
+    const { remote } = fakeRemote({});
+
+    const push = pushWorkflow({ remote, files: memoryFiles().files }, dir);
+
+    await expect(push).rejects.toThrow("no pulled workflow");
+  });
+
+  it("refuses a pinned version", async () => {
+    const manifest = {
+      workflowId: WF,
+      versionId: HEAD,
+      head: false,
+      rev: 1,
+      files: {},
+    };
+    const local = memoryFiles({
+      [`${dir}/${MANIFEST}`]: JSON.stringify(manifest),
+    });
+    const { remote } = fakeRemote({});
+
+    const push = pushWorkflow({ remote, files: local.files }, dir);
+
+    await expect(push).rejects.toThrow("read-only");
+  });
+});
+
+describe("pullExecution", () => {
+  const EX = "33333333-aaaa-bbbb-cccc-000000000000";
+  const execution = {
+    id: EX,
+    status: "completed",
+    workflowId: WF,
+    workflowVersionId: HEAD,
+    workflowName: "Fill Form",
+    createdAt: "2026-10-09T10:00:00Z",
+    executionResult: { outcome: "success", reasoning: "done" },
+  };
+  const file = (id: string, directory: string, name: string, size = 10) => ({
+    id,
+    fileName: name,
+    filePath: `org/${EX}/${directory}/${name}`,
+    fileSize: size,
+    directory,
+  });
+
+  function setup(overrides: Record<string, unknown> = {}) {
+    const local = memoryFiles();
+    const { remote, downloads } = fakeRemote({
+      [`GET /agents/v2/executions/${EX}`]: () => ({
+        ...execution,
+        ...overrides,
+      }),
+      [`GET /agents/v2/executions/${EX}/activities`]: () => [
+        { payload: { activityType: "generic", message: "hello" } },
+      ],
+      [`GET /agents/v2/executions/${EX}/files`]: () => ({
+        directories: {
+          workspace: [file("a", "workspace", "out.csv")],
+          downloads: [file("b", "downloads", "big.zip", 5000)],
+          uploads: [file("c", "uploads", "input.pdf")],
+        },
+      }),
+      [`GET /agents/v2/executions/${EX}/files/a/signed-url`]: () => ({
+        url: "https://s/a",
+      }),
+      [`GET /agents/v2/executions/${EX}/files/b/signed-url`]: () => ({
+        url: "https://s/b",
+      }),
+    });
+    return { sync: { remote, files: local.files }, local, downloads };
+  }
+
+  it("writes the summary, activities and run files under the workflow's executions folder", async () => {
+    const { sync, local } = setup();
+
+    const result = await pullExecution(sync, { executionId: EX, root: "" });
+
+    const dir = "Fill_Form/executions/33333333";
+    expect(result.dir).toBe(dir);
+    expect(JSON.parse(local.text(`${dir}/summary.json`)!).id).toBe(EX);
+    expect(local.text(`${dir}/activities.md`)).toContain("[message]:\nhello");
+    expect(local.text(`${dir}/workspace/out.csv`)).toBe("bytes of https://s/a");
+    expect(local.text(`${dir}/downloads/big.zip`)).toBe("bytes of https://s/b");
+    expect(local.store.has(`${dir}/uploads/input.pdf`)).toBe(false);
+  });
+
+  it("filters files by glob and size and names what it skipped", async () => {
+    const { sync, downloads } = setup();
+
+    const result = await pullExecution(sync, {
+      executionId: EX,
+      root: "",
+      glob: "**/*.zip",
+      maxFileBytes: 100,
+    });
+
+    expect(downloads).toEqual([]);
+    expect(result.skipped).toEqual([
+      "workspace/out.csv (excluded by --files)",
+      "downloads/big.zip (5000 bytes, over the size limit)",
+    ]);
+  });
+
+  it("skips downloads with files off", async () => {
+    const { sync, downloads } = setup();
+
+    const result = await pullExecution(sync, {
+      executionId: EX,
+      root: "",
+      files: false,
+    });
+
+    expect(downloads).toEqual([]);
+    expect(formatExecutionPull(result)).toContain(
+      "downloads skipped (--no-files)",
+    );
+  });
+
+  it("stops at the summary for a purged run", async () => {
+    const { sync, local } = setup({ purgedAt: "2026-10-01T00:00:00Z" });
+
+    const result = await pullExecution(sync, { executionId: EX, root: "" });
+
+    expect(result.written).toEqual(["summary.json"]);
+    expect(local.store.has("Fill_Form/executions/33333333/activities.md")).toBe(
+      false,
+    );
+    expect(formatExecutionPull(result)).toContain("retention-purged");
+  });
+
+  it("uses the folder a caller chooses", async () => {
+    const { sync } = setup();
+
+    const result = await pullExecution(sync, {
+      executionId: EX,
+      root: "",
+      files: false,
+      dir: () => "mine/executions/x",
+    });
+
+    expect(result.dir).toBe("mine/executions/x");
+  });
+});

package/src/sync/types.ts

@@ -0,0 +1,58 @@
+import type { BuiltRequest } from "../request";
+
+export interface LocalFiles {
+  read(path: string): Promise<Uint8Array | undefined>;
+  write(path: string, bytes: Uint8Array): Promise<void>;
+  remove(path: string): Promise<void>;
+  list(dir: string): Promise<string[]>;
+}
+
+export interface Remote {
+  json(request: BuiltRequest): Promise<unknown>;
+  download(url: string): Promise<Uint8Array>;
+}
+
+export interface Sync {
+  remote: Remote;
+  files: LocalFiles;
+}
+
+export interface FileEntry {
+  path: string;
+  kind: string;
+  size: number;
+  content?: string;
+  contentBase64?: string;
+  checksum?: string;
+  fileId?: string;
+}
+
+export interface FileTree {
+  rev: number;
+  version?: number;
+  versionId: string;
+  workflowName: string;
+  files: FileEntry[];
+}
+
+export const MANIFEST = ".asteroid.json";
+
+export interface Manifest {
+  workflowId: string;
+  versionId: string;
+  head: boolean;
+  rev: number;
+  files: Record<string, string>;
+}
+
+export function get(
+  path: string,
+  query: Record<string, string> = {},
+): BuiltRequest {
+  return { method: "GET", path, query: new URLSearchParams(query) };
+}
+
+export function statusOf(error: unknown): number | undefined {
+  const status = (error as { status?: unknown } | null)?.status;
+  return typeof status === "number" ? status : undefined;
+}

package/src/sync/workflow-pull.ts

@@ -0,0 +1,182 @@
+import {
+  isContained,
+  join,
+  sha256,
+  workflowDirName,
+  workflowRoot,
+} from "./paths";
+import {
+  type FileEntry,
+  type FileTree,
+  get,
+  MANIFEST,
+  type Manifest,
+  type Sync,
+} from "./types";
+
+const CONCURRENCY = 16;
+
+export interface WorkflowPullOptions {
+  workflowId: string;
+  versionId?: string;
+  root: string;
+  dir?: string;
+  manifest?: boolean;
+  maxFileBytes?: number;
+}
+
+export interface WorkflowPullResult {
+  workflowId: string;
+  workflowName: string;
+  versionId: string;
+  version?: number;
+  rev: number;
+  dir: string;
+  written: number;
+  shared: number;
+  skipped: string[];
+}
+
+function treeRequest(options: WorkflowPullOptions) {
+  const base = `/agents/v2/workflows/${options.workflowId}`;
+  const path = options.versionId
+    ? `${base}/versions/${options.versionId}/files`
+    : `${base}/head/files`;
+  return get(path, { contents: "all" });
+}
+
+export function pullDir(options: WorkflowPullOptions, tree: FileTree): string {
+  if (options.dir) return options.dir;
+  const root = workflowRoot(options.root, tree.workflowName);
+  if (!options.versionId) return join(root, "head");
+  return join(root, ".versions", workflowDirName(tree.versionId, tree.version));
+}
+
+async function agentFileBytes(
+  sync: Sync,
+  workflowId: string,
+  tree: FileTree,
+  entry: FileEntry,
+): Promise<Uint8Array> {
+  const path = `/agents/v2/workflows/${workflowId}/versions/${tree.versionId}/files/${entry.fileId}/signed-url`;
+  const { url } = (await sync.remote.json(get(path))) as { url: string };
+  return sync.remote.download(url);
+}
+
+export async function entryBytes(
+  sync: Sync,
+  workflowId: string,
+  tree: FileTree,
+  entry: FileEntry,
+): Promise<Uint8Array | undefined> {
+  if (entry.contentBase64 !== undefined)
+    return Buffer.from(entry.contentBase64, "base64");
+  if (entry.content !== undefined) return Buffer.from(entry.content, "utf8");
+  if (!entry.fileId) return undefined;
+  return agentFileBytes(sync, workflowId, tree, entry);
+}
+
+async function writeEntries(
+  sync: Sync,
+  options: WorkflowPullOptions,
+  tree: FileTree,
+  dir: string,
+) {
+  const hashes: Record<string, string> = {};
+  const skipped: string[] = [];
+  let shared = 0;
+  const wanted = tree.files.filter((entry) => {
+    if (!isContained(entry.path)) skipped.push(`${entry.path} (unsafe path)`);
+    else if (
+      options.maxFileBytes !== undefined &&
+      entry.size > options.maxFileBytes
+    )
+      skipped.push(`${entry.path} (${entry.size} bytes, over the size limit)`);
+    else return true;
+    return false;
+  });
+  for (let i = 0; i < wanted.length; i += CONCURRENCY) {
+    await Promise.all(
+      wanted.slice(i, i + CONCURRENCY).map(async (entry) => {
+        const bytes = await entryBytes(sync, options.workflowId, tree, entry);
+        if (!bytes) return void skipped.push(`${entry.path} (no content)`);
+        await sync.files.write(join(dir, entry.path), bytes);
+        hashes[entry.path] = sha256(bytes);
+        if (entry.kind === "agent_file") shared++;
+      }),
+    );
+  }
+  return { hashes, skipped, shared };
+}
+
+async function removeStale(sync: Sync, dir: string, keep: Set<string>) {
+  for (const path of await sync.files.list(dir)) {
+    if (!keep.has(path) && path !== MANIFEST)
+      await sync.files.remove(join(dir, path));
+  }
+}
+
+export async function writeManifest(
+  sync: Sync,
+  dir: string,
+  manifest: Manifest,
+) {
+  const text = `${JSON.stringify(manifest, null, 2)}\n`;
+  await sync.files.write(join(dir, MANIFEST), Buffer.from(text, "utf8"));
+}
+
+export async function pullWorkflow(
+  sync: Sync,
+  options: WorkflowPullOptions,
+): Promise<WorkflowPullResult> {
+  const tree = (await sync.remote.json(treeRequest(options))) as FileTree;
+  const dir = pullDir(options, tree);
+  const { hashes, skipped, shared } = await writeEntries(
+    sync,
+    options,
+    tree,
+    dir,
+  );
+  await removeStale(sync, dir, new Set(Object.keys(hashes)));
+  if (options.manifest !== false) {
+    await writeManifest(sync, dir, {
+      workflowId: options.workflowId,
+      versionId: tree.versionId,
+      head: !options.versionId,
+      rev: tree.rev,
+      files: hashes,
+    });
+  }
+  return {
+    workflowId: options.workflowId,
+    workflowName: tree.workflowName,
+    versionId: tree.versionId,
+    version: tree.version,
+    rev: tree.rev,
+    dir,
+    written: Object.keys(hashes).length,
+    shared,
+    skipped,
+  };
+}
+
+export function formatWorkflowPull(result: WorkflowPullResult): string {
+  const definition = result.written - result.shared;
+  const files =
+    result.written === 0
+      ? "Files: none written"
+      : result.shared === 0
+        ? `Files: ${result.written} written`
+        : `Files: ${result.written} written (${definition} workflow definition, ${result.shared} shared)`;
+  return [
+    `Workflow: ${result.workflowName} (${result.workflowId})`,
+    `Workflow version: ${result.versionId}${result.version ? ` (v${result.version})` : " (draft)"}`,
+    `Written to: ${result.dir}/`,
+    files,
+    result.skipped.length > 0
+      ? `Skipped:\n${result.skipped.map((s) => `  - ${s}`).join("\n")}`
+      : null,
+  ]
+    .filter(Boolean)
+    .join("\n");
+}