@saadnvd1/agent-os 0.3.794 → 0.3.799

package/scripts/update-e2e.test.ts

@@ -0,0 +1,215 @@
+import { execFile, execFileSync } from "node:child_process";
+import fs from "node:fs";
+import net from "node:net";
+import path from "node:path";
+import { promisify } from "node:util";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { tempDir } from "@/lib/test-temp";
+
+// The real scripts/update-run.mjs and scripts/redeploy on a scratch clone of
+// a scratch remote: install and build are stubs, and the "server" is a tiny
+// node script the restart command starts, answering health with the commit
+// it was started from. A commit carrying a `crash` file makes it exit on
+// boot, so the update has to roll itself back.
+const exec = promisify(execFile);
+const here = import.meta.dirname;
+const SCRIPTS = [
+  "redeploy",
+  "update-run.mjs",
+  "lib/build-swap.sh",
+  "lib/pnpm.sh",
+  "lib/deps-swap.sh",
+  "lib/update-verify.mjs",
+];
+
+const SERVER = `
+const http = require("http");
+const fs = require("fs");
+const { execSync } = require("child_process");
+if (fs.existsSync("crash")) process.exit(1);
+const build = execSync("git rev-parse HEAD").toString().trim();
+const startedAt = new Date().toISOString();
+http.createServer((req, res) => {
+  res.setHeader("content-type", "application/json");
+  if (req.url === "/api/updates/health")
+    return res.end(JSON.stringify({ ok: true, build, startedAt, migrations: { known: 1, applied: 1 } }));
+  res.end("{}");
+}).listen(Number(process.env.PORT), "127.0.0.1");
+`;
+
+let root: string;
+let live: string;
+let port: string;
+
+// Nothing of the machine's git config: no global identity or default branch
+// to lean on, as on a CI runner. Every git here and in the scripts under
+// test gets this environment.
+const GIT_ENV = {
+  GIT_CONFIG_GLOBAL: "/dev/null",
+  GIT_CONFIG_NOSYSTEM: "1",
+  GIT_AUTHOR_NAME: "t",
+  GIT_AUTHOR_EMAIL: "t@t",
+  GIT_COMMITTER_NAME: "t",
+  GIT_COMMITTER_EMAIL: "t@t",
+};
+const git = (cwd: string, ...args: string[]) =>
+  execFileSync("git", args, {
+    cwd,
+    encoding: "utf8",
+    env: { ...process.env, ...GIT_ENV },
+  }).trim();
+const commit = (cwd: string, msg: string) => {
+  git(cwd, "add", "-A");
+  git(cwd, "commit", "-q", "--allow-empty", "-m", msg);
+  git(cwd, "push", "-q", "origin", "HEAD:main");
+  return git(cwd, "rev-parse", "HEAD");
+};
+
+const freePort = () =>
+  new Promise<string>((resolve) => {
+    const s = net.createServer().listen(0, "127.0.0.1", () => {
+      const p = (s.address() as net.AddressInfo).port;
+      s.close(() => resolve(String(p)));
+    });
+  });
+
+const health = async () => {
+  try {
+    const res = await fetch(`http://127.0.0.1:${port}/api/updates/health`);
+    return ((await res.json()) as { build: string }).build;
+  } catch {
+    return null;
+  }
+};
+
+beforeEach(async () => {
+  root = tempDir("aos-e2e-");
+  port = await freePort();
+  const origin = path.join(root, "origin.git");
+  const src = path.join(root, "src");
+  live = path.join(root, "live");
+  git(root, "init", "-q", "-b", "main", "--bare", origin);
+  git(root, "init", "-q", "-b", "main", src);
+  git(src, "remote", "add", "origin", origin);
+  for (const f of SCRIPTS) {
+    fs.mkdirSync(path.dirname(path.join(src, "scripts", f)), {
+      recursive: true,
+    });
+    fs.copyFileSync(path.join(here, f), path.join(src, "scripts", f));
+  }
+  fs.chmodSync(path.join(src, "scripts/redeploy"), 0o755);
+  fs.writeFileSync(path.join(src, "server.js"), SERVER);
+  for (const f of ["package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml"])
+    fs.writeFileSync(path.join(src, f), "{}");
+  fs.writeFileSync(
+    path.join(src, ".gitignore"),
+    ".next*\n.deps\nnode_modules\n*.log\n"
+  );
+  commit(src, "first");
+  git(root, "clone", "-q", origin, live);
+  fs.mkdirSync(path.join(live, ".next"));
+  fs.writeFileSync(path.join(live, ".next/BUILD_ID"), "first");
+  fs.mkdirSync(path.join(root, "bin"));
+  fs.writeFileSync(path.join(root, "bin/pnpm"), "#!/bin/sh\nexit 0\n", {
+    mode: 0o755,
+  });
+  // Stop the server on the port (if any), then start this checkout's.
+  fs.writeFileSync(
+    path.join(root, "restart"),
+    `#!/usr/bin/env bash
+[ -f "${root}/pid" ] && kill "$(cat "${root}/pid")" 2>/dev/null
+for _ in $(seq 1 50); do curl -s -o /dev/null "http://127.0.0.1:${port}/" || break; sleep 0.1; done
+cd "${live}" || exit 1
+nohup node server.js > "${root}/server.log" 2>&1 < /dev/null &
+echo $! > "${root}/pid"
+`,
+    { mode: 0o755 }
+  );
+  execFileSync(path.join(root, "restart"), {
+    env: { ...process.env, ...GIT_ENV, PORT: port },
+  });
+  for (let i = 0; i < 50 && !(await health()); i++)
+    await new Promise((r) => setTimeout(r, 100));
+});
+
+afterEach(() => {
+  try {
+    process.kill(Number(fs.readFileSync(path.join(root, "pid"), "utf8")));
+  } catch {}
+});
+
+async function update(from: string, to: string) {
+  const stateFile = path.join(root, "state.json");
+  fs.writeFileSync(
+    stateFile,
+    JSON.stringify({
+      id: "e2e",
+      phase: "starting",
+      from,
+      startedAt: Date.now(),
+    })
+  );
+  const env: NodeJS.ProcessEnv = {
+    ...process.env,
+    ...GIT_ENV,
+    HOME: root,
+    PATH: `${path.join(root, "bin")}:${process.env.PATH}`,
+    PORT: port,
+    AGENTOS_UPDATE_STATE: stateFile,
+    AGENTOS_RESTART: path.join(root, "restart"),
+    AGENTOS_UPDATE_TO: to,
+    AGENTOS_INSTALL_CMD: "mkdir -p node_modules",
+    AGENTOS_NATIVE_CHECK: "true",
+    AGENTOS_BUILD_CMD:
+      'mkdir -p "$AGENTOS_DIST_DIR" && git rev-parse HEAD > "$AGENTOS_DIST_DIR/BUILD_ID"',
+    AGENTOS_UP_WAIT: "3",
+    AGENTOS_UPDATE_MIN_FREE_GB: "0",
+    AGENTOS_VERIFY_UP_MS: "5000",
+    AGENTOS_VERIFY_EVERY_MS: "50",
+    AGENTOS_VERIFY_CHECKS: "2",
+  };
+  for (const k of [
+    "AGENTOS_UPDATE_DB",
+    "AGENTOS_UPDATE_ROOT",
+    "AGENTOS_UPDATE_REDEPLOY",
+    "NODE_ENV",
+  ])
+    delete env[k];
+  await exec(process.execPath, [path.join(live, "scripts/update-run.mjs")], {
+    env,
+    timeout: 60_000,
+  }).catch(() => {});
+  return JSON.parse(fs.readFileSync(stateFile, "utf8"));
+}
+
+describe("an automatic update, end to end", () => {
+  it("updates to a good commit, then rolls back one whose server won't start", async () => {
+    const src = path.join(root, "src");
+    const first = git(live, "rev-parse", "HEAD");
+    expect(await health()).toBe(first);
+
+    fs.writeFileSync(path.join(src, "feature.txt"), "good");
+    const good = commit(src, "good");
+    // A newer head the update must not take: it is pinned to `good`.
+    fs.writeFileSync(path.join(src, "later.txt"), "later");
+    commit(src, "later");
+    const s1 = await update(first, good);
+    expect(s1).toMatchObject({ phase: "done", outcome: "done" });
+    expect(git(live, "rev-parse", "HEAD")).toBe(good);
+    expect(await health()).toBe(good);
+    expect(
+      fs.readFileSync(path.join(live, ".next/BUILD_ID"), "utf8").trim()
+    ).toBe(good);
+
+    fs.writeFileSync(path.join(src, "crash"), "");
+    const bad = commit(src, "bad");
+    const s2 = await update(good, bad);
+    expect(s2).toMatchObject({ phase: "rolled_back", outcome: "rolled_back" });
+    expect(git(live, "rev-parse", "HEAD")).toBe(good);
+    expect(fs.existsSync(path.join(live, "crash"))).toBe(false);
+    expect(
+      fs.readFileSync(path.join(live, ".next/BUILD_ID"), "utf8").trim()
+    ).toBe(good);
+    expect(await health()).toBe(good);
+  }, 90_000);
+});

package/scripts/update-recover.mjs

@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+// Runs before the server starts (npm prestart): repairs an update that was
+// cut off (the machine slept, lost power, the runner was killed), so the
+// server never starts on half of one.
+//   - .next missing between swap_build's two renames: finish the rename.
+//   - a run whose runner is gone, stopped at:
+//       pulled        → the source goes back to where it was
+//       deps_swapped  → the dependencies and the source go back
+//       build_swapped → nothing: source, deps and build all agree (new)
+//     and the run is marked interrupted, so the next quiet window retries.
+// Never fails the start: anything unexpected is logged and left alone.
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+const ACTIVE = new Set([
+  "starting",
+  "backing_up",
+  "pulling",
+  "installing",
+  "building",
+  "restarting",
+  "verifying",
+  "rolling_back",
+]);
+const STALE_MS = 2 * 60_000;
+const OLD_STALE_MS = 30 * 60_000;
+
+const alive = (pid) => {
+  if (!pid) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error.code === "EPERM";
+  }
+};
+
+/** .next is absent only between two renames; finish whichever was under way. */
+export function recoverBuild(root) {
+  const live = path.join(root, ".next");
+  if (fs.existsSync(path.join(live, "BUILD_ID"))) return null;
+  for (const from of [".next-build", ".next-prev"]) {
+    const dir = path.join(root, from);
+    if (fs.existsSync(path.join(dir, "BUILD_ID"))) {
+      if (fs.existsSync(live))
+        fs.rmSync(live, { recursive: true, force: true });
+      fs.renameSync(dir, live);
+      return from;
+    }
+  }
+  return null;
+}
+
+/** True when the run is in an active phase and nothing is running it. */
+export function isInterrupted(run, now = Date.now()) {
+  if (!run || !ACTIVE.has(run.phase)) return false;
+  if (run.updatedAt) return now - run.updatedAt > STALE_MS && !alive(run.pid);
+  return now - (run.startedAt ?? 0) > OLD_STALE_MS;
+}
+
+export function recoverRun(root, stateFile, now = Date.now()) {
+  let run;
+  try {
+    run = JSON.parse(fs.readFileSync(stateFile, "utf8"));
+  } catch {
+    return null;
+  }
+  if (!isInterrupted(run, now)) return null;
+  const done = [];
+  const git = (...args) =>
+    execFileSync("git", args, { cwd: root, stdio: ["ignore", "pipe", "pipe"] })
+      .toString()
+      .trim();
+  // Only a run that recorded its stages says what to undo.
+  if (
+    run.pid &&
+    run.from &&
+    (run.stage === "pulled" || run.stage === "deps_swapped")
+  ) {
+    const prev = path.join(root, ".deps/prev");
+    if (run.stage === "deps_swapped" && fs.existsSync(prev)) {
+      fs.rmSync(path.join(root, "node_modules"), {
+        recursive: true,
+        force: true,
+      });
+      fs.renameSync(prev, path.join(root, "node_modules"));
+      done.push("dependencies");
+    }
+    if (git("rev-parse", "HEAD") !== run.from) {
+      git("reset", "-q", "--keep", run.from);
+      done.push("source");
+    }
+  }
+  const next = {
+    ...run,
+    phase: "failed",
+    outcome: "interrupted",
+    finishedAt: now,
+    updatedAt: now,
+    error:
+      "The update was cut off (the computer slept or restarted), so AgentOS stayed on the version it had. It will try again.",
+  };
+  const tmp = `${stateFile}.${process.pid}.tmp`;
+  fs.writeFileSync(tmp, JSON.stringify(next));
+  fs.renameSync(tmp, stateFile);
+  return done;
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  const root = process.cwd();
+  const dir = process.env.DB_PATH
+    ? path.dirname(path.resolve(process.env.DB_PATH))
+    : root;
+  try {
+    const moved = recoverBuild(root);
+    if (moved) console.log(`Update recovery: put ${moved} back as .next`);
+    const undone = recoverRun(root, path.join(dir, "agent-os-update.json"));
+    if (undone)
+      console.log(
+        `Update recovery: marked a cut-off update as interrupted${undone.length ? `; put back the ${undone.join(" and ")}` : ""}`
+      );
+  } catch (error) {
+    console.error("Update recovery skipped:", error.message);
+  }
+}

package/scripts/update-recover.test.ts

@@ -0,0 +1,144 @@
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { beforeEach, describe, expect, it } from "vitest";
+import { tempDir } from "@/lib/test-temp";
+import { isInterrupted, recoverBuild, recoverRun } from "./update-recover.mjs";
+
+let dir: string;
+const state = () => path.join(dir, "agent-os-update.json");
+const git = (...args: string[]) =>
+  execFileSync("git", args, { cwd: dir, encoding: "utf8" }).trim();
+const commit = (msg: string) => {
+  git(
+    "-c",
+    "user.email=t@t",
+    "-c",
+    "user.name=t",
+    "commit",
+    "-q",
+    "--allow-empty",
+    "-m",
+    msg
+  );
+  return git("rev-parse", "HEAD");
+};
+const write = (run: object) => fs.writeFileSync(state(), JSON.stringify(run));
+const read = () => JSON.parse(fs.readFileSync(state(), "utf8"));
+const NOW = Date.now();
+const GONE = 2 ** 22 + 4321;
+
+beforeEach(() => {
+  dir = tempDir("aos-recover-");
+});
+
+describe("recoverBuild", () => {
+  it("finishes a swap cut between its two renames", () => {
+    fs.mkdirSync(path.join(dir, ".next-build"));
+    fs.writeFileSync(path.join(dir, ".next-build/BUILD_ID"), "new");
+    fs.mkdirSync(path.join(dir, ".next-prev"));
+    fs.writeFileSync(path.join(dir, ".next-prev/BUILD_ID"), "old");
+    expect(recoverBuild(dir)).toBe(".next-build");
+    expect(fs.readFileSync(path.join(dir, ".next/BUILD_ID"), "utf8")).toBe(
+      "new"
+    );
+  });
+
+  it("puts the previous build back when there's no new one", () => {
+    fs.mkdirSync(path.join(dir, ".next-prev"));
+    fs.writeFileSync(path.join(dir, ".next-prev/BUILD_ID"), "old");
+    expect(recoverBuild(dir)).toBe(".next-prev");
+    expect(fs.readFileSync(path.join(dir, ".next/BUILD_ID"), "utf8")).toBe(
+      "old"
+    );
+  });
+
+  it("leaves a live build alone", () => {
+    fs.mkdirSync(path.join(dir, ".next"));
+    fs.writeFileSync(path.join(dir, ".next/BUILD_ID"), "live");
+    fs.mkdirSync(path.join(dir, ".next-build"));
+    fs.writeFileSync(path.join(dir, ".next-build/BUILD_ID"), "built");
+    expect(recoverBuild(dir)).toBeNull();
+    expect(fs.readFileSync(path.join(dir, ".next/BUILD_ID"), "utf8")).toBe(
+      "live"
+    );
+  });
+});
+
+describe("recoverRun", () => {
+  let from: string;
+  beforeEach(() => {
+    git("init", "-q");
+    from = commit("one");
+    commit("two");
+  });
+  const run = (over: object = {}) => ({
+    id: "r",
+    phase: "building",
+    from,
+    startedAt: NOW - 10 * 60_000,
+    updatedAt: NOW - 5 * 60_000,
+    pid: GONE,
+    ...over,
+  });
+
+  it("a run cut off after the pull puts the source back", () => {
+    write(run({ stage: "pulled" }));
+    expect(recoverRun(dir, state(), NOW)).toEqual(["source"]);
+    expect(git("rev-parse", "HEAD")).toBe(from);
+    expect(read()).toMatchObject({ phase: "failed", outcome: "interrupted" });
+  });
+
+  it("a run cut off after the dependencies swapped puts them back too", () => {
+    fs.mkdirSync(path.join(dir, "node_modules"));
+    fs.writeFileSync(path.join(dir, "node_modules/which"), "new");
+    fs.mkdirSync(path.join(dir, ".deps/prev"), { recursive: true });
+    fs.writeFileSync(path.join(dir, ".deps/prev/which"), "old");
+    write(run({ stage: "deps_swapped" }));
+    expect(recoverRun(dir, state(), NOW)).toEqual(["dependencies", "source"]);
+    expect(fs.readFileSync(path.join(dir, "node_modules/which"), "utf8")).toBe(
+      "old"
+    );
+  });
+
+  it("a run cut off after the build swapped keeps the new version", () => {
+    const head = git("rev-parse", "HEAD");
+    write(run({ stage: "build_swapped", phase: "restarting" }));
+    expect(recoverRun(dir, state(), NOW)).toEqual([]);
+    expect(git("rev-parse", "HEAD")).toBe(head);
+    expect(read().outcome).toBe("interrupted");
+  });
+
+  it("leaves a run whose runner is alive, or that is finished", () => {
+    write(run({ stage: "pulled", pid: process.pid }));
+    expect(recoverRun(dir, state(), NOW)).toBeNull();
+    write(run({ stage: "pulled", phase: "done" }));
+    expect(recoverRun(dir, state(), NOW)).toBeNull();
+    write(run({ stage: "pulled", updatedAt: NOW - 30_000 }));
+    expect(recoverRun(dir, state(), NOW)).toBeNull();
+  });
+
+  it("an old-style run without stages is only marked, never reset", () => {
+    const head = git("rev-parse", "HEAD");
+    write({ id: "o", phase: "building", from, startedAt: NOW - 3_600_000 });
+    expect(recoverRun(dir, state(), NOW)).toEqual([]);
+    expect(git("rev-parse", "HEAD")).toBe(head);
+  });
+
+  it("isInterrupted needs both a stale heartbeat and a dead runner", () => {
+    expect(isInterrupted(run(), NOW)).toBe(true);
+    expect(isInterrupted(run({ pid: process.pid }), NOW)).toBe(false);
+    expect(isInterrupted(null, NOW)).toBe(false);
+  });
+
+  it("runs as a script without failing the start", () => {
+    write(run({ stage: "pulled" }));
+    const out = execFileSync(
+      process.execPath,
+      [path.join(import.meta.dirname, "update-recover.mjs")],
+      { cwd: dir, encoding: "utf8", env: { ...process.env, DB_PATH: "" } }
+    );
+    expect(out).toContain("interrupted");
+    expect(git("rev-parse", "HEAD")).toBe(from);
+  });
+});