@dsdenes/devboard 0.1.559 → 0.1.562

package/server/updates.test.ts

@@ -0,0 +1,444 @@
+import { afterAll, describe, expect, test } from "bun:test";
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { join, resolve } from "node:path";
+import { createServer } from "node:net";
+import type { AddressInfo, Socket } from "node:net";
+import {
+  checksumName,
+  installRelease,
+  ReleaseFeedError,
+  tarballName,
+  type ReleaseInfo,
+} from "../lib/release";
+import type { Journal, JournalKind } from "./journal";
+import {
+  DEFAULT_UPDATE_INTERVAL_MS,
+  DEFAULT_MAX_BACKOFF_MS,
+  startUpdater,
+  updateRetryDelay,
+} from "./updates";
+
+const harnessRoot = resolve(import.meta.dir, "..");
+const testRoot = join(harnessRoot, ".tmp");
+const temporaryDirectories: string[] = [];
+
+async function temporaryDirectory(): Promise<string> {
+  await mkdir(testRoot, { recursive: true });
+  const directory = join(testRoot, `updates-${randomUUID()}`);
+  await mkdir(directory, { recursive: true });
+  temporaryDirectories.push(directory);
+  return directory;
+}
+
+async function installedPrefix(version: string): Promise<string> {
+  const prefix = await temporaryDirectory();
+  const directory = join(prefix, "node_modules", "@dsdenes", "devboard");
+  await mkdir(directory, { recursive: true });
+  await writeFile(join(directory, "package.json"), `${JSON.stringify({ version })}\n`);
+  return prefix;
+}
+
+function journalRecorder(): { journal: Journal; records: Array<[JournalKind, string]> } {
+  const records: Array<[JournalKind, string]> = [];
+  return {
+    records,
+    journal: { path: "test", write: (kind, text) => records.push([kind, text]) },
+  };
+}
+
+function release(version: string): ReleaseInfo {
+  return { version, tag: `v${version}`, assets: [] };
+}
+
+function manualSchedule() {
+  const delays: number[] = [];
+  const callbacks: Array<() => Promise<unknown> | void> = [];
+  const cancelled: number[] = [];
+  return {
+    delays,
+    cancelled,
+    schedule: (run: () => Promise<unknown> | void, delayMs: number) => {
+      const index = delays.length;
+      delays.push(delayMs);
+      callbacks.push(run);
+      return {
+        cancel: () => {
+          cancelled.push(index);
+        },
+      };
+    },
+    async fire(index: number) {
+      await callbacks[index]?.();
+    },
+  };
+}
+
+afterAll(async () => {
+  await Promise.all(
+    temporaryDirectories.map((directory) => rm(directory, { recursive: true, force: true })),
+  );
+});
+
+describe("updateRetryDelay", () => {
+  test("uses the base interval until a failure and then backs off", () => {
+    expect(updateRetryDelay(0, 1000, 60_000)).toBe(1000);
+    expect(updateRetryDelay(1, 1000, 60_000)).toBe(2000);
+    expect(updateRetryDelay(2, 1000, 60_000)).toBe(4000);
+    expect(updateRetryDelay(20, 1000, 60_000)).toBe(60_000);
+  });
+
+  test("defaults match the documented polling contract", () => {
+    expect(DEFAULT_UPDATE_INTERVAL_MS).toBe(60_000);
+    expect(DEFAULT_MAX_BACKOFF_MS).toBe(60 * 60_000);
+  });
+});
+
+describe("startUpdater", () => {
+  test("checks immediately, records an install, and reports the latest release", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal, records } = journalRecorder();
+    const updates: string[] = [];
+    let checks = 0;
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      intervalMs: 60_000,
+      startImmediately: false,
+      fetchRelease: async () => {
+        checks += 1;
+        return release("0.1.1");
+      },
+      install: async (info) => {
+        updates.push(info.version);
+        return info.version;
+      },
+      onInstalled: (version) => updates.push(`callback:${version}`),
+    });
+    try {
+      await updater.checkNow();
+      expect(checks).toBe(1);
+      expect(updater.latest()).toBe("0.1.1");
+      expect(updates).toEqual(["0.1.1", "callback:0.1.1"]);
+      expect(records).toEqual([["update", "installed 0.1.1 (was 0.1.0); a restart loads it"]]);
+    } finally {
+      updater.stop();
+    }
+  });
+
+  test("does not install into a prefix without the package", async () => {
+    const prefix = await temporaryDirectory();
+    const { journal } = journalRecorder();
+    let installs = 0;
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: null,
+      journal,
+      startImmediately: false,
+      fetchRelease: async () => release("0.1.4"),
+      install: async () => {
+        installs += 1;
+        return "0.1.4";
+      },
+    });
+    try {
+      expect(await updater.checkNow()).toBe("not-installed");
+      expect(installs).toBe(0);
+      expect(updater.latest()).toBe("0.1.4");
+    } finally {
+      updater.stop();
+    }
+  });
+
+  test("journals a failure and keeps the previous version", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal, records } = journalRecorder();
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      intervalMs: 60_000,
+      startImmediately: false,
+      fetchRelease: async () => release("0.1.2"),
+      install: async () => {
+        throw new Error("checksum mismatch");
+      },
+    });
+    try {
+      expect(await updater.checkNow()).toBe("failed");
+      expect(records).toEqual([
+        ["update-error", "could not install 0.1.2: checksum mismatch"],
+      ]);
+    } finally {
+      updater.stop();
+    }
+  });
+
+  test("journals a stalled download as an update-error and reschedules the check", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const stateDir = await temporaryDirectory();
+    const sockets = new Set<Socket>();
+    const stall = createServer((socket) => {
+      sockets.add(socket);
+      socket.on("close", () => sockets.delete(socket));
+    });
+    await new Promise<void>((settle) => stall.listen(0, "127.0.0.1", () => settle()));
+    const baseUrl = `http://127.0.0.1:${(stall.address() as AddressInfo).port}`;
+    const { journal, records } = journalRecorder();
+    const manual = manualSchedule();
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir,
+      currentVersion: "0.1.0",
+      journal,
+      intervalMs: 1000,
+      schedule: manual.schedule,
+      fetchRelease: async () => ({
+        version: "0.1.2",
+        tag: "v0.1.2",
+        assets: [
+          {
+            name: tarballName("0.1.2"),
+            id: 11,
+            browser_download_url: `${baseUrl}/stall/${tarballName("0.1.2")}`,
+          },
+          {
+            name: checksumName("0.1.2"),
+            id: 12,
+            browser_download_url: `${baseUrl}/stall/${checksumName("0.1.2")}`,
+          },
+        ],
+      }),
+      install: (info) =>
+        installRelease({
+          source: { kind: "gitea", baseUrl, repo: "acme/fixture" },
+          release: info,
+          prefix,
+          stateDir,
+          downloadTimeoutMs: 500,
+        }),
+    });
+    try {
+      await manual.fire(0);
+      expect(records).toEqual([
+        [
+          "update-error",
+          "could not install 0.1.2: release asset download timed out after 500 ms: " +
+            "devboard-0.1.2.tgz",
+        ],
+      ]);
+      // The failed check schedules its own backoff retry, so a stalled
+      // download never turns the update loop off.
+      expect(manual.delays).toEqual([0, 2000]);
+    } finally {
+      updater.stop();
+      for (const socket of sockets) socket.destroy();
+      stall.close();
+    }
+  }, 20_000);
+
+  test("journals an unreachable feed as context, never an update-error", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal, records } = journalRecorder();
+    const manual = manualSchedule();
+    const offline =
+      'gh api releases/latest failed: Get "https://api.github.com/repos/dsdenes/devboard/releases/latest": ' +
+      "dial tcp 140.82.121.5:443: connect: network is unreachable";
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      intervalMs: 1000,
+      schedule: manual.schedule,
+      fetchRelease: async () => {
+        throw new Error(offline);
+      },
+      install: async (info) => info.version,
+    });
+    await manual.fire(0);
+    expect(records).toEqual([["activity", `release check skipped: ${offline}`]]);
+    // The offline check still backs off instead of polling at the base interval.
+    expect(manual.delays).toEqual([0, 2000]);
+    updater.stop();
+  });
+
+  test("journals a missing release client as context, never an update-error", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal, records } = journalRecorder();
+    const manual = manualSchedule();
+    const missing = Object.assign(
+      new Error("ENOENT: no such file or directory, posix_spawn 'gh'"),
+      { code: "ENOENT" },
+    );
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      intervalMs: 1000,
+      schedule: manual.schedule,
+      fetchRelease: async () => {
+        throw missing;
+      },
+      install: async (info) => info.version,
+    });
+    await manual.fire(0);
+    expect(records).toEqual([["activity", `release check skipped: ${missing.message}`]]);
+    // The skipped check still backs off instead of polling at the base interval.
+    expect(manual.delays).toEqual([0, 2000]);
+    updater.stop();
+  });
+
+  test("journals an unreadable release description as context, never an update-error", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal, records } = journalRecorder();
+    const manual = manualSchedule();
+    const unreadable = new ReleaseFeedError(
+      "gh returned an unreadable release description: unexpected gh arguments: api releases/latest",
+    );
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      intervalMs: 1000,
+      schedule: manual.schedule,
+      fetchRelease: async () => {
+        throw unreadable;
+      },
+      install: async (info) => info.version,
+    });
+    await manual.fire(0);
+    expect(records).toEqual([["activity", `release check skipped: ${unreadable.message}`]]);
+    // The skipped check still backs off instead of polling at the base interval.
+    expect(manual.delays).toEqual([0, 2000]);
+    updater.stop();
+  });
+
+  test("journals a feed error that is not connectivity as an update-error", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal, records } = journalRecorder();
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      startImmediately: false,
+      fetchRelease: async () => {
+        throw new Error("gh api releases/latest failed: HTTP 401: Bad credentials");
+      },
+      install: async (info) => info.version,
+    });
+    try {
+      expect(await updater.checkNow()).toBe("failed");
+      expect(records).toEqual([
+        [
+          "update-error",
+          "release check failed: gh api releases/latest failed: HTTP 401: Bad credentials",
+        ],
+      ]);
+    } finally {
+      updater.stop();
+    }
+  });
+
+  test("single-flights concurrent checks", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal } = journalRecorder();
+    let calls = 0;
+    const gate = Promise.withResolvers<ReleaseInfo>();
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      startImmediately: false,
+      fetchRelease: async () => {
+        calls += 1;
+        return gate.promise;
+      },
+      install: async (info) => info.version,
+    });
+    try {
+      const first = updater.checkNow();
+      const second = updater.checkNow();
+      gate.resolve(release("0.1.5"));
+      expect(await second).toBe("installed");
+      expect(await first).toBe("installed");
+      expect(calls).toBe(1);
+    } finally {
+      updater.stop();
+    }
+  });
+
+  test("checks at the startup turn and then on the overridden interval", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal } = journalRecorder();
+    const manual = manualSchedule();
+    let calls = 0;
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      intervalMs: 1000,
+      schedule: manual.schedule,
+      fetchRelease: async () => {
+        calls += 1;
+        return release("0.1.0");
+      },
+      install: async (info) => info.version,
+    });
+    await manual.fire(0);
+    expect(calls).toBe(1);
+    expect(manual.delays).toEqual([0, 1000]);
+    await manual.fire(1);
+    expect(calls).toBe(2);
+    expect(manual.delays).toEqual([0, 1000, 1000]);
+    updater.stop();
+    expect(manual.cancelled).toEqual([2]);
+  });
+
+  test("backs off after a failure instead of retrying at the base interval", async () => {
+    const prefix = await installedPrefix("0.1.0");
+    const { journal } = journalRecorder();
+    const manual = manualSchedule();
+    let calls = 0;
+    const updater = startUpdater({
+      repoDir: prefix,
+      prefixDir: prefix,
+      stateDir: prefix,
+      currentVersion: "0.1.0",
+      journal,
+      intervalMs: 1000,
+      schedule: manual.schedule,
+      fetchRelease: async () => {
+        calls += 1;
+        throw new Error("gh is not authenticated");
+      },
+      install: async (info) => info.version,
+    });
+    await manual.fire(0);
+    expect(calls).toBe(1);
+    expect(manual.delays).toEqual([0, 2000]);
+    await manual.fire(1);
+    expect(calls).toBe(2);
+    expect(manual.delays).toEqual([0, 2000, 4000]);
+    updater.stop();
+  });
+});

package/server/updates.ts

@@ -0,0 +1,198 @@
+import {
+  installRelease,
+  installedHarnessVersion,
+  isNewerVersion,
+  isReleaseUnreachable,
+  latestRelease,
+  releaseSource,
+  ReleaseFeedError,
+  type ReleaseInfo,
+  type ReleaseSource,
+} from "../lib/release";
+import type { Journal } from "./journal";
+
+/**
+ * Automatic harness updates. The server checks npm at startup and then on an
+ * interval, installs a newer artifact into the host prefix while it keeps
+ * serving, and leaves the restart to the operator's reload control. When npm
+ * cannot serve the artifact and a Gitea release is configured, the release
+ * tarball and its checksum are the fallback. A failure is journaled and
+ * retried with exponential backoff; the running package is never replaced by
+ * a failed or partial install.
+ */
+
+export const DEFAULT_UPDATE_INTERVAL_MS = 60_000;
+/** A repeated failure backs off to this ceiling, not past it. */
+export const DEFAULT_MAX_BACKOFF_MS = 60 * 60_000;
+
+export type UpdateOutcome = "current" | "installed" | "not-installed" | "failed" | "stopped";
+
+export interface UpdateOptions {
+  repoDir: string;
+  /** The host's npm prefix: `<host>/.devboard`. */
+  prefixDir: string;
+  stateDir: string;
+  /** Version this process started with. */
+  currentVersion: string | null;
+  journal: Journal;
+  env?: NodeJS.ProcessEnv;
+  intervalMs?: number;
+  maxBackoffMs?: number;
+  /** Injectable feed for tests and fixtures; defaults to the environment's. */
+  source?: ReleaseSource;
+  fetchRelease?: () => Promise<ReleaseInfo>;
+  install?: (release: ReleaseInfo) => Promise<string>;
+  onLatest?: (version: string) => void;
+  onInstalled?: (version: string) => void;
+  /** Test seam; production checks as soon as the event loop turns. */
+  startImmediately?: boolean;
+  /** Injectable scheduler; production uses an unref'd timeout. */
+  schedule?: (run: () => Promise<unknown> | void, delayMs: number) => { cancel: () => void };
+}
+
+export interface Updater {
+  /** The newest release a check has seen, or null before the first check. */
+  latest(): string | null;
+  /** Runs one check now; concurrent calls share the in-flight check. */
+  checkNow(): Promise<UpdateOutcome>;
+  /** Stops the next scheduled check; an in-flight check finishes. */
+  stop(): void;
+}
+
+function positiveNumber(value: string | undefined, fallback: number): number {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function detailOf(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/** The next check delay: the base interval until a failure backs it off. */
+export function updateRetryDelay(failures: number, intervalMs: number, maxBackoffMs: number): number {
+  if (failures <= 0) return intervalMs;
+  return Math.min(intervalMs * 2 ** Math.min(failures, 16), maxBackoffMs);
+}
+
+export function startUpdater(options: UpdateOptions): Updater {
+  const env = options.env ?? process.env;
+  const intervalMs =
+    options.intervalMs ?? positiveNumber(env.DEVBOARD_UPDATE_INTERVAL_MS, DEFAULT_UPDATE_INTERVAL_MS);
+  const maxBackoffMs =
+    options.maxBackoffMs ??
+    positiveNumber(env.DEVBOARD_UPDATE_BACKOFF_MAX_MS, DEFAULT_MAX_BACKOFF_MS);
+  const source = options.source ?? releaseSource(env, options.stateDir);
+  const fetchRelease = options.fetchRelease ?? (() => latestRelease(source));
+  const install =
+    options.install ??
+    ((release: ReleaseInfo) =>
+      installRelease({
+        source,
+        release,
+        prefix: options.prefixDir,
+        stateDir: options.stateDir,
+        env,
+      }));
+
+  let latestVersion: string | null = null;
+  let failures = 0;
+  let timer: { cancel: () => void } | undefined;
+  let stopped = false;
+  let inFlight: Promise<UpdateOutcome> | undefined;
+  const scheduleTimer =
+    options.schedule ??
+    ((run: () => Promise<unknown> | void, delayMs: number) => {
+      const handle = setTimeout(() => void run(), delayMs);
+      handle.unref?.();
+      return { cancel: () => clearTimeout(handle) };
+    });
+
+  function schedule(delayMs: number) {
+    if (stopped) return;
+    timer = scheduleTimer(() => {
+      timer = undefined;
+      return checkNow();
+    }, delayMs);
+  }
+
+  function nextDelay(): number {
+    return updateRetryDelay(failures, intervalMs, maxBackoffMs);
+  }
+
+  async function runCheck(): Promise<UpdateOutcome> {
+    let release: ReleaseInfo;
+    try {
+      release = await fetchRelease();
+    } catch (error) {
+      failures += 1;
+      const detail = detailOf(error);
+      if (isReleaseUnreachable(error) || error instanceof ReleaseFeedError) {
+        // An offline host, a missing release client, or a feed that answers
+        // with something other than a release description is not a harness
+        // defect: the check backs off like any other failure, but the record
+        // is context, never a watchdog finding.
+        options.journal.write("activity", `release check skipped: ${detail}`);
+      } else {
+        options.journal.write("update-error", `release check failed: ${detail}`);
+      }
+      return "failed";
+    }
+    latestVersion = release.version;
+    options.onLatest?.(release.version);
+
+    try {
+      const installed = await installedHarnessVersion(options.prefixDir);
+      // The canonical source checkout has no npm prefix; never install into it.
+      if (installed === null) return "not-installed";
+      if (!isNewerVersion(release.version, installed)) {
+        failures = 0;
+        return "current";
+      }
+      const installedVersion = await install(release);
+      failures = 0;
+      options.journal.write(
+        "update",
+        `installed ${installedVersion} (was ${installed}); a restart loads it`,
+      );
+      options.onInstalled?.(installedVersion);
+      return "installed";
+    } catch (error) {
+      failures += 1;
+      options.journal.write(
+        "update-error",
+        `could not install ${release.version}: ${detailOf(error)}`,
+      );
+      return "failed";
+    }
+  }
+
+  function checkNow(): Promise<UpdateOutcome> {
+    if (stopped) return Promise.resolve("stopped");
+    if (inFlight !== undefined) return inFlight;
+    inFlight = runCheck()
+      .catch((error: unknown) => {
+        failures += 1;
+        options.journal.write("update-error", `update check crashed: ${detailOf(error)}`);
+        return "failed" as const;
+      })
+      .finally(() => {
+        inFlight = undefined;
+        schedule(nextDelay());
+      });
+    return inFlight;
+  }
+
+  // The first check runs as soon as the event loop turns, so the server binds
+  // its port first and an idle process still checks immediately.
+  if (options.startImmediately ?? true) schedule(0);
+
+  return {
+    latest: () => latestVersion,
+    checkNow,
+    stop: () => {
+      stopped = true;
+      timer?.cancel();
+      timer = undefined;
+    },
+  };
+}