@norskvideo/ctl-test-harness 0.1.70 → 0.1.72

package/daemon.d.ts

@@ -110,11 +110,24 @@ export declare function daemonSpawnEnv(o: {
     overrides?: Record<string, string>;
 }): Record<string, string | undefined>;
 export declare function startDaemon(storeDir: string, options?: StartDaemonOptions): StartedDaemon;
+/** How long one `instance delete` may take before teardown stops waiting for
+ *  it and reaps the instance's containers itself. A healthy delete is seconds. */
+export declare const INSTANCE_DELETE_TIMEOUT_MS = 20000;
 /**
  * Tear down a daemon-based test: delete instances, stop proxy, stop daemon, kill
  * process, rm store. The teardown actions are injected as pre-bound thunks
  * (`deleteInstance`/`stopProxy`/`stopDaemon`) — the caller binds them to
  * norsk-ctl's typed CLI, so no command type leaks into this package.
+ *
+ * Deleting through the daemon is not enough on its own. When bun times a test
+ * out it kills the test's dangling processes, the daemon among them, and the
+ * afterAll that then calls this has a dead daemon to ask: every delete fails
+ * (or hangs into the hook's own timeout) and the instance's containers run on
+ * with nothing left that knows they exist. commentary's publish-docs left a
+ * composition guide's engine running that way for 2.6 hours on 2026-10-03; it
+ * grew to 28 GB and the host OOM-killed the next morning's product CI. So each
+ * delete is bounded, and whatever an instance leaves behind is then removed by
+ * its `norsk-ctl.instance` label, which needs only Docker.
  */
 export declare function cleanupDaemon(opts: {
     deleteInstance: (id: string) => Promise<unknown>;
@@ -125,6 +138,9 @@ export declare function cleanupDaemon(opts: {
     daemon: DaemonProcess | null;
     storeDir: string;
     containers: string[];
+    /** Injected for tests; defaults to the Docker-label reap. */
+    reapInstance?: (id: string) => string[];
+    deleteTimeoutMs?: number;
 }): Promise<void>;
 /** Tear down a direct-API test: stop proxy, remove instances, rm temp dir. */
 export declare function cleanupDirect(opts: {

package/daemon.js

@@ -15,6 +15,7 @@ import { join } from "node:path";
 import { cliCommand } from "./cli-command.js";
 import { rememberDaemonStore } from "./daemon-stores.js";
 import { pollUntil } from "./poll.js";
+import { reapInstance } from "./reap.js";
 import { makeStoreDir, makeTempDir, TEST_TMP_BASE } from "./temp-dir.js";
 export const HEALTH_TIMEOUT_MS = 120_000;
 // The daemon's readiness probe (/api/ready) cannot answer until `serve` has
@@ -28,6 +29,22 @@ export const HEALTH_TIMEOUT_MS = 120_000;
 export const DAEMON_READY_TIMEOUT_MS = 60_000;
 export { makeStoreDir, makeTempDir, pollUntil, TEST_TMP_BASE };
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** "done" once `p` settles, or "timeout" after `ms` — whichever is first. A
+ *  rejection inside the window rejects; one after it is dropped (the caller has
+ *  moved on). */
+async function settleWithin(p, ms) {
+    let timer;
+    const late = new Promise((resolve) => {
+        timer = setTimeout(() => resolve("timeout"), ms);
+    });
+    p.catch(() => { });
+    try {
+        return await Promise.race([p.then(() => "done"), late]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
 function splitLines(text) {
     return text
         .split("\n")
@@ -301,22 +318,48 @@ async function awaitContainerShutdown(containers) {
     if (still.length)
         throw new Error(`Containers still running after cleanup: ${still.join(", ")}`);
 }
+/** How long one `instance delete` may take before teardown stops waiting for
+ *  it and reaps the instance's containers itself. A healthy delete is seconds. */
+export const INSTANCE_DELETE_TIMEOUT_MS = 20_000;
 /**
  * Tear down a daemon-based test: delete instances, stop proxy, stop daemon, kill
  * process, rm store. The teardown actions are injected as pre-bound thunks
  * (`deleteInstance`/`stopProxy`/`stopDaemon`) — the caller binds them to
  * norsk-ctl's typed CLI, so no command type leaks into this package.
+ *
+ * Deleting through the daemon is not enough on its own. When bun times a test
+ * out it kills the test's dangling processes, the daemon among them, and the
+ * afterAll that then calls this has a dead daemon to ask: every delete fails
+ * (or hangs into the hook's own timeout) and the instance's containers run on
+ * with nothing left that knows they exist. commentary's publish-docs left a
+ * composition guide's engine running that way for 2.6 hours on 2026-10-03; it
+ * grew to 28 GB and the host OOM-killed the next morning's product CI. So each
+ * delete is bounded, and whatever an instance leaves behind is then removed by
+ * its `norsk-ctl.instance` label, which needs only Docker.
  */
 export async function cleanupDaemon(opts) {
+    const deleteTimeoutMs = opts.deleteTimeoutMs ?? INSTANCE_DELETE_TIMEOUT_MS;
     for (const id of opts.instances ?? []) {
         try {
-            await opts.deleteInstance(id);
+            if ((await settleWithin(opts.deleteInstance(id), deleteTimeoutMs)) === "timeout")
+                console.error(`cleanup: delete ${id} gave no answer in ${deleteTimeoutMs}ms`);
         }
         catch (e) {
             console.error(`cleanup: delete ${id} failed: ${e}`);
         }
     }
-    if (opts.proxy)
+    const reap = opts.reapInstance ?? reapInstance;
+    for (const id of opts.instances ?? []) {
+        const left = reap(id);
+        if (left.length > 0)
+            console.error(`cleanup: removed ${left.length} container(s) instance ${id} left running`);
+    }
+    // Stop the proxy unless the caller says it never ran one (`proxy: false`).
+    // It used to need `proxy: true`, and three callers passing a real stopProxy
+    // omitted it — so `shutdown --daemon-only` left their proxy pair running.
+    // On the shared ar-arm-vm runner a leftover pair then answered for the next
+    // job's proxy (2026-10-02, norsk-ctl Checks 37048720935).
+    if (opts.proxy ?? true)
         try {
             await opts.stopProxy();
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-test-harness",
-  "version": "0.1.70",
+  "version": "0.1.72",
   "type": "module",
   "exports": {
     ".": {