@norskvideo/ctl-test-harness 0.1.34 → 0.1.35

package/demo/run.js

@@ -27,13 +27,14 @@
 import { spawn, spawnSync } from "node:child_process";
 import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSync, rmSync, writeFileSync, } from "node:fs";
 import { homedir } from "node:os";
-import { basename, dirname, isAbsolute, join, resolve } from "node:path";
+import { dirname, isAbsolute, join, resolve } from "node:path";
 import { parseManifestSeed, repoOf } from "@norskvideo/ctl-sdk/manifest-seed";
 import { DOCKER_NETWORK_NAME, ensureRunnerOnNetwork, netReachMode } from "../container-net.js";
 import { cleanupDaemon, requireLicenseFile, runCli, startDaemon, } from "../daemon.js";
 import { hashSlot } from "../harness-config.js";
 import { ctlSupportsNoPublish, runnerContainerUser } from "../launch.js";
 import { pollUntil } from "../poll.js";
+import { nukePathAsRoot, reportForeignOwners } from "../root-nuke.js";
 import { startSrtSources } from "../source-pump.js";
 import { makeStoreDir } from "../temp-dir.js";
 const DEMO_PORT_BASE = 35000;
@@ -41,7 +42,6 @@ const DEMO_BAND_WIDTH = 20;
 const DEMO_BANDS = 50;
 const DEFAULT_DAEMON_PORT = 8333;
 const DEFAULT_PROXY_PORT = 443;
-const NUKE_IMAGE = "alpine:3";
 const DEFAULT_DEV_READY_PATH = "/manifest.json";
 const SEED_FILE = "manifest.seed.json";
 const TEMPLATES_DIR = "product-templates";
@@ -290,21 +290,7 @@ export function defaultDemoDeps(cwd) {
                 await h.stop();
         },
         cleanup: (opts) => cleanupDaemon({ ...opts, stopProxy: async () => { }, proxy: false }),
-        nukeStoreAsRoot: (storeDir) => {
-            spawnSync("docker", [
-                "run",
-                "--rm",
-                "--user",
-                "0:0",
-                "-v",
-                `${dirname(storeDir)}:/base`,
-                "--entrypoint",
-                "sh",
-                NUKE_IMAGE,
-                "-c",
-                `rm -rf /base/${basename(storeDir)}`,
-            ]);
-        },
+        nukeStoreAsRoot: (storeDir) => nukePathAsRoot(storeDir),
         storeExists: (storeDir) => existsSync(storeDir),
         ensureNetwork: () => ensureRunnerOnNetwork(),
         supportsNoPublish: ctlSupportsNoPublish,
@@ -658,6 +644,7 @@ export class DemoSession {
         }
         catch (e) {
             this.deps.log(`cleanup could not remove the store itself (${e instanceof Error ? e.message : String(e)}); nuking as root`);
+            reportForeignOwners(this.storeDir, (l) => this.deps.log(l));
         }
         this.stopExtras();
         this.dev?.kill();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-test-harness",
-  "version": "0.1.34",
+  "version": "0.1.35",
   "type": "module",
   "exports": {
     ".": {

package/root-nuke.d.ts

@@ -0,0 +1,46 @@
+/** Minimal shape of `spawnSync` this needs, so tests can inject. */
+export type NukeSpawn = (cmd: string, argv: string[], opts?: unknown) => {
+    error?: unknown;
+    status?: number | null;
+};
+/** Refuse a target that would turn the nuke into something catastrophic. The
+ *  mount is the target's PARENT, so a target with no leaf ("/" or a trailing
+ *  slash) means "delete the mount", and a relative path resolves against a cwd
+ *  the container does not share. These are programming errors, not runtime
+ *  conditions: throw loudly rather than run a root rm against a guess. */
+export declare function assertNukeTarget(target: string): void;
+/** Docker argv for the nuke. Two things matter here:
+ *
+ *  1. The PARENT is mounted and the leaf removed by name -- mounting the target
+ *     itself would let the container empty it but never unlink it, leaving the
+ *     dir behind and the caller's existence check true.
+ *  2. The leaf is passed as a positional ARGUMENT, never interpolated into the
+ *     `sh -c` script. The prune feeds this arbitrary readdir names, and a
+ *     directory called `x; rm -rf /base` would otherwise wipe every sibling --
+ *     including a live sibling run -- from inside a root container. */
+export declare function nukeArgv(target: string): string[];
+/** Remove `target` as root. Returns whether the nuke actually ran: `spawnSync`
+ *  reports a missing docker in its RESULT rather than throwing, and a caller
+ *  that retries a doomed spawn forever (silently) is the failure mode to avoid.
+ *  Throws only for an unsafe target -- see {@link assertNukeTarget}. */
+export declare function nukePathAsRoot(target: string, spawn?: NukeSpawn): boolean;
+export interface OwnerScanDeps {
+    readdir: (path: string) => string[];
+    isDir: (path: string) => boolean;
+    uidOf: (path: string) => number;
+}
+/** Paths under `dir` owned by a uid that is not `selfUid`, with the owning uid.
+ *
+ *  The diagnostic that ends the guessing. Teardown deletes this evidence every
+ *  run, so nobody has ever observed WHICH uid owns the litter -- and "root" and
+ *  "a container's baked-in service account" EACCES a non-root runner
+ *  identically while implying completely different causes. Bounded so a large
+ *  tree cannot flood a CI log. */
+export declare function foreignOwnedPaths(dir: string, selfUid: number, deps: OwnerScanDeps, limit?: number): Array<{
+    path: string;
+    uid: number;
+}>;
+/** Log which uids own what under `dir`, for a teardown whose non-root rm just
+ *  failed. Without this the harness deletes the only evidence of the cause on
+ *  every run, leaving "root-owned" an inference nobody has ever checked. */
+export declare function reportForeignOwners(dir: string, log: (line: string) => void): void;

package/root-nuke.js

@@ -0,0 +1,129 @@
+// Removing a path a container left owned by someone the runner is not. A
+// launched instance bind-mounts host dirs into containers; unless the launch
+// delivers a container user, whatever those containers write is owned by the
+// image's own uid (root, or a baked-in service account) and the non-root runner
+// cannot remove it. This is the tolerance half -- a throwaway container,
+// running as root, deletes the path from the inside. launch.ts is the
+// prevention half.
+import { spawnSync } from "node:child_process";
+import { lstatSync, readdirSync } from "node:fs";
+import { basename, dirname, isAbsolute } from "node:path";
+const NUKE_IMAGE = "alpine:3";
+const NUKE_TIMEOUT_MS = 60_000;
+/** Refuse a target that would turn the nuke into something catastrophic. The
+ *  mount is the target's PARENT, so a target with no leaf ("/" or a trailing
+ *  slash) means "delete the mount", and a relative path resolves against a cwd
+ *  the container does not share. These are programming errors, not runtime
+ *  conditions: throw loudly rather than run a root rm against a guess. */
+export function assertNukeTarget(target) {
+    if (!isAbsolute(target))
+        throw new Error(`refusing to nuke a relative path: ${target}`);
+    const leaf = basename(target);
+    if (leaf === "" || leaf === "." || leaf === "..")
+        throw new Error(`refusing to nuke a path with no leaf: ${target}`);
+    if (leaf === "*" || leaf.includes("*"))
+        throw new Error(`refusing to nuke a glob: ${target}`);
+    // dirname is what gets bind-mounted; "/" would hand the whole host to a root
+    // container. Every real target (a store under test-temp) is nested deeper.
+    if (dirname(target) === "/" || dirname(target) === target) {
+        throw new Error(`refusing to nuke a top-level path (its parent would be the mount): ${target}`);
+    }
+}
+/** Docker argv for the nuke. Two things matter here:
+ *
+ *  1. The PARENT is mounted and the leaf removed by name -- mounting the target
+ *     itself would let the container empty it but never unlink it, leaving the
+ *     dir behind and the caller's existence check true.
+ *  2. The leaf is passed as a positional ARGUMENT, never interpolated into the
+ *     `sh -c` script. The prune feeds this arbitrary readdir names, and a
+ *     directory called `x; rm -rf /base` would otherwise wipe every sibling --
+ *     including a live sibling run -- from inside a root container. */
+export function nukeArgv(target) {
+    return [
+        "run",
+        "--rm",
+        "--user",
+        "0:0",
+        "-v",
+        `${dirname(target)}:/base`,
+        "--entrypoint",
+        "sh",
+        NUKE_IMAGE,
+        "-c",
+        'rm -rf -- "/base/$1"',
+        "sh",
+        basename(target),
+    ];
+}
+/** Remove `target` as root. Returns whether the nuke actually ran: `spawnSync`
+ *  reports a missing docker in its RESULT rather than throwing, and a caller
+ *  that retries a doomed spawn forever (silently) is the failure mode to avoid.
+ *  Throws only for an unsafe target -- see {@link assertNukeTarget}. */
+export function nukePathAsRoot(target, spawn = spawnSync) {
+    assertNukeTarget(target);
+    const r = spawn("docker", nukeArgv(target), { timeout: NUKE_TIMEOUT_MS });
+    return !r?.error;
+}
+/** Paths under `dir` owned by a uid that is not `selfUid`, with the owning uid.
+ *
+ *  The diagnostic that ends the guessing. Teardown deletes this evidence every
+ *  run, so nobody has ever observed WHICH uid owns the litter -- and "root" and
+ *  "a container's baked-in service account" EACCES a non-root runner
+ *  identically while implying completely different causes. Bounded so a large
+ *  tree cannot flood a CI log. */
+export function foreignOwnedPaths(dir, selfUid, deps, limit = 40) {
+    const found = [];
+    const stack = [dir];
+    while (stack.length > 0 && found.length < limit) {
+        const cur = stack.pop();
+        if (cur === undefined)
+            break;
+        let names;
+        try {
+            names = deps.readdir(cur);
+        }
+        catch {
+            continue;
+        }
+        for (const name of names) {
+            if (found.length >= limit)
+                break;
+            const p = `${cur}/${name}`;
+            try {
+                const uid = deps.uidOf(p);
+                if (uid !== selfUid)
+                    found.push({ path: p, uid });
+                if (deps.isDir(p))
+                    stack.push(p);
+            }
+            catch {
+                // vanished mid-scan, or unreadable -- diagnostics are best-effort
+            }
+        }
+    }
+    return found;
+}
+/** Log which uids own what under `dir`, for a teardown whose non-root rm just
+ *  failed. Without this the harness deletes the only evidence of the cause on
+ *  every run, leaving "root-owned" an inference nobody has ever checked. */
+export function reportForeignOwners(dir, log) {
+    const selfUid = process.getuid?.();
+    if (selfUid === undefined)
+        return;
+    const found = foreignOwnedPaths(dir, selfUid, {
+        readdir: (p) => readdirSync(p),
+        isDir: (p) => lstatSync(p).isDirectory(),
+        uidOf: (p) => lstatSync(p).uid,
+    });
+    if (found.length === 0) {
+        log(`owners: nothing under ${dir} is owned by a uid other than ${selfUid}`);
+        return;
+    }
+    const byUid = new Map();
+    for (const f of found)
+        byUid.set(f.uid, (byUid.get(f.uid) ?? 0) + 1);
+    const summary = [...byUid.entries()].map(([uid, n]) => `uid ${uid} x${n}`).join(", ");
+    log(`owners: runner is uid ${selfUid}; foreign-owned under ${dir}: ${summary}`);
+    for (const f of found.slice(0, 10))
+        log(`owners:   uid ${f.uid}  ${f.path}`);
+}

package/smoke.js

@@ -17,15 +17,15 @@
 // studio-state, container-net). Argv, not Command: @norskvideo/ctl-commands is
 // not a dependency of this package (daemon.ts's cleanupDaemon has the same
 // rule); a product wanting typed commands bridges with toArgv in its own spec.
-import { spawnSync } from "node:child_process";
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { basename, dirname, join } from "node:path";
+import { join } from "node:path";
 import { dumpInstanceContainerLogs } from "./container-logs.js";
 import { ensureRunnerOnNetwork, netReachMode, STUDIO_INTERNAL_PORT, studioBaseFrom, } from "./container-net.js";
 import { cleanupDaemon, requireLicenseFile, runCli, startDaemon, } from "./daemon.js";
 import { hashSlot } from "./harness-config.js";
 import { ctlSupportsNoPublish, runnerContainerUser } from "./launch.js";
 import { pollUntil } from "./poll.js";
+import { nukePathAsRoot, reportForeignOwners } from "./root-nuke.js";
 import { startSrtSources } from "./source-pump.js";
 import { applyTestHost, fetchComponentState, fetchComponents, fetchStreamMappings, isSrtListenerState, } from "./studio-state.js";
 import { makeStoreDir } from "./temp-dir.js";
@@ -33,7 +33,6 @@ const SMOKE_PORT_BASE = 33000;
 const SMOKE_BAND_WIDTH = 20;
 const SMOKE_BANDS = 50;
 const STUDIO_HOST_PORT_PARAM = "STUDIO_HOST_PORT";
-const NUKE_IMAGE = "alpine:3";
 export function smokePorts(slug, overrides = {}) {
     const daemonPort = SMOKE_PORT_BASE + hashSlot(`smoke:${slug}`, SMOKE_BANDS) * SMOKE_BAND_WIDTH;
     return {
@@ -63,21 +62,7 @@ export const defaultSmokeDeps = {
         fetch: (url) => fetch(url, { signal: AbortSignal.timeout(5000) }),
     },
     cleanup: (opts) => cleanupDaemon({ ...opts, stopProxy: async () => { }, proxy: false }),
-    nukeStoreAsRoot: (storeDir) => {
-        spawnSync("docker", [
-            "run",
-            "--rm",
-            "--user",
-            "0:0",
-            "-v",
-            `${dirname(storeDir)}:/base`,
-            "--entrypoint",
-            "sh",
-            NUKE_IMAGE,
-            "-c",
-            `rm -rf /base/${basename(storeDir)}`,
-        ]);
-    },
+    nukeStoreAsRoot: (storeDir) => nukePathAsRoot(storeDir),
     storeExists: (storeDir) => existsSync(storeDir),
     ensureNetwork: () => ensureRunnerOnNetwork(),
     supportsNoPublish: ctlSupportsNoPublish,
@@ -418,6 +403,7 @@ export async function runProductSmoke(slug, spec, deps = defaultSmokeDeps) {
     }
     catch (e) {
         deps.log(`${slug}: cleanup could not remove the store itself (${e instanceof Error ? e.message : String(e)}); nuking as root`);
+        reportForeignOwners(storeDir, (l) => deps.log(l));
     }
     deps.nukeStoreAsRoot(storeDir);
     if (journeyError !== undefined)

package/temp-dir.d.ts

@@ -1,6 +1,20 @@
 /** Create a uniquely-named temp dir under an explicit base. */
 export declare function makeTempDirUnder(base: string, prefix: string): string;
 export declare const TEST_TMP_BASE: string;
+/** Injection seam for {@link pruneStaleTempDirs}. */
+export interface PruneDeps {
+    now: () => number;
+    readdir: (base: string) => string[];
+    mtimeMs: (path: string) => number;
+    /** The pid that created this dir, if it recorded one. */
+    ownerPid: (path: string) => number | undefined;
+    pidAlive: (pid: number) => boolean;
+    rm: (path: string) => void;
+    /** Returns whether the nuke could actually run (docker present). */
+    nukeAsRoot: (path: string) => boolean;
+    log: (line: string) => void;
+}
+export declare function pruneStaleTempDirsWith(base: string, deps: PruneDeps): void;
 /** Create a uniquely-named temp dir under the consumer-local test base. */
 export declare function makeTempDir(prefix?: string): string;
 export declare function makeStoreDir(prefix?: string): string;

package/temp-dir.js

@@ -11,36 +11,124 @@
 // forwards /tmp events too, which is why that env never surfaced it.
 // Native-Linux Docker is unaffected (real kernel inotify). A repo-local
 // `test-temp/` is on the shared filesystem so events forward.
-import { mkdirSync, mkdtempSync, readdirSync, rmSync, statSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
+import { nukePathAsRoot } from "./root-nuke.js";
+const OWNER_FILE = ".harness-owner";
 /** Create a uniquely-named temp dir under an explicit base. */
 export function makeTempDirUnder(base, prefix) {
     mkdirSync(base, { recursive: true });
     pruneStaleTempDirs(base);
-    return mkdtempSync(join(base, prefix));
+    const dir = mkdtempSync(join(base, prefix));
+    // Records who owns this dir so a later prune can tell a live session from
+    // litter -- see pruneStaleTempDirsWith. Best-effort: an unstamped dir simply
+    // falls back to the age test.
+    try {
+        writeFileSync(join(dir, OWNER_FILE), `${process.pid}\n`);
+    }
+    catch { }
+    return dir;
 }
 export const TEST_TMP_BASE = process.env.NORSK_CTL_TEST_TMP ?? join(process.cwd(), "test-temp");
+const STALE_AFTER_MS = 60 * 60 * 1000;
+function isPermissionError(e) {
+    const code = e?.code;
+    return code === "EACCES" || code === "EPERM";
+}
 // Best-effort cleanup of dirs orphaned by a crashed run whose teardown never got
 // to rm them (a repo-local base isn't OS-reclaimed like /tmp, so prune here).
-// The 1h cutoff never races a live run — no test runs near that long — so this
-// can't delete a sibling run's active dir.
-function pruneStaleTempDirs(base) {
+//
+// A plain rm is NOT enough: a job killed mid-run (CI cancel, timeout, SIGKILL)
+// never reaches its teardown, so its containers' foreign-owned bind-mount dirs
+// survive and EACCES every later non-root rm. Swallowing that let the litter
+// accumulate forever on a shared runner -- the mess `clean: false` exists to
+// route around. So escalate to a root container.
+//
+// Age alone is NOT a safe staleness test once that escalation exists. A
+// directory's mtime only moves when its direct entries change, and `dev-loop up`
+// holds a daemon indefinitely against a makeStoreDir store -- so an idle but
+// very much LIVE store looks stale, and the escalation would guarantee its
+// deletion. Every dir records its creating pid; a dir whose owner is still
+// running is never touched. Skipping wrongly just leaves litter, deleting
+// wrongly destroys a running session, so the tie breaks toward skipping.
+export function pruneStaleTempDirsWith(base, deps) {
+    let names;
     try {
-        const cutoff = Date.now() - 60 * 60 * 1000;
-        for (const name of readdirSync(base)) {
-            const p = join(base, name);
+        names = deps.readdir(base);
+    }
+    catch {
+        return; // base doesn't exist yet -- nothing to prune
+    }
+    const cutoff = deps.now() - STALE_AFTER_MS;
+    let nukeUnavailable = false;
+    for (const name of names) {
+        if (name === OWNER_FILE)
+            continue;
+        const p = join(base, name);
+        try {
+            if (deps.mtimeMs(p) >= cutoff)
+                continue;
+        }
+        catch {
+            continue; // racing a sibling run's teardown, or a transient stat error
+        }
+        const owner = deps.ownerPid(p);
+        if (owner !== undefined && deps.pidAlive(owner))
+            continue;
+        try {
+            deps.rm(p);
+        }
+        catch (e) {
+            // Only a permission error means "someone else owns this"; EBUSY or a
+            // teardown race does not justify spinning up a root container.
+            if (!isPermissionError(e) || nukeUnavailable)
+                continue;
+            if (!deps.nukeAsRoot(p)) {
+                nukeUnavailable = true;
+                deps.log(`temp-dir: cannot root-nuke ${p} (docker unavailable); leaving stale dirs in ${base}`);
+            }
+        }
+    }
+}
+// The latch is here rather than in the pure function so it spans calls: prune
+// runs on every makeTempDir, and re-spawning a doomed docker each time would be
+// a silent per-test cost.
+let nukeKnownUnavailable = false;
+function pruneStaleTempDirs(base) {
+    pruneStaleTempDirsWith(base, {
+        now: () => Date.now(),
+        readdir: (b) => readdirSync(b),
+        mtimeMs: (p) => statSync(p).mtimeMs,
+        ownerPid: (p) => {
             try {
-                if (statSync(p).mtimeMs < cutoff)
-                    rmSync(p, { recursive: true, force: true });
+                const raw = Number.parseInt(readFileSync(join(p, OWNER_FILE), "utf8").trim(), 10);
+                return Number.isFinite(raw) ? raw : undefined;
             }
             catch {
-                // ignore — racing teardown of a sibling run, or a transient stat error
+                return undefined;
             }
-        }
-    }
-    catch {
-        // base doesn't exist yet — nothing to prune
-    }
+        },
+        pidAlive: (pid) => {
+            try {
+                process.kill(pid, 0);
+                return true;
+            }
+            catch (e) {
+                // EPERM means it exists but belongs to someone else -- still alive.
+                return e.code === "EPERM";
+            }
+        },
+        rm: (p) => rmSync(p, { recursive: true, force: true }),
+        nukeAsRoot: (p) => {
+            if (nukeKnownUnavailable)
+                return false;
+            const ran = nukePathAsRoot(p);
+            if (!ran)
+                nukeKnownUnavailable = true;
+            return ran;
+        },
+        log: (line) => console.warn(line),
+    });
 }
 /** Create a uniquely-named temp dir under the consumer-local test base. */
 export function makeTempDir(prefix = "norsk-ctl-") {