@magland/mochi 0.3.1 → 0.4.0

package/dist/runner/actions.js

@@ -45,12 +45,17 @@ const os = __importStar(require("os"));
 const path = __importStar(require("path"));
 const YAML = __importStar(require("yaml"));
 const actionref_1 = require("../ci/actionref");
+const download_1 = require("./download");
 // Fetching actions and reading their definitions. Actions come from a forge
 // over HTTPS (github.com by default) as a source tarball, and are cached on
 // the runner's disk so a repeated job does not re-download them.
 //
 // Nothing here executes anything: the store hands back a directory and a
 // parsed action.yml, and steps.ts decides what to do with them.
+// The most an action's source archive may be. The largest actions in common
+// use are a few megabytes of bundled JavaScript; this is a ceiling on what a
+// `uses:` line can make the runner hold, not a budget anyone should approach.
+const MAX_ACTION_BYTES = 256 * 1024 * 1024;
 class ActionError extends Error {
 }
 exports.ActionError = ActionError;
@@ -400,7 +405,13 @@ class ActionStore {
         fs.rmSync(tmp, { recursive: true, force: true });
         fs.mkdirSync(tmp, { recursive: true });
         const tarball = path.join(tmp, 'source.tar.gz');
-        fs.writeFileSync(tarball, Buffer.from(await res.arrayBuffer()));
+        try {
+            await (0, download_1.downloadTo)(res, tarball, MAX_ACTION_BYTES);
+        }
+        catch (e) {
+            fs.rmSync(tmp, { recursive: true, force: true });
+            throw new ActionError(`could not fetch ${ref.raw}: ${e instanceof Error ? e.message : e}`);
+        }
         const extracted = path.join(tmp, 'src');
         fs.mkdirSync(extracted);
         try {

package/dist/runner/client.js

@@ -84,6 +84,7 @@ class JobClient {
     buffer = [];
     flushing = false;
     cancelSeen = false;
+    cancelListeners = [];
     timer = null;
     constructor(runner, spec) {
         this.runner = runner;
@@ -139,17 +140,34 @@ class JobClient {
     cancelled() {
         return this.cancelSeen;
     }
+    // Called once, the first time a heartbeat learns the job should stop. The
+    // flag above is polled between steps; this is for the step that is running
+    // now, which nothing polls and which only removing the container can end.
+    onCancel(listener) {
+        if (this.cancelSeen)
+            listener();
+        else
+            this.cancelListeners.push(listener);
+    }
+    markCancelled() {
+        if (this.cancelSeen)
+            return;
+        this.cancelSeen = true;
+        const listeners = this.cancelListeners.splice(0, this.cancelListeners.length);
+        for (const l of listeners)
+            l();
+    }
     async heartbeat() {
         try {
             const res = await fetch(`${this.base()}/heartbeat`, { method: 'POST', headers: this.headers() });
             if (res.status === 409) {
                 // The server took the job back (lease expired, run cancelled). Stop.
-                this.cancelSeen = true;
+                this.markCancelled();
                 return;
             }
             const body = (await res.json());
             if (body.cancel)
-                this.cancelSeen = true;
+                this.markCancelled();
         }
         catch {
             // A transient failure is not a cancellation; keep working.
@@ -198,6 +216,7 @@ class Runner {
     images;
     workDir;
     network;
+    limits;
     idleSeconds;
     wake;
     // When this runner last had something to do. A wake request counts, so that
@@ -223,6 +242,7 @@ class Runner {
         this.images = { ...exports.DEFAULT_IMAGES, ...(config.images ?? {}) };
         this.workDir = config.workDir ?? (0, job_1.defaultWorkDir)();
         this.network = config.network;
+        this.limits = config.limits;
         this.idleSeconds = config.idleSeconds && config.idleSeconds > 0 ? config.idleSeconds : null;
         this.wake = config.wakePort && config.wakeSecret ? { port: config.wakePort, secret: config.wakeSecret } : null;
         const actionCache = config.cacheDir ? path.join(config.cacheDir, 'actions') : (0, actions_1.defaultActionCacheDir)();
@@ -294,6 +314,17 @@ class Runner {
             throw new Error(`none of the requested labels (${this.labels.join(', ')}) are registered for this runner (${identity.labels.join(', ')})`);
         }
         this.name = identity.name;
+        // Whatever a previous life of this runner left running. A container is
+        // removed when its job ends, so any found here belong to a job that was
+        // under way when the runner last died, and nothing else will ever remove them.
+        try {
+            const stale = await (0, docker_1.removeStaleJobContainers)();
+            if (stale > 0)
+                console.log(`Removed ${stale} job container(s) left behind by an earlier run.`);
+        }
+        catch (e) {
+            console.log(`Could not look for leftover job containers: ${e instanceof Error ? e.message : String(e)}`);
+        }
         // Listening before the banner, because a runner that cannot be woken
         // should say so at the top rather than after claiming to be ready. A
         // failure to bind throws out of here: a runner meant to stop when idle
@@ -313,6 +344,12 @@ class Runner {
         console.log(`  workdir: ${this.workDir}`);
         console.log(`  labels:  ${labels.join(', ')}`);
         console.log(`  serving: ${identity.allow.join(', ')}`);
+        const limits = [
+            `pids ${this.limits?.pids ?? docker_1.DEFAULT_PIDS_LIMIT}`,
+            this.limits?.memory ? `memory ${this.limits.memory}` : null,
+            this.limits?.cpus ? `cpus ${this.limits.cpus}` : null,
+        ].filter((l) => l !== null);
+        console.log(`  limits:  ${limits.join(', ')} per job`);
         if (this.idleSeconds !== null)
             console.log(`  idle:    stopping after ${this.idleSeconds}s with no job`);
         if (listener)
@@ -398,6 +435,7 @@ async function executeSpec(spec, deps) {
             log: (i, line) => client.log(i, line),
             progress: (steps) => void client.progress(steps),
             cancelled: () => client.cancelled(),
+            onCancel: (listener) => client.onCancel(listener),
         });
     }
     catch (e) {

package/dist/runner/docker.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DockerError = void 0;
+exports.JOB_CONTAINER_LABEL = exports.DEFAULT_PIDS_LIMIT = exports.DockerError = void 0;
 exports.setContainerEngine = setContainerEngine;
 exports.containerEngine = containerEngine;
 exports.dockerAvailable = dockerAvailable;
@@ -8,10 +8,9 @@ exports.probeEngine = probeEngine;
 exports.imagePresent = imagePresent;
 exports.pullImage = pullImage;
 exports.startContainer = startContainer;
+exports.removeStaleJobContainers = removeStaleJobContainers;
 exports.removeContainer = removeContainer;
 exports.execInContainer = execInContainer;
-exports.writeFileInContainer = writeFileInContainer;
-exports.readFileInContainer = readFileInContainer;
 const child_process_1 = require("child_process");
 // The Docker surface the runner needs, as thin wrappers over the CLI rather
 // than the Engine API socket. Shelling out keeps the runner dependency-free
@@ -94,12 +93,24 @@ function pullImage(image, onLine) {
         child.on('close', (code) => code === 0 ? resolve() : reject(new DockerError(`docker pull ${image} exited ${code}`)));
     });
 }
+/**
+ * Processes without a bound are the cheapest way to take a machine down and
+ * the one no job legitimately needs, so the ceiling is on unless the operator
+ * says otherwise; a build that wants more than this many at once is rare
+ * enough to be asked for by name. Memory and CPU have no default: what a job
+ * may reasonably want depends on the machine, and a cap that is wrong for it
+ * fails honest builds rather than protecting anything.
+ */
+exports.DEFAULT_PIDS_LIMIT = 4096;
+/** The label every job container carries, so a runner can find the ones a crash left behind. */
+exports.JOB_CONTAINER_LABEL = 'mochi.job';
 // One container per job, kept alive by a sleep so each step can exec into
 // it. This is how a container job behaves on GitHub: steps share a
 // filesystem and a process namespace, and anything a step installs is there
 // for the next one.
 async function startContainer(opts) {
     const args = ['run', '--detach', '--name', opts.name, '--workdir', opts.workdir, '--entrypoint', ''];
+    args.push('--label', `${exports.JOB_CONTAINER_LABEL}=1`);
     for (const b of opts.binds) {
         args.push('--volume', `${b.host}:${b.container}${b.readonly ? ':ro' : ''}`);
     }
@@ -108,10 +119,36 @@ async function startContainer(opts) {
     }
     if (opts.network)
         args.push('--network', opts.network);
+    const limits = opts.limits ?? {};
+    const pids = limits.pids ?? exports.DEFAULT_PIDS_LIMIT;
+    if (pids > 0)
+        args.push('--pids-limit', String(pids));
+    if (limits.memory)
+        args.push('--memory', limits.memory);
+    if (limits.cpus)
+        args.push('--cpus', limits.cpus);
     args.push(opts.image, 'sh', '-c', 'while true; do sleep 3600; done');
     const { stdout } = await run(args);
     return stdout.trim();
 }
+/**
+ * Remove every job container on this daemon, and say how many there were.
+ *
+ * A job's container is removed when the job finishes or times out, and by
+ * nothing else: a runner killed mid-job, or a machine that lost power, leaves
+ * a container running its idle sleep for good, and on a machine whose docker
+ * directory is a persistent volume, they accumulate across restarts. This
+ * runs when a runner starts, when by construction no job of its own is in
+ * flight. A daemon shared with a second runner would lose that runner's
+ * running job here, which is one more reason not to share one.
+ */
+async function removeStaleJobContainers() {
+    const { stdout } = await run(['ps', '--all', '--quiet', '--filter', `label=${exports.JOB_CONTAINER_LABEL}`]);
+    const ids = stdout.split('\n').map((l) => l.trim()).filter((l) => l !== '');
+    for (const id of ids)
+        await removeContainer(id);
+    return ids.length;
+}
 async function removeContainer(id) {
     try {
         await run(['rm', '--force', '--volumes', id]);
@@ -170,28 +207,3 @@ function execInContainer(id, argv, onLine, opts = {}) {
     });
     return { child, done };
 }
-// Copy a file into a container. `docker cp` from stdin needs a tar stream, so
-// the simpler route for the small files the runner writes (step scripts, the
-// environment files) is to pipe the bytes through a shell redirect.
-function writeFileInContainer(id, containerPath, content) {
-    return new Promise((resolve, reject) => {
-        const child = (0, child_process_1.spawn)(engineBin, ['exec', '--interactive', id, 'sh', '-c', `cat > "$0"`, containerPath], { stdio: ['pipe', 'ignore', 'pipe'] });
-        let stderr = '';
-        child.stderr?.on('data', (b) => {
-            stderr += b.toString('utf8');
-        });
-        child.on('error', (e) => reject(new DockerError(e.message)));
-        child.on('close', (code) => code === 0 ? resolve() : reject(new DockerError(`writing ${containerPath} failed: ${stderr.trim()}`)));
-        child.stdin?.write(content);
-        child.stdin?.end();
-    });
-}
-async function readFileInContainer(id, containerPath) {
-    try {
-        const { stdout } = await run(['exec', id, 'sh', '-c', `cat "$0" 2>/dev/null || true`, containerPath]);
-        return stdout;
-    }
-    catch {
-        return '';
-    }
-}

package/dist/runner/download.js

@@ -0,0 +1,77 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DownloadTooLarge = void 0;
+exports.downloadTo = downloadTo;
+const fs = __importStar(require("fs"));
+const stream_1 = require("stream");
+const promises_1 = require("stream/promises");
+// A response body written to a file, with a ceiling.
+//
+// The two callers fetch archives -- an action's source from a forge, a node
+// build from nodejs.org -- and used to read each whole into one Buffer before
+// writing it out. The size of that Buffer was whatever the far end chose to
+// serve, and a `uses:` line names the far end, so a workflow could point the
+// runner at a tarball the size of its memory. Streaming through a counter
+// bounds the transfer at the cap and costs one chunk of memory at a time.
+class DownloadTooLarge extends Error {
+}
+exports.DownloadTooLarge = DownloadTooLarge;
+async function downloadTo(res, file, maxBytes) {
+    if (!res.body)
+        throw new Error('the response had no body');
+    // A declared length over the cap is refused before a byte is read; a body
+    // that declares less than it sends, or nothing, is caught as it arrives.
+    const declared = Number(res.headers.get('content-length') ?? '0');
+    if (declared > maxBytes)
+        throw new DownloadTooLarge(`${declared} bytes, over the limit of ${maxBytes}`);
+    let seen = 0;
+    const cap = new stream_1.Transform({
+        transform(chunk, _encoding, callback) {
+            seen += chunk.length;
+            if (seen > maxBytes)
+                callback(new DownloadTooLarge(`more than ${maxBytes} bytes`));
+            else
+                callback(null, chunk);
+        },
+    });
+    try {
+        await (0, promises_1.pipeline)(stream_1.Readable.fromWeb(res.body), cap, fs.createWriteStream(file));
+    }
+    catch (e) {
+        fs.rmSync(file, { force: true });
+        throw e;
+    }
+}

package/dist/runner/externals.js

@@ -40,6 +40,7 @@ const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
 const docker_1 = require("./docker");
+const download_1 = require("./download");
 // JavaScript actions need a node interpreter inside the job's container.
 // GitHub's runner ships its own and mounts it in, which is why an image
 // without node still runs `actions/checkout`. We do the same, but only when
@@ -49,6 +50,10 @@ const docker_1 = require("./docker");
 // The downloaded builds are the official linux-x64/arm64 ones, which are
 // linked against glibc. An Alpine or other musl image therefore cannot use
 // them, and gets a clear message rather than a confusing loader error.
+// An official node build is around fifty megabytes compressed; a download
+// several times that is not one, whatever nodejs.org or the proxy in front
+// of it is serving.
+const MAX_NODE_BYTES = 256 * 1024 * 1024;
 class ExternalsError extends Error {
 }
 exports.ExternalsError = ExternalsError;
@@ -160,7 +165,13 @@ class Externals {
         fs.rmSync(tmp, { recursive: true, force: true });
         fs.mkdirSync(tmp);
         const tarball = path.join(tmp, 'node.tar.gz');
-        fs.writeFileSync(tarball, Buffer.from(await res.arrayBuffer()));
+        try {
+            await (0, download_1.downloadTo)(res, tarball, MAX_NODE_BYTES);
+        }
+        catch (e) {
+            fs.rmSync(tmp, { recursive: true, force: true });
+            throw new ExternalsError(`could not download node ${version}: ${e instanceof Error ? e.message : e}`);
+        }
         const extracted = path.join(tmp, 'out');
         fs.mkdirSync(extracted);
         await run('tar', ['-xzf', tarball, '-C', extracted, '--strip-components=1']);

package/dist/runner/job.js

@@ -89,13 +89,23 @@ async function runJob(spec, ctx, hooks) {
     let containerId = null;
     // `timeout-minutes` is enforced here rather than by the server: the lease
     // sweep only notices a runner that has stopped heartbeating, and a job stuck
-    // inside a step heartbeats perfectly well. Cancellation is cooperative and
-    // checked between steps, so a deadline that only set the flag would never
-    // interrupt the step that is actually hanging; removing the container is what
-    // makes the in-flight exec fail and the loop unwind.
+    // inside a step heartbeats perfectly well. The cancelled flag is checked
+    // between steps, so neither a deadline nor a cancel that only set it would
+    // ever interrupt the step that is actually hanging; removing the container is
+    // what makes the in-flight exec fail and the loop unwind. Both do that, and
+    // `stopped` records that they did, so nothing tries to use the container
+    // afterwards.
     let timedOut = false;
+    let stopped = false;
     let deadline = null;
     const cancelled = () => hooks.cancelled() || timedOut;
+    const stopContainer = (why) => {
+        if (stopped || !containerId)
+            return;
+        stopped = true;
+        log(why);
+        void (0, docker_1.removeContainer)(containerId);
+    };
     const finish = async (conclusion) => {
         currentStep = RUNNER_STEP;
         if (deadline)
@@ -158,10 +168,10 @@ async function runJob(spec, ctx, hooks) {
             env,
             workdir: context_1.WORKSPACE,
             network: ctx.network,
+            limits: ctx.limits,
         });
         await (0, docker_1.execInContainer)(containerId, ['mkdir', '-p', context_1.RUNNER_TOOL_CACHE], () => { }).done;
         log(`Container started from ${image}`);
-        const id = containerId;
         // A runner and its server are separate long-running processes and need not
         // be the same version, so a spec that carries no usable timeout falls back
         // to GitHub's six hours rather than to NaN, which setTimeout would treat as
@@ -174,9 +184,13 @@ async function runJob(spec, ctx, hooks) {
         const delay = Math.min(limit * 60_000, 2 ** 31 - 1);
         deadline = setTimeout(() => {
             timedOut = true;
-            log(`Job exceeded its timeout of ${limit} minute${limit === 1 ? '' : 's'}`);
-            void (0, docker_1.removeContainer)(id);
+            stopContainer(`Job exceeded its timeout of ${limit} minute${limit === 1 ? '' : 's'}`);
         }, delay);
+        // A cancel that arrives while a step is running is served the same way.
+        // One that arrived earlier is caught by the step loop before it runs
+        // anything, and the listener fires at once, so the container goes either
+        // way.
+        hooks.onCancel?.(() => stopContainer('The run was cancelled; stopping the job container'));
         const exec = {
             containerId,
             spec,
@@ -231,10 +245,11 @@ async function runJob(spec, ctx, hooks) {
         });
         failed = failed || result.failed;
         // Post steps run even when the job failed or was cancelled, which is what
-        // makes them useful for cleanup. A timeout is the exception: the container
-        // they would run in is the thing that was torn down to stop the job, so
-        // attempting them would only add a failure for each one to the log.
-        if (rt.postHooks.length && !timedOut) {
+        // makes them useful for cleanup. A job stopped mid-step, by its timeout or
+        // by a cancel, is the exception: the container they would run in is the
+        // thing that was torn down to stop the job, so attempting them would only
+        // add a failure for each one to the log.
+        if (rt.postHooks.length && !stopped) {
             currentStep = RUNNER_STEP;
             await (0, steps_1.runPostHooks)(exec, failed);
         }

package/dist/runner/steps.js

@@ -77,9 +77,11 @@ async function runSteps(steps, scope, ctx, hooks = {}) {
             }),
             functions: (0, context_1.statusFunctions)(failed || ctx.status().failed, ctx.status().cancelled),
         });
+        // A step that failed under continue-on-error never set `failed`, so it
+        // does not close the gate for the steps after it.
         let shouldRun;
         try {
-            shouldRun = step.if === undefined ? !failed : (0, expr_1.evalCondition)(step.if, evalEnv());
+            shouldRun = (0, expr_1.evalGatedCondition)(step.if, !failed, evalEnv());
         }
         catch (e) {
             ctx.log(`Error evaluating if: ${e instanceof Error ? e.message : String(e)}`);
@@ -213,9 +215,42 @@ async function makeFileCommands(ctx, tag) {
         state: `${context_1.FILE_COMMAND_DIR}/state-${tag}`,
     };
     for (const f of Object.values(files))
-        await (0, docker_1.writeFileInContainer)(ctx.containerId, f, '');
+        writeJobFile(ctx, f, '');
     return files;
 }
+// The file-command directory is a bind mount of a directory the runner owns,
+// so the files a step appends to are created and read back on the host. This
+// used to go through `docker exec --interactive ... cat`, one exec per file,
+// which has a race: when stdin is closed before the exec has attached, the
+// EOF is lost and cat waits forever, and so did the job, until its timeout.
+// An empty file is the widest case of that window, since there is no data
+// write to sequence against. The files are world-writable because the step
+// appends to them as whatever user the image runs, which need not be the
+// runner's.
+function writeJobFile(ctx, containerPath, content) {
+    const host = ctx.hostPath(containerPath);
+    if (host === null)
+        throw new Error(`${containerPath} is not on a mount the runner can write`);
+    fs.writeFileSync(host, content);
+    try {
+        fs.chmodSync(host, 0o666);
+    }
+    catch {
+        // the runner and the container being the same user, which is the common
+        // case, needs no wider mode
+    }
+}
+function readJobFile(ctx, containerPath) {
+    const host = ctx.hostPath(containerPath);
+    if (host === null)
+        return '';
+    try {
+        return fs.readFileSync(host, 'utf8');
+    }
+    catch {
+        return '';
+    }
+}
 function fileCommandEnv(files) {
     return {
         GITHUB_OUTPUT: files.output,
@@ -228,20 +263,20 @@ function fileCommandEnv(files) {
 // Read back everything a step wrote through the file commands, and fold it
 // into the scope and the job runtime.
 async function collectFileCommands(ctx, scope, files, stepId) {
-    const outputText = await (0, docker_1.readFileInContainer)(ctx.containerId, files.output);
+    const outputText = readJobFile(ctx, files.output);
     const parsed = (0, context_1.parseKeyValueFile)(outputText);
     if (stepId && Object.keys(parsed).length) {
         scope.outputs[stepId] = { ...(scope.outputs[stepId] ?? {}), ...parsed };
     }
-    const envText = await (0, docker_1.readFileInContainer)(ctx.containerId, files.envFile);
+    const envText = readJobFile(ctx, files.envFile);
     Object.assign(ctx.rt.env, (0, context_1.parseKeyValueFile)(envText));
-    const pathText = await (0, docker_1.readFileInContainer)(ctx.containerId, files.pathFile);
+    const pathText = readJobFile(ctx, files.pathFile);
     for (const line of pathText.split('\n')) {
         const p = line.trim();
         if (p !== '' && !ctx.rt.extraPath.includes(p))
             ctx.rt.extraPath.unshift(p);
     }
-    const summaryText = await (0, docker_1.readFileInContainer)(ctx.containerId, files.summary);
+    const summaryText = readJobFile(ctx, files.summary);
     if (summaryText.trim() !== '')
         ctx.rt.summaries.push(summaryText);
 }
@@ -332,7 +367,7 @@ async function runShellStep(step, scope, ctx, stepEnv, evalEnv, index) {
     const tag = envFileTag(index);
     const files = await makeFileCommands(ctx, tag);
     const scriptPath = `${context_1.FILE_COMMAND_DIR}/step-${tag}`;
-    await (0, docker_1.writeFileInContainer)(ctx.containerId, scriptPath, script.endsWith('\n') ? script : script + '\n');
+    writeJobFile(ctx, scriptPath, script.endsWith('\n') ? script : script + '\n');
     const shell = step.shell ?? (scope.depth === 0 ? ctx.spec.defaults?.shell : undefined);
     const env = {
         ...stepEnv,