@secureci/cli 1.0.1 → 1.2.0

package/dist/runner/job-container.js

@@ -38,6 +38,8 @@ exports.installContainerCleanup = installContainerCleanup;
 exports.noRuntimeMessage = noRuntimeMessage;
 exports.startJobContainer = startJobContainer;
 exports.runnerImage = runnerImage;
+exports.clearIsolatedWorkspace = clearIsolatedWorkspace;
+exports.removeInContainer = removeInContainer;
 exports.stopJobContainer = stopJobContainer;
 exports.startJobServices = startJobServices;
 exports.stopJobServices = stopJobServices;
@@ -50,6 +52,13 @@ const expression_1 = require("./expression");
 const container_runtime_1 = require("./container-runtime");
 const shell_1 = require("./shell");
 const container_policy_1 = require("./container-policy");
+/**
+ * `jobs.<id>.container.options` the CLI sets on the job container itself:
+ * it idles with its own entrypoint and steps run in the mounted workspace.
+ */
+const JOB_CONTAINER_RESERVED = new Set(['--entrypoint', '--workdir', '-w']);
+/** Also reserved when `--container-memory-limit` caps the job container. */
+const MEMORY_OPTIONS = ['--memory', '-m', '--memory-swap'];
 /**
  * Image for jobs whose `runs-on` has no image mapping. Runner-like (Node.js,
  * git, curl and common tools), as used by nektos/act.
@@ -123,14 +132,6 @@ async function startJobContainer(job, ctx, options, runtime, runDir) {
         logger_1.logger.error(noRuntimeMessage(preference));
         return false;
     }
-    if (spec &&
-        'image' in spec &&
-        (('options' in spec && spec.options) ||
-            ('volumes' in spec && spec.volumes) ||
-            ('ports' in spec && spec.ports) ||
-            ('credentials' in spec && spec.credentials))) {
-        logger_1.logger.warn('jobs.<id>.container: only image and env are supported; ignoring options/volumes/ports/credentials.');
-    }
     // `container: ${{ inputs.image || null }}` that comes out empty means no
     // container image, as on GitHub: use the image for runs-on.
     const specImage = spec?.image
@@ -140,19 +141,30 @@ async function startJobContainer(job, ctx, options, runtime, runDir) {
         options.containerImage ||
         runnerImage(job?.['runs-on'], ctx, options);
     const containerEnv = (0, expression_1.substituteRecord)(specImage && spec && 'env' in spec ? spec.env : undefined, ctx);
+    const extras = specImage && spec
+        ? jobContainerExtras(spec, ctx, options, runtime.workspace)
+        : {};
+    if (!extras)
+        return false;
+    if (specImage &&
+        spec &&
+        !(await pullWithCredentials(`jobs.${ctx.github.job ?? '<id>'}.container`, image, spec.credentials, ctx, options, containerRuntime))) {
+        return false;
+    }
     const exchangeDir = fs.mkdtempSync(path.join(runDir, 'container-'));
     fs.chmodSync(exchangeDir, 0o777);
     fs.copyFileSync(runtime.eventPath, path.join(exchangeDir, 'event.json'));
     logger_1.logger.info(`Starting job container: ${image} (${containerRuntime.name})`);
     const started = await containerRuntime.start({
         image,
-        workingDir: process.cwd(),
+        workingDir: runtime.workspace,
         readOnlyPaths: protectedWorkspacePaths(options),
         exchangeDir,
         env: containerEnv,
         memoryLimit: options.containerMemoryLimit,
         networkDisabled: options.networkDisabled === true,
-        network: runtime.services?.network
+        network: runtime.services?.network,
+        ...extras
     });
     if (!started.success || !started.id) {
         logger_1.logger.error(`Failed to start job container (${image}): ${started.error}`);
@@ -173,6 +185,74 @@ async function startJobContainer(job, ctx, options, runtime, runDir) {
     activeContainers.add(runtime.container);
     return true;
 }
+/**
+ * The `options`, `ports` and `volumes` of `jobs.<id>.container`, checked
+ * with the same policy as service containers: options and host mounts that
+ * would reach this machine are refused (undefined: the job fails), options
+ * the CLI sets itself are dropped with a warning.
+ */
+function jobContainerExtras(spec, ctx, options, workspace) {
+    const label = `jobs.${ctx.github.job ?? '<id>'}.container`;
+    const args = spec.options
+        ? (0, shell_1.splitArgs)((0, expression_1.substituteExpressions)(spec.options, ctx))
+        : [];
+    const unsafe = (0, container_policy_1.unsafeServiceOptions)(args);
+    if (unsafe.length > 0) {
+        logger_1.logger.error(`${label}: refusing unsafe options ${[...new Set(unsafe)].join(', ')}: they would give the container access to this machine.`);
+        return undefined;
+    }
+    const reserved = new Set([
+        ...JOB_CONTAINER_RESERVED,
+        ...(options.containerMemoryLimit ? MEMORY_OPTIONS : [])
+    ]);
+    const { kept, dropped } = (0, container_policy_1.splitReservedOptions)(args, reserved);
+    if (dropped.length > 0) {
+        logger_1.logger.warn(`${label}: ignoring options the CLI sets itself: ${[...new Set(dropped)].join(', ')}.`);
+    }
+    const volumes = (spec.volumes ?? []).map((v) => (0, expression_1.substituteExpressions)(v, ctx));
+    for (const volume of volumes) {
+        const reason = (0, container_policy_1.unsafeServiceVolumeReason)(volume, workspace);
+        if (reason) {
+            logger_1.logger.error(`${label}: ${reason}.`);
+            return undefined;
+        }
+    }
+    return {
+        extraArgs: kept,
+        ports: (spec.ports ?? []).map((port) => (0, expression_1.substituteExpressions)(String(port), ctx)),
+        volumes
+    };
+}
+/**
+ * `credentials:` of a job or service container. Without --registry-login
+ * they're only warned about. With it, the image is pulled with them first
+ * (a private, temporary login: see ContainerRuntime.pull), and the
+ * password is masked in the logs. False (the job fails) when they're
+ * incomplete or the pull fails.
+ */
+async function pullWithCredentials(label, image, credentials, ctx, options, runtime) {
+    if (!credentials)
+        return true;
+    if (!options.registryLogin) {
+        logger_1.logger.warn(`${label}: credentials are ignored; pass --registry-login to log in for this run, or log in with \`docker login\`.`);
+        return true;
+    }
+    const username = (0, expression_1.substituteExpressions)(String(credentials.username ?? ''), ctx).trim();
+    const password = (0, expression_1.substituteExpressions)(String(credentials.password ?? ''), ctx);
+    logger_1.logger.registerSecrets({ password });
+    if (!username || !password) {
+        logger_1.logger.error(`${label}: credentials need a username and a password.`);
+        return false;
+    }
+    const registry = (0, container_runtime_1.registryOf)(image);
+    logger_1.logger.info(`${label}: pulling ${image} with credentials for ${registry}`);
+    const pulled = await runtime.pull(image, { registry, username, password });
+    if (!pulled.success) {
+        logger_1.logger.error(`${label}: couldn't pull ${image}: ${pulled.error}`);
+        return false;
+    }
+    return true;
+}
 /**
  * The image for a job's `runs-on`: the first label with a mapping in
  * `runner-images` (or the built-in one), else the default image, with a
@@ -204,6 +284,59 @@ function runnerImage(runsOn, ctx, options) {
     }
     return fallback;
 }
+/**
+ * Empty a job's --isolate-workspace copy from inside its container, as
+ * root, before the container is stopped: files a step created as root
+ * (rootful engines) can't always be removed by the host user afterwards.
+ * Removing them from inside works with rootful and rootless engines alike
+ * (a chown back would map to a subordinate uid when rootless). `.git` is
+ * mounted read-only, so it's left for the host to remove. Only ever call
+ * this for an isolated copy, never for the user's working copy.
+ */
+async function clearIsolatedWorkspace(runtime) {
+    const container = runtime.container;
+    if (!container)
+        return;
+    const result = await container.runtime.exec(container.id, {
+        args: [
+            'find',
+            container_runtime_1.CONTAINER_WORKDIR,
+            '-mindepth',
+            '1',
+            '-maxdepth',
+            '1',
+            '!',
+            '-name',
+            '.git',
+            '-exec',
+            'rm',
+            '-rf',
+            '--',
+            '{}',
+            '+'
+        ],
+        user: '0',
+        workingDir: '/'
+    });
+    if (!result.success) {
+        logger_1.logger.debug(`Couldn't empty the workspace copy in the job container: ${result.stderr.trim()}`);
+    }
+}
+/**
+ * Remove `containerPath` from inside the job's container, as root, so a
+ * copy holding files a step created as root can then be removed on the
+ * host. No-op without a container.
+ */
+async function removeInContainer(runtime, containerPath) {
+    const container = runtime.container;
+    if (!container)
+        return;
+    await container.runtime.exec(container.id, {
+        args: ['rm', '-rf', '--', containerPath],
+        user: '0',
+        workingDir: '/'
+    });
+}
 async function stopJobContainer(runtime) {
     const container = runtime.container;
     if (container) {
@@ -267,16 +400,16 @@ async function startJobServices(job, ctx, options, runtime) {
             return false;
         }
         for (const volume of volumes) {
-            const reason = (0, container_policy_1.unsafeServiceVolumeReason)(volume, process.cwd());
+            const reason = (0, container_policy_1.unsafeServiceVolumeReason)(volume, runtime.workspace);
             if (reason) {
                 logger_1.logger.error(`services.${serviceId}: ${reason}.`);
                 return false;
             }
         }
-        if (service.credentials) {
-            logger_1.logger.warn(`services.${serviceId}: credentials are ignored; log in with \`docker login\` instead.`);
-        }
         const image = (0, expression_1.substituteExpressions)(service.image, ctx);
+        if (!(await pullWithCredentials(`services.${serviceId}`, image, service.credentials, ctx, options, containerRuntime))) {
+            return false;
+        }
         logger_1.logger.info(`Starting service ${serviceId}: ${image}`);
         const started = await containerRuntime.start({
             image,

package/dist/runner/job-runtime.js

@@ -107,7 +107,7 @@ function runnerContext(runtime) {
     return {
         name: 'secureci',
         os: runtime.runnerEnv.RUNNER_OS,
-        arch: process.arch === 'arm64' ? 'ARM64' : 'X64',
+        arch: runtime.runnerEnv.RUNNER_ARCH,
         temp: runtime.runnerEnv.RUNNER_TEMP,
         tool_cache: runtime.runnerEnv.RUNNER_TOOL_CACHE
     };
@@ -126,12 +126,13 @@ function hostHasBash() {
     }
     return hostBash;
 }
-function createJobRuntime(github, runDir, defaults = {}) {
+function createJobRuntime(github, runDir, defaults = {}, workspace = process.cwd()) {
     const homeDir = process.env.HOME || process.env.USERPROFILE || os.tmpdir();
     const eventPath = path.join(runDir, 'event.json');
     return {
+        workspace,
         runnerEnv: (0, runner_env_1.buildRunnerEnv)(github, {
-            workspace: process.cwd(),
+            workspace,
             eventPath,
             // Per job, so concurrent jobs don't share $RUNNER_TEMP.
             tempDir: fs.mkdtempSync(path.join(runDir, 'tmp-')),

package/dist/runner/node-action.js

@@ -144,7 +144,7 @@ async function executeNodeAction(action, actionPath, step, actionInputs, callerC
         }
         try {
             const { stdout, stderr } = await execFileAsync(node.path, [path.join(actionDir, script)], {
-                cwd: process.cwd(),
+                cwd: runtime.workspace,
                 maxBuffer: 64 * 1024 * 1024,
                 timeout: limit.ms,
                 killSignal: 'SIGKILL',

package/dist/runner/runner-env.js

@@ -38,6 +38,7 @@ exports.createStepFiles = createStepFiles;
 exports.readStepFiles = readStepFiles;
 exports.cleanupStepFiles = cleanupStepFiles;
 exports.processWorkflowCommands = processWorkflowCommands;
+exports.runnerArch = runnerArch;
 exports.buildRunnerEnv = buildRunnerEnv;
 exports.prependPath = prependPath;
 exports.hostBaseEnv = hostBaseEnv;
@@ -248,6 +249,24 @@ function runnerOs() {
             return 'Linux';
     }
 }
+/**
+ * `RUNNER_ARCH` / `runner.arch` for a Node.js architecture, using GitHub's
+ * values (X86, X64, ARM, ARM64). Jobs run natively (a job container has the
+ * host's architecture), so the host's is the runner's. Anything else reports
+ * X64, as before this was mapped.
+ */
+function runnerArch(arch = process.arch) {
+    switch (arch) {
+        case 'arm64':
+            return 'ARM64';
+        case 'arm':
+            return 'ARM';
+        case 'ia32':
+            return 'X86';
+        default:
+            return 'X64';
+    }
+}
 /**
  * The default environment variables GitHub's runner exports to every step,
  * derived from the (local, best-effort) `github` context.
@@ -274,6 +293,7 @@ function buildRunnerEnv(github, options) {
         GITHUB_WORKSPACE: options.workspace,
         GITHUB_EVENT_PATH: options.eventPath,
         RUNNER_OS: runnerOs(),
+        RUNNER_ARCH: runnerArch(),
         RUNNER_TEMP: options.tempDir,
         RUNNER_TOOL_CACHE: options.toolCache
     };

package/dist/runner/scheduler.js

@@ -33,9 +33,10 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Slots = void 0;
+exports.ConcurrencyGroups = exports.Slots = void 0;
 exports.defaultParallelism = defaultParallelism;
 exports.parseParallel = parseParallel;
+exports.parseConcurrencyMode = parseConcurrencyMode;
 exports.runMatrix = runMatrix;
 exports.runJobGraph = runJobGraph;
 const os = __importStar(require("os"));
@@ -84,6 +85,65 @@ function parseParallel(raw) {
     }
     return n;
 }
+/** Parse `--concurrency` / config `concurrency`. */
+function parseConcurrencyMode(raw) {
+    const mode = raw.trim().toLowerCase();
+    if (mode === 'serialize' || mode === 'github')
+        return mode;
+    throw new Error(`Invalid value "${raw}" for concurrency. Must be serialize or github.`);
+}
+/** `concurrency:` groups of one run, keyed case-insensitively. */
+class ConcurrencyGroups {
+    constructor(mode, note = () => undefined) {
+        this.mode = mode;
+        this.note = note;
+        this.holders = new Map();
+        this.waiting = new Map();
+    }
+    /**
+     * Wait to run in `group`. `abort` cancels this job once it's running
+     * (github mode, for a newcomer with cancel-in-progress).
+     */
+    enter(group, cancelInProgress, abort, label) {
+        const key = group.toLowerCase();
+        return new Promise((admit) => {
+            const entry = { label, abort, admit };
+            const holder = this.holders.get(key);
+            if (!holder) {
+                this.admit(key, entry);
+                return;
+            }
+            const queue = this.waiting.get(key) ?? [];
+            this.waiting.set(key, queue);
+            if (this.mode === 'github') {
+                if (cancelInProgress) {
+                    this.note(`Cancelling ${holder.label}: ${label} joined concurrency group "${group}" with cancel-in-progress`);
+                    holder.abort();
+                }
+                for (const older of queue.splice(0)) {
+                    this.note(`Cancelling ${older.label}: ${label} is now pending in concurrency group "${group}"`);
+                    older.admit('cancelled');
+                }
+            }
+            queue.push(entry);
+        });
+    }
+    admit(key, entry) {
+        this.holders.set(key, entry);
+        let left = false;
+        entry.admit(() => {
+            if (left)
+                return;
+            left = true;
+            const next = this.waiting.get(key)?.shift();
+            if (next)
+                this.admit(key, next);
+            else
+                this.holders.delete(key);
+        });
+    }
+}
+exports.ConcurrencyGroups = ConcurrencyGroups;
 /**
  * Run `count` matrix combinations, at most `maxParallel` at once and each
  * holding one of the global `slots` while it runs. With fail-fast, a
@@ -105,6 +165,29 @@ async function runMatrix(count, run, options) {
     };
     options.signal?.addEventListener('abort', onParentAbort);
     const runOne = async (index) => {
+        const controller = new AbortController();
+        let leaveGroup;
+        if (options.gate) {
+            const admission = await options.gate(index, () => controller.abort());
+            if (admission === 'cancelled' || controller.signal.aborted) {
+                if (admission !== 'cancelled')
+                    admission();
+                if (!stopped && options.onGateCancelled) {
+                    outcomes[index] = await options.onGateCancelled(index);
+                    finished++;
+                }
+                return;
+            }
+            leaveGroup = admission;
+        }
+        try {
+            await runAdmitted(index, controller);
+        }
+        finally {
+            leaveGroup?.();
+        }
+    };
+    const runAdmitted = async (index, controller) => {
         const releaseLocal = await local.acquire();
         try {
             if (stopped)
@@ -115,7 +198,6 @@ async function runMatrix(count, run, options) {
             try {
                 if (stopped)
                     return;
-                const controller = new AbortController();
                 controllers.set(index, controller);
                 try {
                     const outcome = await run(index, controller.signal);

package/dist/runner/steps.js

@@ -143,8 +143,8 @@ async function executeStep(step, options, runtime, ctx) {
                     timeout: limit.ms,
                     killSignal: 'SIGKILL',
                     cwd: workingDirectory
-                        ? path.resolve(process.cwd(), workingDirectory)
-                        : process.cwd(),
+                        ? path.resolve(runtime.workspace, workingDirectory)
+                        : runtime.workspace,
                     env: {
                         ...(0, runner_env_1.hostBaseEnv)(),
                         ...runtime.runnerEnv,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@secureci/cli",
-  "version": "1.0.1",
+  "version": "1.2.0",
   "description": "SecureCI: validate, check against pipeline standards, and run GitHub Actions workflows locally",
   "main": "dist/index.js",
   "bin": {