@intentius/chant 0.124.29 → 0.124.31

package/dist/workspace/ci-run.js

@@ -0,0 +1,691 @@
+/**
+ * `chant ci run plan|apply` (#4094, epic #4101): run one pull-request job as
+ * a pipeline Op, the first slice of the chant runner.
+ *
+ * The job is the typed value `prLoopJobs` (../components/pr-pipeline.ts)
+ * returns, the one the github, gitlab and forgejo generators wrap in a
+ * workflow, so the runner and a generated workflow run the same command with
+ * the same values. Nothing reads generated YAML.
+ *
+ * The job runs as an Op on the local executor (../op/local-executor.ts):
+ *
+ * - a `setup` phase with the setup lines the default image needs,
+ * - a `build` phase with the lines given to run before the command
+ *   (`--before-script`, which a generated workflow runs in the same place),
+ *   then the job's build and lint lines with `--pr-build` (#4088),
+ * - and a phase named for the job, `plan` or `apply`, with the job's own
+ *   setup lines and its command in one shell.
+ *
+ * Each line is one shell step with a clean environment: a short list of the
+ * runner's variables (PATH, HOME and the like), the forge's connection and
+ * token variables the commands read, any `--pass-env` names, and the job's
+ * own variables. The forge values a generated job reads from `${{ ... }}`
+ * expressions or GitLab's predefined variables come from flags here:
+ * `BASE_SHA` from `--base`, `PR_NUMBER` from `--pr`, and the commit from
+ * `--commit`. The values a forge job gets from repository secrets come from
+ * the host's secrets file (`--secrets`, ./ci-secrets.ts, #4096), and every
+ * secret value is redacted from the run's events, record, copied report and
+ * output by the executor's `redact` option.
+ *
+ * The steps run in a fresh checkout of the commit: a detached `git worktree`
+ * in a temporary directory outside the repository, removed when the run
+ * ends. A worktree shares the repository's refs, so what the job records on
+ * `chant/lifecycle` (pr-apply's pending gate facts) and the remote it pushes
+ * to are the repository's own, as in a forge job's checkout. The report the
+ * job writes under `.chant/pr` is copied out of the worktree before it goes,
+ * where a forge would keep it as an artifact.
+ *
+ * The run is recorded as Op `pr-plan` or `pr-apply` in the run ledger
+ * (`<env>/runs__<op>.jsonl` on `chant/lifecycle`), and signed when a runner
+ * key is given, so `chant ci tick --source runs` (#4044) judges the commit
+ * from it. Its events are `kind: "pipeline"` and `runtime: "ci"`, and the
+ * record and the events carry `runOrigin: { forge: "chant" }`.
+ *
+ * With a forge client (#4097) the job also sets a commit status at its
+ * commit: `pending` when it starts, `success` or `failure` when it settles.
+ * A status the forge refuses is a warning and never fails the job.
+ */
+import { execFileSync } from "node:child_process";
+import { copyFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { PR_APPLY_RECORD, PR_LOOP_REPORT_DIR, prLoopJobs, prLoopSetup, PR_LOOP_IMAGE } from "../components/pr-pipeline.js";
+import { detectForge, ForgeEnvironmentError, forgeFromEnv, FORGE_KINDS } from "../pr-forge.js";
+import { PR_APPLY_GATE } from "../pr-loop.js";
+import { formatError, formatWarning } from "../cli/format.js";
+import { pushLifecycleReport } from "../lifecycle/git.js";
+import { ACTIVITY_PROFILES } from "../op/activity-profiles.js";
+import { shellCmd } from "../op/activities/shell.js";
+import { OpRunFailure, runOpLocally } from "../op/local-executor.js";
+import { redactText } from "../op/redact.js";
+import { findWorkspaceRoot } from "../project-root.js";
+import { parseReportUrl, reportKey } from "../pr-report-url.js";
+import { ciTagSigner, RUNNER_KEY_ENV } from "./ci-green.js";
+import { ciRedactor, ciSecretsFor, CI_SECRETS_ENV, readCiSecrets } from "./ci-secrets.js";
+import { updateCiRunIndex } from "./ci-run-index.js";
+import { resolveMemberContext } from "./member-pipeline.js";
+import { signOpRunRecord } from "./trust/op-run-statement.js";
+import { gitTop } from "./tree.js";
+import { hostRunnerKey } from "./ci-runner-key.js";
+export const CI_JOB_NAMES = ["plan", "apply"];
+/** The Op each job is recorded as, which `ci.green` phases match. */
+export const CI_JOB_OPS = { plan: "pr-plan", apply: "pr-apply" };
+/** Where a job the chant runner ran says it ran (#4094). */
+export const CHANT_RUNNER_ORIGIN = { forge: "chant" };
+/** The runner's variables every job gets, when the runner has them. */
+export const CI_RUN_BASE_ENV = ["PATH", "HOME", "USER", "LOGNAME", "SHELL", "LANG", "LC_ALL", "LC_CTYPE", "TMPDIR", "TZ"];
+/**
+ * The forge's connection and token variables the pull-request commands read
+ * (`forgeFromEnv` in ../pr-forge.ts), which a forge job gets from the forge
+ * and a job here gets from the runner's environment.
+ */
+export const CI_RUN_FORGE_ENV = [
+    "CHANT_FORGE_TOKEN",
+    "GITHUB_TOKEN",
+    "GH_TOKEN",
+    "GITHUB_REPOSITORY",
+    "GITHUB_SERVER_URL",
+    "GITHUB_API_URL",
+    "GITLAB_TOKEN",
+    "CI_API_V4_URL",
+    "CI_PROJECT_ID",
+    "CI_SERVER_HOST",
+    "CI_SERVER_URL",
+];
+/** How long one step may run: a forge job's own limit, six hours. */
+const STEP_TIMEOUT = "6h";
+export class CiRunError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "CiRunError";
+    }
+}
+/**
+ * The job's environment: the runner's base variables, the forge variables
+ * and `passEnv` names it has, the job's secrets from the host's secrets file
+ * (#4096), `CHANT_ENV` as a generated workflow sets it, the commit as the
+ * forge spells it, and the job's own variables. A job variable the forge
+ * fills from an expression comes from a flag, or from a secret or the
+ * runner's environment for a token such as `GITHUB_TOKEN`; a literal one,
+ * such as `CHANT_RUN_EVENTS_DIR`, is kept as it is. A secret replaces the
+ * runner's variable of the same name, and none replaces a variable the
+ * runner sets for the job. `CHANT_RUNNER_KEY` is never in it.
+ */
+export function ciJobEnv(job, input) {
+    if (input.passEnv?.includes(RUNNER_KEY_ENV))
+        throw new CiRunError(`--pass-env names ${RUNNER_KEY_ENV}, and the runner key is never passed to a job`);
+    const out = {};
+    for (const name of [...CI_RUN_BASE_ENV, ...CI_RUN_FORGE_ENV, ...(input.passEnv ?? [])]) {
+        const value = input.hostEnv[name];
+        if (value !== undefined)
+            out[name] = value;
+    }
+    for (const [name, value] of Object.entries(input.secrets ?? {}))
+        if (name !== RUNNER_KEY_ENV)
+            out[name] = value;
+    out.CHANT_ENV = input.env;
+    out[input.forge === "gitlab" ? "CI_COMMIT_SHA" : "GITHUB_SHA"] = input.values.commit;
+    const flags = {
+        BASE_SHA: { value: input.values.base, flag: "--base <sha>" },
+        PR_NUMBER: { value: input.values.pr !== undefined ? String(input.values.pr) : undefined, flag: "--pr <number>" },
+    };
+    for (const [name, value] of Object.entries(job.env)) {
+        const given = flags[name];
+        if (given) {
+            if (given.value === undefined)
+                throw new CiRunError(`the ${job.jobName} job reads ${name}, which a forge fills from the event; pass ${given.flag}`);
+            out[name] = given.value;
+        }
+        else if (!value.includes("$")) {
+            out[name] = value;
+        }
+        else if (input.secrets?.[name] !== undefined) {
+            out[name] = input.secrets[name];
+        }
+        else if (input.hostEnv[name] !== undefined) {
+            out[name] = input.hostEnv[name];
+        }
+    }
+    delete out[RUNNER_KEY_ENV];
+    return out;
+}
+/** `line` with a `chant` at its start replaced by the command the caller names, when it names one. */
+export function withChant(line, chant) {
+    return chant && line.startsWith("chant ") ? `${chant}${line.slice("chant".length)}` : line;
+}
+/** The job's own setup lines and its command, as one shell script, with `chant` at its start replaced by the command the caller names. */
+export function ciJobCommand(job, chant) {
+    return [...(job.setup ?? []), withChant(job.command, chant)].join("\n");
+}
+/**
+ * The job as an Op: `setup`, `build` and the job's own phase, each line one
+ * shell step in `cwd` with the job's environment and nothing else. A phase
+ * with no lines is left out.
+ */
+export function ciJobOp(input) {
+    const step = (cmd) => ({
+        kind: "activity",
+        fn: "shellCmd",
+        args: { cmd, cwd: input.cwd, env: input.jobEnv, inheritEnv: false },
+        profile: "atMostOnce",
+        timeout: STEP_TIMEOUT,
+    });
+    const phases = [
+        { name: "setup", steps: input.setup.map(step) },
+        { name: "build", steps: input.build.map((line) => step(withChant(line, input.chant))) },
+        { name: input.job.jobName, steps: [step(ciJobCommand(input.job, input.chant))] },
+    ].filter((p) => p.steps.length > 0);
+    const op = CI_JOB_OPS[input.job.jobName];
+    return {
+        name: op,
+        overview: input.job.jobName === "plan" ? "Plan the members this pull request reaches" : "Apply the plan a reviewer approved",
+        labels: { Env: input.env, Pipeline: "pr", Job: input.job.jobName },
+        phases,
+    };
+}
+/** The pull-request report statuses that mean the job did not do its work, which a failed job's `failure.code` takes. */
+const FAILED_REPORT_STATUSES = new Set(["plan-failed", "failed", "refused"]);
+/** The exit code a failed shell step's error names (`command exited <n>`), when it names one. */
+function exitCodeOf(error) {
+    const m = /command exited (\d+)/.exec(error ?? "");
+    return m ? Number(m[1]) : undefined;
+}
+/** The first line of a step's error, for a reason. */
+const firstLine = (text) => text.split("\n").find((l) => l.trim() !== "")?.trim() ?? text;
+/**
+ * Why a failed job failed (#4094), from its step records and, when the
+ * command wrote one, its pull-request report (`pr-plan.json` or
+ * `pr-apply.json` under `reportDir`):
+ *
+ * - a failed setup or build line: `setup-failed` or `build-failed`, the line
+ *   and its exit code as the reason, and the phase;
+ * - a failed command with a report: the report's `refusal`, else its
+ *   `status`, with its `message` as the reason, else the members that failed;
+ * - a failed command with no report: `command-failed` and the step's error.
+ *
+ * Undefined when no step failed, such as a run that stopped on an error of
+ * its own before any step ran.
+ */
+export function ciJobFailure(input) {
+    const failed = input.records.find((r) => r.status === "fail");
+    if (!failed)
+        return undefined;
+    const exitCode = exitCodeOf(failed.error);
+    const exit = exitCode !== undefined ? { exitCode } : {};
+    if (failed.phase === "setup" || failed.phase === "build") {
+        const at = input.records.filter((r) => r.phase === failed.phase).indexOf(failed);
+        const step = input.op.phases.find((p) => p.name === failed.phase)?.steps[at];
+        const line = typeof step?.args?.cmd === "string" ? step.args.cmd : undefined;
+        const reason = line !== undefined
+            ? `${line} ${exitCode !== undefined ? `exited ${exitCode}` : `failed: ${firstLine(failed.error ?? "")}`}`
+            : firstLine(failed.error ?? "the step failed");
+        return { code: `${failed.phase}-failed`, reason, phase: failed.phase, ...exit };
+    }
+    const file = join(input.reportDir, `${CI_JOB_OPS[input.job]}.json`);
+    let report;
+    try {
+        if (existsSync(file))
+            report = JSON.parse(readFileSync(file, "utf-8"));
+    }
+    catch {
+        report = undefined;
+    }
+    const status = typeof report?.status === "string" ? report.status : undefined;
+    if (report && status && (FAILED_REPORT_STATUSES.has(status) || typeof report.refusal === "string")) {
+        const code = typeof report.refusal === "string" ? report.refusal : status;
+        let reason = typeof report.message === "string" && report.message.trim() !== "" ? firstLine(report.message) : undefined;
+        if (reason === undefined) {
+            const members = Array.isArray(report.members) ? report.members : [];
+            const bad = members.filter((m) => m.status === "failed" || m.status === "plan-failed");
+            reason = bad.length > 0
+                ? `${bad.length === 1 ? "member" : "members"} ${bad.map((m) => `${String(m.member)} ${String(m.status)}${typeof m.error === "string" ? ` (${firstLine(m.error)})` : ""}`).join(", ")}`
+                : `the ${CI_JOB_OPS[input.job]} report says ${status}`;
+        }
+        return { code, reason, phase: failed.phase, ...exit };
+    }
+    return { code: "command-failed", reason: firstLine(failed.error ?? "the command failed"), phase: failed.phase, ...exit };
+}
+/** A status description is cut to what GitHub and Forgejo keep. */
+const STATUS_DESCRIPTION = 140;
+const cut = (text, max = STATUS_DESCRIPTION) => (text.length > max ? `${text.slice(0, max - 3)}...` : text);
+/**
+ * A job's status context: `chant/pr-plan` for the plan, and for the apply
+ * `chant/<gate>`, which is `chant/pr-apply` unless `--gate` renamed the gate.
+ * pr-plan's own status is `chant/plan` (../pr-loop.ts), a different context.
+ */
+export function ciStatusContext(job, gate) {
+    return job === "apply" ? `chant/${gate ?? PR_APPLY_GATE}` : `chant/${CI_JOB_OPS.plan}`;
+}
+/**
+ * The link a job's status carries: the `--status-url` template with `{key}`
+ * filled, else the https `--report-url` followed by the key and a slash, else
+ * none (an `s3://` or `file://` URL is not one a browser opens).
+ */
+export function ciStatusUrl(input) {
+    const key = reportKey(input.pr ?? null, input.commit, input.member);
+    if (input.urlTemplate)
+        return input.urlTemplate.split("{key}").join(key);
+    const url = input.reportUrl?.trim();
+    if (url && /^https:\/\//.test(url))
+        return `${url.replace(/\/+$/, "")}/${key}/`;
+    return undefined;
+}
+/** What the settled status says: a short summary, or on failure the record's code and reason. */
+export function ciStatusSettled(op, result, seconds) {
+    if (result.status === "ok")
+        return { state: "success", description: cut(`${op} passed in ${seconds}s`) };
+    const why = result.record.failure;
+    const text = why ? `${why.code}: ${why.reason}` : result.failure ? `${result.failure.phase}: ${firstLine(result.failure.error)}` : `${op} failed`;
+    return { state: "failure", description: cut(text) };
+}
+/**
+ * The status client for `chant ci run` and `chant ci serve` (#4097), or none:
+ * off with `--no-status`, on GitLab (its client has no commit status call
+ * here), and without a forge token. A token with the rest of the forge's
+ * variables missing gives a warning instead.
+ */
+export function ciStatusFromEnv(input) {
+    if (input.noStatus || input.forge === "gitlab")
+        return {};
+    if (input.statusUrl !== undefined && !/^https?:\/\/\S+$/.test(input.statusUrl)) {
+        throw new CiRunError(`--status-url takes an http or https URL template such as https://host/reports/{key}, not ${JSON.stringify(input.statusUrl)}`);
+    }
+    const env = input.hostEnv;
+    const hasToken = Boolean(env.CHANT_FORGE_TOKEN ?? env.GITHUB_TOKEN ?? env.GH_TOKEN);
+    if (!hasToken)
+        return {};
+    try {
+        const client = forgeFromEnv(input.forge, env);
+        if (!client.setCommitStatus)
+            return {};
+        return { status: { client: { setCommitStatus: client.setCommitStatus.bind(client) }, ...(input.statusUrl ? { urlTemplate: input.statusUrl } : {}) } };
+    }
+    catch (err) {
+        if (err instanceof ForgeEnvironmentError)
+            return { warning: `no commit status will be set: ${err.message}` };
+        throw err;
+    }
+}
+const message = (err) => (err instanceof Error ? err.message : String(err));
+function git(cwd, args) {
+    return execFileSync("git", args, { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] }).trim();
+}
+function tryGit(cwd, args) {
+    try {
+        return git(cwd, args);
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The full commit `rev` names, fetched from `origin` when the repository lacks it. */
+function resolveJobCommit(top, rev) {
+    if (rev.startsWith("-"))
+        throw new CiRunError(`--commit takes a commit, not ${JSON.stringify(rev)}`);
+    const local = tryGit(top, ["rev-parse", "--verify", "--quiet", `${rev}^{commit}`]);
+    if (local)
+        return local;
+    if (tryGit(top, ["fetch", "--quiet", "--no-tags", "origin", rev]) !== undefined) {
+        const fetched = tryGit(top, ["rev-parse", "--verify", "--quiet", "FETCH_HEAD^{commit}"]);
+        if (fetched && (fetched === rev || fetched.startsWith(rev)))
+            return fetched;
+    }
+    throw new CiRunError(`${rev} names no commit here or on origin`);
+}
+/** The member a job runs for, and its directory from the repository root, as the generator decides it. */
+export function ciJobPlace(cwd, top) {
+    const found = findWorkspaceRoot(cwd);
+    const member = found ? resolveMemberContext(cwd, found) : undefined;
+    const at = member ? member.memberRoot : cwd;
+    const dir = relative(top, realpathSync(at)).split(sep).join("/");
+    return { ...(member ? { member: member.member.name } : {}), dir: dir === "" ? "." : dir };
+}
+/** A detached worktree of `commit` in a new temporary directory, and how to remove it. */
+function freshCheckout(top, commit) {
+    const holder = mkdtempSync(join(realpathSync(tmpdir()), "chant-ci-run-"));
+    const dir = join(holder, "checkout");
+    try {
+        git(top, ["worktree", "add", "--quiet", "--detach", dir, commit]);
+    }
+    catch (err) {
+        rmSync(holder, { recursive: true, force: true });
+        const stderr = err.stderr;
+        throw new CiRunError(`could not check out ${commit.slice(0, 12)} in a fresh worktree: ${String(stderr ?? err).trim()}`);
+    }
+    return {
+        dir,
+        remove() {
+            tryGit(top, ["worktree", "remove", "--force", dir]);
+            rmSync(holder, { recursive: true, force: true });
+            tryGit(top, ["worktree", "prune"]);
+        },
+    };
+}
+/**
+ * Add the settled job to the run index at its report URL (#4098). The pull
+ * request is the one the job's own report names, since a pr-apply finds it
+ * itself, else `--pr`. A warning, never a failure, when the index can't be
+ * updated.
+ */
+async function recordInIndex(input) {
+    const target = parseReportUrl(input.url);
+    if ("error" in target)
+        return input.warn(`the run index is not updated: ${target.error}`);
+    let pr = input.pr ?? null;
+    try {
+        const file = join(input.reportDir, `${CI_JOB_OPS[input.job]}.json`);
+        if (existsSync(file)) {
+            const said = JSON.parse(readFileSync(file, "utf-8")).pr;
+            if (Number.isInteger(said) && said > 0)
+                pr = said;
+        }
+    }
+    catch {
+        // The report is not readable; the flag stands.
+    }
+    const failure = input.record.failure;
+    const clean = (text) => (input.redact ? input.redact(text) : text);
+    const entry = {
+        job: input.job,
+        pr,
+        commit: input.commit,
+        status: input.record.status === "ok" ? "ok" : "fail",
+        ...(input.record.status !== "ok" ? { failure: { code: clean(failure?.code ?? "command-failed") } } : {}),
+        ...(input.member ? { member: clean(input.member) } : {}),
+        started: input.record.started,
+        ended: input.record.ended,
+        runId: clean(input.record.id),
+        key: clean(reportKey(pr, input.commit, input.member)),
+        runOrigin: CHANT_RUNNER_ORIGIN,
+    };
+    await updateCiRunIndex({ target, entry, ...(input.limit ? { limit: input.limit } : {}), warn: input.warn });
+}
+const ACTIVITIES = new Map([["shellCmd", shellCmd]]);
+/**
+ * Run one pull-request job as a pipeline Op in a fresh checkout of the
+ * commit, record it, and copy its report out. Throws {@link CiRunError}
+ * before anything runs when the request can't be run.
+ */
+export async function ciRun(req) {
+    const hostEnv = req.hostEnv ?? process.env;
+    const cwd = realpathSync(resolve(req.cwd));
+    const top = gitTop(cwd);
+    if (!top)
+        throw new CiRunError("a job runs in a fresh checkout of a commit, and this directory is not in a git repository");
+    const commit = resolveJobCommit(top, req.commit ?? "HEAD");
+    const place = ciJobPlace(cwd, top);
+    const jobs = prLoopJobs(req.forge, req.env, req.loop ?? {}, place.member);
+    const job = jobs.find((j) => j.jobName === req.job);
+    if (!job)
+        throw new CiRunError(`the pull-request pipeline has no ${req.job} job; it has ${jobs.map((j) => j.jobName).join(" and ")}`);
+    const jobEnv = ciJobEnv(job, {
+        forge: req.forge,
+        env: req.env,
+        values: { base: req.base, pr: req.pr, commit },
+        hostEnv,
+        passEnv: req.passEnv,
+        secrets: ciSecretsFor(req.secrets, req.job),
+    });
+    // #4096: the secrets and the job's secret-looking variables, redacted from everything the run writes and returns.
+    const redact = ciRedactor(jobEnv, req.secrets);
+    try {
+        const result = await runCiJob(req, { cwd, top, commit, place, job, jobEnv, ...(redact ? { redact } : {}) });
+        return redact ? { ...result, warnings: result.warnings.map(redact), ...(result.pushWarning ? { pushWarning: redact(result.pushWarning) } : {}) } : result;
+    }
+    catch (err) {
+        if (redact && err instanceof Error)
+            err.message = redact(err.message);
+        throw err;
+    }
+}
+/** {@link ciRun} once the job and its environment are known. */
+async function runCiJob(req, at) {
+    const { cwd, top, commit, place, job, jobEnv, redact } = at;
+    // The record is signed against .chant/trust.json at the commit the job runs, so a key that commit does not list is refused before anything runs.
+    let signer;
+    const early = [];
+    if (req.runnerKey !== undefined) {
+        try {
+            signer = ciTagSigner(top, commit, req.runnerKey, "the runs it signed");
+        }
+        catch (err) {
+            if (!req.runnerKeyFromEnv)
+                throw new CiRunError(`the run would not be signed: ${message(err)}`);
+            early.push(`${RUNNER_KEY_ENV} is set, and the run will not be signed: ${message(err)}`);
+        }
+    }
+    const checkout = freshCheckout(top, commit);
+    try {
+        const jobDir = place.dir === "." ? checkout.dir : join(checkout.dir, place.dir);
+        const op = ciJobOp({
+            job,
+            env: req.env,
+            cwd: jobDir,
+            jobEnv,
+            setup: prLoopSetup(req.image ?? PR_LOOP_IMAGE),
+            build: [...(req.beforeScript ?? []), ...(job.build ?? [])],
+            ...(req.chant ? { chant: req.chant } : {}),
+        });
+        const runId = req.runId ?? `chant-${Date.now()}`;
+        const warnings = [...early];
+        const resume = req.job === "apply" && req.resumeRecord ? { host: resolve(req.resumeRecord), job: join(jobDir, PR_APPLY_RECORD) } : undefined;
+        if (resume && existsSync(resume.host)) {
+            mkdirSync(dirname(resume.job), { recursive: true });
+            copyFileSync(resume.host, resume.job);
+        }
+        const statusContext = ciStatusContext(req.job, req.loop?.gate);
+        const statusUrl = req.status
+            ? ciStatusUrl({ ...(req.status.urlTemplate ? { urlTemplate: req.status.urlTemplate } : {}), ...(req.loop?.reportUrl ? { reportUrl: req.loop.reportUrl } : {}), ...(req.pr !== undefined ? { pr: req.pr } : {}), commit, ...(place.member ? { member: place.member } : {}) })
+            : undefined;
+        const postStatus = async (state, description) => {
+            if (!req.status)
+                return;
+            try {
+                await req.status.client.setCommitStatus(commit, { context: statusContext, state, description: cut(redact ? redact(description) : description), ...(statusUrl ? { targetUrl: statusUrl } : {}) });
+            }
+            catch (err) {
+                warnings.push(`could not set the ${statusContext} status: ${message(err)}`);
+            }
+        };
+        const startedAt = Date.now();
+        await postStatus("pending", `${op.name} running`);
+        let ledgerError;
+        let result;
+        try {
+            result = await runOpLocally(op, ACTIVITIES, ACTIVITY_PROFILES, req.signal, {
+                runId,
+                env: req.env,
+                cwd: jobDir,
+                commit,
+                ledger: { cwd: jobDir },
+                runner: { kind: "pipeline", runtime: "ci", runOrigin: CHANT_RUNNER_ORIGIN },
+                describeFailure: (records) => ciJobFailure({ job: req.job, op, records, reportDir: join(jobDir, PR_LOOP_REPORT_DIR) }),
+                ...(req.runEvents ? { runEvents: { write: req.runEvents } } : {}),
+                ...(req.stepOutput ? { stepOutput: req.stepOutput } : {}),
+                ...(signer ? { signRecord: (record) => signOpRunRecord(record, signer) } : {}),
+                onSignError: (err) => warnings.push(`the run was not signed: ${message(err)}`),
+                ...(redact ? { redact } : {}),
+                onLedgerError: (err) => {
+                    ledgerError = `the job ran, and its record could not be written to chant/lifecycle: ${message(err)}`;
+                },
+            });
+        }
+        catch (err) {
+            if (!(err instanceof OpRunFailure)) {
+                await postStatus("error", `${op.name} stopped: ${firstLine(message(err))}`);
+                throw err;
+            }
+            result = err.result;
+        }
+        const failedStep = result.records.find((r) => r.status === "fail");
+        const settled = ciStatusSettled(op.name, { status: result.status, record: result.record, ...(failedStep ? { failure: { phase: failedStep.phase, error: failedStep.error ?? "the step failed" } } : {}) }, Math.round((Date.now() - startedAt) / 1000));
+        await postStatus(settled.state, settled.description);
+        if (resume && existsSync(resume.job)) {
+            try {
+                mkdirSync(dirname(resume.host), { recursive: true });
+                copyFileSync(resume.job, resume.host);
+            }
+            catch (err) {
+                warnings.push(`the resume record could not be kept at ${resume.host}: ${message(err)}`);
+            }
+        }
+        const written = join(jobDir, PR_LOOP_REPORT_DIR);
+        let report = null;
+        if (existsSync(written)) {
+            report = resolve(cwd, req.output ?? PR_LOOP_REPORT_DIR);
+            cpSync(written, report, { recursive: true });
+            if (redact)
+                redactTree(report, redact);
+        }
+        if (req.loop?.reportUrl) {
+            await recordInIndex({ url: req.loop.reportUrl, job: req.job, commit, member: place.member, pr: req.pr, record: result.record, reportDir: written, limit: req.runIndexLimit, ...(redact ? { redact } : {}), warn: (w) => warnings.push(w) });
+        }
+        if (ledgerError)
+            throw new CiRunError(ledgerError);
+        const pushed = req.push === false ? { pushed: false, pushWarning: "chant/lifecycle was not pushed" } : await pushLifecycleReport({ cwd: top });
+        const failed = result.records.find((r) => r.status === "fail");
+        return {
+            op: op.name,
+            status: result.status,
+            record: result.record,
+            ...(failed ? { failure: { phase: failed.phase, error: failed.error ?? "the step failed" } } : {}),
+            report,
+            signer: signer?.principal ?? null,
+            pushed: pushed.pushed,
+            ...(pushed.pushWarning ? { pushWarning: pushed.pushWarning } : {}),
+            warnings,
+            checkout: checkout.dir,
+        };
+    }
+    finally {
+        checkout.remove();
+    }
+}
+/**
+ * Redact every text file under `dir` in place (#4096): the job's report, its
+ * change set and its component events, as `ci run` copied them out. A
+ * JSON file stays JSON. A file with a NUL byte is taken as binary and left.
+ */
+export function redactTree(dir, redact) {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const path = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            redactTree(path, redact);
+            continue;
+        }
+        if (!entry.isFile())
+            continue;
+        const bytes = readFileSync(path);
+        if (bytes.includes(0))
+            continue;
+        const text = bytes.toString("utf-8");
+        const clean = redactText(text, redact);
+        if (clean !== text)
+            writeFileSync(path, clean);
+    }
+}
+/** The text `chant ci run` prints when the run settles. */
+export function formatCiRun(r) {
+    const lines = [`${r.op} ${r.status === "ok" ? "passed" : r.status === "fail" ? "failed" : r.status} at ${r.record.gitSha?.slice(0, 12) ?? "an unknown commit"}, run ${r.record.id} in ${r.record.env}`];
+    const why = r.record.failure;
+    if (why)
+        lines.push(`  failure: ${why.code}${why.phase ? ` in ${why.phase}` : ""}${why.exitCode !== undefined ? `, exit ${why.exitCode}` : ""}: ${why.reason}`);
+    else if (r.failure)
+        lines.push(`  ${r.failure.phase}: ${r.failure.error}`);
+    lines.push(`  record: ${r.record.env}/runs__${r.op}.jsonl on chant/lifecycle, ${r.signer ? `signed by ${r.signer}` : "unsigned, so ci tick from runs does not count it"}`);
+    if (r.report)
+        lines.push(`  report: ${r.report}`);
+    if (!r.pushed)
+        lines.push(`  ${r.pushWarning ?? "chant/lifecycle was not pushed"}`);
+    return lines.join("\n");
+}
+// ── The command ──────────────────────────────────────────────────────────────
+const USAGE = "chant ci run plan|apply --env <env> --base <sha> [--pr <n>] [--commit <sha>] [--forge github|gitlab|forgejo] " +
+    "[--gate <name>] [--branch <name>] [--events-json] [--report-url <url>] [--run-index-limit <n>] [--image <image>] [--before-script <line>]... " +
+    "[--pr-build] [--pr-build-command <line>] [--no-status] [--status-url <url template>] " +
+    "[--chant <command>] [--pass-env <NAME>]... [--secrets <file>] [--key <runner key.pem>] [-o <dir>] | chant ci run --setup-key [--trust [--principal <name>]] [--state <dir>] [--branch <name>]";
+const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+/** The host's runner key in `--state`, or in ci serve's default state directory, when `--setup-key` made one there (#4099). */
+function hostRunnerKeyAt(cwd, statePath) {
+    const common = statePath ? null : (() => {
+        try {
+            return execFileSync("git", ["rev-parse", "--path-format=absolute", "--git-common-dir"], { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] }).trim();
+        }
+        catch {
+            return null;
+        }
+    })();
+    const dir = statePath ? resolve(statePath) : common ? join(common, "chant", "ci-serve") : null;
+    return dir ? hostRunnerKey(dir) : undefined;
+}
+/** The secrets file `--secrets` names, else the one `CHANT_CI_SECRETS` names, read; undefined when neither names one (#4096). */
+export function ciSecretsOf(flag, env = process.env) {
+    const file = flag ?? (env[CI_SECRETS_ENV] || undefined);
+    return file === undefined ? undefined : readCiSecrets(resolve(file));
+}
+/** `chant ci run plan|apply` (#4094). */
+export async function runCiRun(ctx) {
+    const { args } = ctx;
+    try {
+        if (args.setupKey)
+            return (await import("./ci-serve.js")).runnerKeySetupCommand(ctx);
+        const job = args.extraPositional;
+        if (job !== "plan" && job !== "apply")
+            throw new CiRunError(`ci run takes the job to run, plan or apply${job ? `, not ${JSON.stringify(job)}` : ""}`);
+        if (!args.env)
+            throw new CiRunError("--env names the environment the pipeline is for, as chant build --components --generate <lexicon> --pr-loop took it: --env <env>");
+        const forge = args.forge ?? detectForge(process.env);
+        if (!forge)
+            throw new CiRunError("--forge names the forge the job posts to: --forge github|gitlab|forgejo");
+        if (!FORGE_KINDS.includes(forge))
+            throw new CiRunError(`--forge takes github, gitlab or forgejo, not ${JSON.stringify(forge)}`);
+        for (const name of args.passEnv ?? [])
+            if (!ENV_NAME.test(name))
+                throw new CiRunError(`--pass-env takes a variable's name, not ${JSON.stringify(name)}`);
+        const fromFlag = args.key !== undefined;
+        // #4099: without --key or CHANT_RUNNER_KEY, the host's key that --setup-key made, if there is one. Like the variable, an unlisted one only warns.
+        const hostKey = fromFlag || process.env[RUNNER_KEY_ENV] ? undefined : hostRunnerKeyAt(process.cwd(), args.statePath);
+        const runnerKey = fromFlag ? readFileSync(resolve(args.key), "utf-8") : process.env[RUNNER_KEY_ENV] || hostKey?.pem;
+        const hostEnv = process.env;
+        const { status, warning } = ciStatusFromEnv({ forge: forge, hostEnv, noStatus: args.noStatus, statusUrl: args.statusUrl });
+        if (warning)
+            console.error(formatWarning({ message: warning }));
+        const secrets = ciSecretsOf(args.ciSecrets);
+        const result = await ciRun({
+            cwd: process.cwd(),
+            ...(status ? { status } : {}),
+            job,
+            env: args.env,
+            forge: forge,
+            ...(args.base ? { base: args.base } : {}),
+            ...(args.pr !== undefined ? { pr: args.pr } : {}),
+            ...(args.commit ? { commit: args.commit } : {}),
+            loop: {
+                ...(args.gate ? { gate: args.gate } : {}),
+                ...(args.branch ? { branch: args.branch } : {}),
+                ...(args.eventsJson ? { events: true } : {}),
+                ...(args.reportUrl ? { reportUrl: args.reportUrl } : {}),
+                ...(args.prBuild ? { build: true } : {}),
+                ...(args.prBuildCommand ? { buildCommand: args.prBuildCommand } : {}),
+            },
+            ...(args.runIndexLimit ? { runIndexLimit: args.runIndexLimit } : {}),
+            ...(args.image ? { image: args.image } : {}),
+            ...(args.beforeScript?.length ? { beforeScript: args.beforeScript } : {}),
+            ...(args.ciChant ? { chant: args.ciChant } : {}),
+            ...(args.passEnv?.length ? { passEnv: args.passEnv } : {}),
+            ...(secrets ? { secrets } : {}),
+            ...(args.output ? { output: args.output } : {}),
+            ...(runnerKey !== undefined ? { runnerKey, runnerKeyFromEnv: !fromFlag } : {}),
+            ...(args.eventsJson ? { runEvents: (line) => void process.stdout.write(line) } : {}),
+            stepOutput: (line) => void process.stderr.write(`${line}\n`),
+        });
+        for (const w of result.warnings)
+            console.error(formatWarning({ message: w }));
+        console.error(formatCiRun(result));
+        return result.status === "ok" ? 0 : 1;
+    }
+    catch (err) {
+        console.error(formatError({ message: `chant ci run: ${message(err)}`, hint: USAGE }));
+        return 1;
+    }
+}