forgedock 1.17.0 → 1.19.0

package/scripts/check-test-tmp-cleanup.test.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# Tests for scripts/check-test-tmp-cleanup.sh (passing and failing fixtures, plus the real tree).
+# Run: bash scripts/check-test-tmp-cleanup.test.sh   (bash 3.2 compatible)
+
+set -u
+DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+LINT="$DIR/check-test-tmp-cleanup.sh"
+T=$(mktemp -d "${TMPDIR:-/tmp}/check-test-tmp-cleanup-test.XXXXXX")
+trap 'rm -rf "$T"' EXIT
+
+PASS=0; FAIL=0
+ok()  { PASS=$((PASS + 1)); echo "  ok - $1"; }
+bad() { FAIL=$((FAIL + 1)); echo "  FAIL - $1"; }
+
+# run_case NAME EXPECT_RC EXPECT_OUT_SUBSTR  (fixture tree = $T/$NAME)
+run_case() {
+  local name="$1" want_rc="$2" want_out="$3" out rc
+  out=$(bash "$LINT" "$T/$name" 2>&1); rc=$?
+  [ "$rc" -eq "$want_rc" ] && ok "$name: exit $want_rc" || bad "$name: exit $rc, wanted $want_rc ($out)"
+  case "$out" in *"$want_out"*) ok "$name: output mentions '$want_out'" ;; *) bad "$name: output lacks '$want_out' ($out)" ;; esac
+}
+
+mk() { mkdir -p "$T/$1/bin/tests" "$T/$1/scripts"; }
+
+mk pass_node
+printf 'import { tmpDir } from "./_tmp.mjs";\nconst d = tmpDir("fd-x-");\n' > "$T/pass_node/bin/tests/a.test.mjs"
+run_case pass_node 0 "OK"
+
+mk pass_node_nested
+mkdir -p "$T/pass_node_nested/bin/stats"
+printf 'import { tmpDir } from "../tests/_tmp.mjs";\nconst d = tmpDir("fd-x-");\n' > "$T/pass_node_nested/bin/stats/a.test.mjs"
+run_case pass_node_nested 0 "OK"
+
+mk fail_node
+printf 'import { mkdtempSync } from "node:fs";\nconst d = mkdtempSync("/tmp/fd-x-");\n' > "$T/fail_node/bin/tests/b.test.mjs"
+run_case fail_node 1 "b.test.mjs"
+
+mk fail_node_multiline_import
+printf 'import {\n  mkdtempSync,\n  rmSync,\n} from "node:fs";\n' > "$T/fail_node_multiline_import/bin/tests/c.test.mjs"
+run_case fail_node_multiline_import 1 "c.test.mjs"
+
+mk pass_sh
+printf '#!/usr/bin/env bash\nT=$(mktemp -d)\ntrap '"'"'rm -rf "$T"'"'"' EXIT\n' > "$T/pass_sh/scripts/a.test.sh"
+run_case pass_sh 0 "OK"
+
+mk fail_sh
+printf '#!/usr/bin/env bash\nT=$(mktemp -d)\necho hi\n' > "$T/fail_sh/scripts/b.test.sh"
+run_case fail_sh 1 "b.test.sh"
+
+mk fail_sh_trap_not_exit
+printf '#!/usr/bin/env bash\nT=$(mktemp)\ntrap '"'"'echo x'"'"' INT\n' > "$T/fail_sh_trap_not_exit/scripts/c.test.sh"
+run_case fail_sh_trap_not_exit 1 "c.test.sh"
+
+mk pass_no_temp
+printf 'const x = 1;\n' > "$T/pass_no_temp/bin/tests/d.test.mjs"
+printf '#!/usr/bin/env bash\n# mktemp is only mentioned in a comment\n' > "$T/pass_no_temp/scripts/d.test.sh"
+run_case pass_no_temp 0 "OK"
+
+out=$(bash "$LINT" "$T/does-not-exist" 2>&1); rc=$?
+[ "$rc" -eq 2 ] && ok "bad ROOT exits 2" || bad "bad ROOT exit $rc ($out)"
+
+out=$(bash "$LINT" 2>&1); rc=$?
+[ "$rc" -eq 0 ] && ok "lint is clean on the real repo tree" || bad "real tree has offenders: $out"
+
+echo "passed=$PASS failed=$FAIL"
+[ "$FAIL" -eq 0 ]

package/scripts/ci-local/adapters/azure-pipelines.mjs

@@ -0,0 +1,113 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// azure-pipelines.mjs - azure-pipelines.yml -> normalised jobs. pr: triggers, stages/jobs/steps, script/bash steps,
+// a small set of known no-op tasks, $(Var) macros from declared and predefined variables. Templates, deployment jobs,
+// variable groups, container jobs and unknown tasks are deferred-to-ci.
+
+import os from 'node:os';
+import { Unsupported, branchMatches, matchesFilter, jobId, deferred, notApplicable, hostClass, DEPLOY_NAME } from '../lib.mjs';
+import { stepsFromScript, loadDoc, parseSafe, baseJob } from './common.mjs';
+
+export const id = 'azure-pipelines';
+export const DEF_RE = /^azure-pipelines\.ya?ml$/;
+const NOOP_TASKS = /^(NodeTool|UsePythonVersion|GoTool|JavaToolInstaller|UseDotNet|Cache|PublishBuildArtifacts|PublishTestResults|PublishCodeCoverageResults|PublishPipelineArtifact)@/;
+
+function varMap(v) {
+  const m = {}; if (!v) return m;
+  if (Array.isArray(v)) { for (const x of v) { if (x.group) throw new Unsupported('variable group'); if (x.template) throw new Unsupported('variable template'); if (x.name) m[x.name] = String(x.value ?? ''); } } else for (const [k, x] of Object.entries(v)) m[k] = String(x && typeof x === 'object' ? x.value ?? '' : x);
+  return m;
+}
+
+function expandMacros(text, vars) {
+  if (/\$\{\{/.test(text)) throw new Unsupported('compile-time ${{ }} expression');
+  return String(text).replace(/\$\(([A-Za-z0-9_.]+)\)/g, (_, n) => { if (n in vars) return vars[n]; throw new Unsupported(`variable $(${n})`); });
+}
+
+function prOk(doc, a) {
+  const pr = doc.pr;
+  if (pr === 'none' || pr === false) return 'pr: none';
+  if (pr === undefined || pr === null) return '';
+  const br = Array.isArray(pr) ? { include: pr } : (pr.branches || {});
+  if (br.include && !branchMatches(a.baseBranch, [].concat(br.include))) return `pr branches exclude base ${a.baseBranch}`;
+  if (br.exclude && branchMatches(a.baseBranch, [].concat(br.exclude))) return `pr branches exclude base ${a.baseBranch}`;
+  if (pr.paths && a.changed.length) {
+    const inc = pr.paths.include || []; const exc = pr.paths.exclude || [];
+    if (!a.changed.some((f) => (!inc.length || matchesFilter(f, inc)) && !matchesFilter(f, exc.length ? exc : ['\0']))) return 'pr paths select none of the changed files';
+  }
+  return '';
+}
+
+function condOk(cond) {
+  if (cond === undefined) return true;
+  const c = String(cond).replace(/\s+/g, '');
+  if (/^(succeeded\(\)|succeeded\('[^']*'\)|always\(\)|succeededOrFailed\(\))$/.test(c)) return true;
+  if (/^eq\(variables\['Build\.Reason'\],'PullRequest'\)$/.test(c)) return true;
+  if (/^ne\(variables\['Build\.Reason'\],'(Manual|Schedule|IndividualCI|BatchedCI)'\)$/.test(c)) return true;
+  if (/^(eq|ne)\(variables\['Build\.Reason'\],/.test(c)) return false;
+  throw new Unsupported('condition ' + String(cond).slice(0, 60));
+}
+
+function osOf(pool) { const img = String((pool && (pool.vmImage || pool.name)) || '').toLowerCase(); if (/windows/.test(img)) return 'win32'; if (/mac/.test(img)) return 'darwin'; if (/ubuntu|linux/.test(img)) return 'linux'; return ''; }
+const runnable = (o) => (o === 'linux' ? hostClass() !== 'win32' : o === hostClass());
+
+export function plan(a) {
+  const out = []; const warnings = []; const used = [];
+  for (const file of a.defFiles(DEF_RE)) {
+    const doc = loadDoc(a, file, parseSafe);
+    if (doc === undefined) { out.push(deferred(jobId('azure', 'pipeline'), 'azure-pipelines', file, 'pipeline YAML unreadable')); continue; }
+    if (!doc || typeof doc !== 'object') continue;
+    used.push(file);
+    const why = prOk(doc, a);
+    if (why) { out.push(notApplicable(jobId('azure', 'pipeline'), 'azure-pipelines', file, why)); continue; }
+    let gvars;
+    try { gvars = varMap(doc.variables); } catch (e) { out.push(deferred(jobId('azure', 'pipeline'), 'azure-pipelines', file, e.message)); continue; }
+    const jobs = [];
+    if (Array.isArray(doc.stages)) for (const st of doc.stages) for (const j of st.jobs || []) jobs.push({ j, pool: doc.pool, stage: st.stage });
+    else if (Array.isArray(doc.jobs)) for (const j of doc.jobs) jobs.push({ j, pool: doc.pool });
+    else if (Array.isArray(doc.steps)) jobs.push({ j: { job: 'build', steps: doc.steps }, pool: doc.pool });
+    for (const { j, pool, stage } of jobs) {
+      const key = j.job || j.deployment || j.template || 'job';
+      const jid = jobId('azure', stage || '', key); const name = j.displayName || key;
+      const def = (r) => out.push(deferred(jid, 'azure-pipelines', name, r));
+      try {
+        if (j.template) { def('job template'); continue; }
+        if (j.deployment) { def('deployment job'); continue; }
+        if (j.container || j.services) { def('container/service job'); continue; }
+        if (DEPLOY_NAME.test(key)) { def('deploy/publish job'); continue; }
+        if (!condOk(j.condition)) { out.push(notApplicable(jid, 'azure-pipelines', name, 'condition is false for pull requests')); continue; }
+        const jvars = { ...gvars, ...varMap(j.variables) };
+        const p = j.pool || pool; const o = osOf(p);
+        if (!o) { def('unknown agent pool'); continue; }
+        if (!runnable(o)) { def(`${o} agent (host is ${hostClass()})`); continue; }
+        const pre = { 'Build.SourcesDirectory': a.repo, 'Build.Repository.LocalPath': a.repo, 'Build.SourceVersion': a.headSha, 'Build.SourceBranch': `refs/heads/${a.headBranch || ''}`, 'Build.SourceBranchName': a.headBranch || '', 'System.PullRequest.TargetBranch': a.baseBranch, 'System.PullRequest.SourceBranch': a.headBranch || '', 'Build.Reason': 'PullRequest', 'Agent.OS': { linux: 'Linux', darwin: 'Darwin', win32: 'Windows_NT' }[hostClass()], 'Agent.TempDirectory': os.tmpdir(), 'Pipeline.Workspace': a.repo, 'Build.ArtifactStagingDirectory': os.tmpdir() };
+        const vars = { ...pre, ...jvars };
+        let combos = [{}]; let tag; let variants = 0;
+        const mx = j.strategy && j.strategy.matrix;
+        if (mx && typeof mx === 'object') { const ks = Object.keys(mx); combos = ks.map((k) => ({ __n: k, ...mx[k] })); variants = combos.length - 1; tag = ks[0]; }
+        const env = {}; for (const [k, v] of Object.entries(combos[0])) if (k !== '__n') { env[k] = String(v); vars[k] = String(v); }
+        const steps = []; let defer = '';
+        for (const s of j.steps || []) {
+          if (s.condition !== undefined && !condOk(s.condition)) continue;
+          if (s.template || s.download || s.getPackage) { defer = defer || 'step template/download'; break; }
+          if (s.checkout !== undefined) continue;
+          if (s.task) {
+            if (NOOP_TASKS.test(s.task)) continue;
+            if (/^Npm@1$/.test(s.task) && s.inputs && ['ci', 'install'].includes(s.inputs.command || 'install')) { steps.push({ name: s.displayName || 'npm ' + (s.inputs.command || 'install'), command: 'npm ' + (s.inputs.command || 'install'), cwd: '.', env: {}, kind: 'install' }); continue; }
+            defer = defer || `unknown task ${s.task}`; break;
+          }
+          const body = s.script ?? s.bash;
+          if (body === undefined) { defer = defer || (s.pwsh || s.powershell ? 'PowerShell step' : 'unsupported step'); break; }
+          const sh = s.bash !== undefined ? 'bash' : undefined;
+          const r = stepsFromScript(expandMacros(body, vars), { name: s.displayName || 'script', cwd: s.workingDirectory ? expandMacros(s.workingDirectory, vars) : '.', env: Object.fromEntries(Object.entries(s.env || {}).map(([k, v]) => [k, expandMacros(v, vars)])), shell: sh, repo: a.repo, continue_on_error: s.continueOnError === true });
+          if (r.defer) { defer = defer || r.defer; break; }
+          steps.push(...r.steps);
+        }
+        if (defer) { def(defer); continue; }
+        out.push(baseJob({ id: jid, source: 'azure-pipelines', name, env: { ...env, BUILD_SOURCESDIRECTORY: a.repo }, steps, needs: [].concat(j.dependsOn || []).map((d) => jobId('azure', stage || '', d)), matrix: tag, timeout_s: j.timeoutInMinutes ? Number(j.timeoutInMinutes) * 60 : undefined }));
+        if (variants) out.push(deferred(jobId(jid, 'variants'), 'azure-pipelines', name, `${variants} matrix variant(s) not run locally`));
+      } catch (e) { if (e instanceof Unsupported) { def(`unsupported construct: ${e.message}`); continue; } throw e; }
+    }
+  }
+  return { jobs: out, warnings, used };
+}

package/scripts/ci-local/adapters/bitbucket.mjs

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// bitbucket.mjs - bitbucket-pipelines.yml -> normalised jobs. Pull-request (else default) pipeline, step/parallel/stage
+// items, condition.changesets path filters, definition services on host networking. Pipes, deployments, manual
+// triggers, OIDC and credential-like variables are deferred-to-ci.
+
+import { Unsupported, branchMatches, matchesFilter, jobId, deferred, notApplicable, secretLikeVar, hostClass, DEPLOY_NAME } from '../lib.mjs';
+import { stepsFromScript, loadDoc, parseSafe, baseJob, globList } from './common.mjs';
+
+export const id = 'bitbucket';
+export const DEF_RE = /^bitbucket-pipelines\.ya?ml$/;
+
+function flatten(items, parallel = false, out = []) {
+  for (const it of items || []) {
+    if (it.step) out.push({ step: it.step, parallel });
+    else if (it.parallel) flatten(Array.isArray(it.parallel) ? it.parallel : it.parallel.steps, true, out);
+    else if (it.stage) flatten(it.stage.steps, parallel, out);
+  }
+  return out;
+}
+
+export function plan(a) {
+  const out = []; const warnings = []; const used = [];
+  for (const file of a.defFiles(DEF_RE)) {
+    const doc = loadDoc(a, file, parseSafe);
+    if (doc === undefined) { out.push(deferred(jobId('bitbucket', 'pipeline'), 'bitbucket', file, 'pipeline YAML unreadable')); continue; }
+    if (!doc || !doc.pipelines) continue;
+    used.push(file);
+    let items = null; let which = 'default';
+    const pr = doc.pipelines['pull-requests'];
+    if (pr) for (const [pat, v] of Object.entries(pr)) if (branchMatches(a.headBranch || '', [pat])) { items = v; which = `pull-requests:${pat}`; break; }
+    if (!items && Array.isArray(doc.pipelines.default)) items = doc.pipelines.default;
+    if (!items) { out.push(notApplicable(jobId('bitbucket', 'pipeline'), 'bitbucket', file, 'no pull-request or default pipeline matches this branch')); continue; }
+    const steps = flatten(items); let prev = []; let prevIds = [];
+    steps.forEach((entry, idx) => {
+      const s = entry.step; const name = s.name || `step-${idx + 1}`; const jid = jobId('bitbucket', which, String(idx + 1), name);
+      const def = (r) => out.push(deferred(jid, 'bitbucket', name, r));
+      const needs = entry.parallel ? prev : prevIds;
+      const next = () => { if (!entry.parallel) { prevIds = [jid]; prev = [jid]; } else prev = prev.concat([jid]); };
+      try {
+        if (s.trigger === 'manual') { def('manual trigger'); next(); return; }
+        if (s.deployment || DEPLOY_NAME.test(name)) { def('deployment step'); next(); return; }
+        if (s.oidc) { def('OIDC token'); next(); return; }
+        if (s['runs-on']) { def('self-hosted runner'); next(); return; }
+        const inc = s.condition && s.condition.changesets && globList(s.condition.changesets.includePaths);
+        if (inc && inc.length && a.changed.length && !a.changed.some((f) => matchesFilter(f, inc))) { out.push(notApplicable(jid, 'bitbucket', name, 'condition.changesets selects none of the changed files')); next(); return; }
+        const script = s.script || [];
+        if (script.some((l) => typeof l !== 'string')) { def('script uses a pipe'); next(); return; }
+        const svcNames = (s.services || []).filter((x) => x !== 'docker');
+        const services = svcNames.map((n) => { const d = doc.definitions && doc.definitions.services && doc.definitions.services[n]; if (!d || !d.image) throw new Unsupported(`service ${n}`); return { name: n, image: typeof d.image === 'string' ? d.image : d.image.name, env: d.variables || {}, ports: [], options: '', host_network: true }; });
+        if (services.length && hostClass() !== 'linux') { def('service containers need host networking (Linux only)'); next(); return; }
+        const sv = secretLikeVar(script.join('\n'));
+        if (sv) { def(`references credential-like variable $${sv}`); next(); return; }
+        const r = stepsFromScript(script.join('\n'), { name, cwd: '.', env: {}, repo: a.repo });
+        if (r.defer) { def(r.defer); next(); return; }
+        const env = { BITBUCKET_COMMIT: a.headSha, BITBUCKET_BRANCH: a.headBranch || '', BITBUCKET_PR_DESTINATION_BRANCH: a.baseBranch, BITBUCKET_CLONE_DIR: a.repo };
+        out.push(baseJob({ id: jid, source: 'bitbucket', name, env, steps: r.steps, services, needs, timeout_s: s['max-time'] ? Number(s['max-time']) * 60 : undefined, warnings: s.image ? [`image ${typeof s.image === 'string' ? s.image : s.image.name} ignored; running with the host toolchain`] : [] }));
+        next();
+      } catch (e) { if (e instanceof Unsupported) { def(`unsupported construct: ${e.message}`); next(); return; } throw e; }
+    });
+  }
+  return { jobs: out, warnings, used };
+}

package/scripts/ci-local/adapters/buildkite.mjs

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// buildkite.mjs - .buildkite/pipeline.yml -> normalised jobs. Command steps, wait barriers, groups, branch filters and
+// simple `if:` conditions. Plugins, block/input/trigger steps, buildkite-agent calls, parallelism/matrix and
+// credential-like variables are deferred-to-ci.
+
+import { Unsupported, evalExpr, branchMatches, jobId, deferred, notApplicable, secretLikeVar, hostClass, DEPLOY_NAME } from '../lib.mjs';
+import { stepsFromScript, loadDoc, parseSafe, baseJob } from './common.mjs';
+
+export const id = 'buildkite';
+export const DEF_RE = /^(\.buildkite\/pipeline\.ya?ml|buildkite\.ya?ml)$/;
+
+function flat(steps, out = []) { for (const s of steps || []) { if (s && s.group) flat(s.steps, out); else out.push(s); } return out; }
+
+export function plan(a) {
+  const out = []; const warnings = []; const used = [];
+  for (const file of a.defFiles(DEF_RE)) {
+    const doc = loadDoc(a, file, parseSafe);
+    if (doc === undefined) { out.push(deferred(jobId('buildkite', 'pipeline'), 'buildkite', file, 'pipeline YAML unreadable')); continue; }
+    if (!doc || !Array.isArray(doc.steps)) continue;
+    used.push(file);
+    const roots = { build: { branch: a.headBranch || '', source: 'webhook', pull_request: { id: '1', base_branch: a.baseBranch }, tag: '', message: '', commit: a.headSha }, pipeline: { slug: 'local' } };
+    const keys = {}; let barrier = []; const emitted = [];
+    const items = flat(doc.steps);
+    items.forEach((s, idx) => {
+      if (typeof s === 'string' || (s && (s.wait !== undefined || s.waiter !== undefined))) { barrier = emitted.slice(); return; }
+      if (!s) return;
+      const label = String(s.label || s.name || (typeof s.block === 'string' ? s.block : '') || (typeof s.input === 'string' ? s.input : '') || s.key || `step-${idx + 1}`); const jid = jobId('buildkite', s.key || String(idx + 1), label);
+      if (s.key) keys[s.key] = jid;
+      const def = (r) => { out.push(deferred(jid, 'buildkite', label, r)); emitted.push(jid); };
+      try {
+        if (s.block || s.input || s.trigger) { def('block/input/trigger step'); return; }
+        if (s.command === undefined && s.commands === undefined) return;
+        if (s.skip) { out.push(notApplicable(jid, 'buildkite', label, 'skip is set')); return; }
+        if (s.branches && !branchMatches(a.headBranch || '', String(s.branches).split(/\s+/).filter(Boolean))) { out.push(notApplicable(jid, 'buildkite', label, 'branches filter excludes this branch')); return; }
+        if (s.if !== undefined && !Boolean(evalExpr(String(s.if).replace(/"([^"]*)"/g, "'$1'"), { __roots: roots }))) { out.push(notApplicable(jid, 'buildkite', label, 'if: is false')); return; }
+        if (s.plugins && (Array.isArray(s.plugins) ? s.plugins.length : Object.keys(s.plugins).length)) { def('uses plugins (docker/containers)'); return; }
+        const ag = s.agents || {}; const agOs = String(Array.isArray(ag) ? '' : ag.os || '').toLowerCase();
+        if (agOs && agOs !== hostClass() && !(agOs === 'linux' && hostClass() === 'darwin')) { def(`agent os=${agOs}`); return; }
+        if (s.parallelism || s.matrix) { def('parallelism/matrix step'); return; }
+        if (DEPLOY_NAME.test(label)) { def('deploy/publish step'); return; }
+        const cmds = [].concat(s.commands || s.command).map(String);
+        if (cmds.some((c) => /buildkite-agent/.test(c))) { def('uses buildkite-agent'); return; }
+        const env = Object.fromEntries(Object.entries(s.env || {}).map(([k, v]) => [k, String(v)]));
+        const sv = secretLikeVar(cmds.join('\n'), new Set(Object.keys(env)));
+        if (sv) { def(`references credential-like variable $${sv}`); return; }
+        const r = stepsFromScript(cmds.join('\n'), { name: label, cwd: '.', env: {}, repo: a.repo, continue_on_error: s.soft_fail === true, timeout_s: s.timeout_in_minutes ? Number(s.timeout_in_minutes) * 60 : undefined });
+        if (r.defer) { def(r.defer); return; }
+        const needs = (s.depends_on !== undefined ? [].concat(s.depends_on).map((k) => keys[typeof k === 'string' ? k : k && k.step]).filter(Boolean) : barrier);
+        Object.assign(env, { BUILDKITE_BRANCH: a.headBranch || '', BUILDKITE_COMMIT: a.headSha, BUILDKITE_BUILD_CHECKOUT_PATH: a.repo, BUILDKITE_PULL_REQUEST_BASE_BRANCH: a.baseBranch });
+        out.push(baseJob({ id: jid, source: 'buildkite', name: label, env, steps: r.steps, needs, timeout_s: s.timeout_in_minutes ? Number(s.timeout_in_minutes) * 60 : undefined }));
+        emitted.push(jid);
+      } catch (e) { if (e instanceof Unsupported) { def(`unsupported construct: ${e.message}`); return; } throw e; }
+    });
+  }
+  return { jobs: out, warnings, used };
+}

package/scripts/ci-local/adapters/circleci.mjs

@@ -0,0 +1,101 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// circleci.mjs - .circleci/config.yml -> normalised jobs. Workflow job lists, branch filters, requires ordering,
+// commands/executors with simple parameters, docker service containers (host network). Contexts (secrets), orbs,
+// approvals, macOS executors off macOS, add_ssh_keys and deploy-named jobs are deferred-to-ci.
+
+import { Unsupported, branchMatches, jobId, deferred, notApplicable, secretLikeVar, DEPLOY_NAME, hostClass, parseDuration } from '../lib.mjs';
+import { stepsFromScript, loadDoc, parseSafe, baseJob } from './common.mjs';
+
+export const id = 'circleci';
+export const DEF_RE = /^\.circleci\/config\.ya?ml$/;
+const NOOP = new Set(['checkout', 'restore_cache', 'save_cache', 'persist_to_workspace', 'attach_workspace', 'store_artifacts', 'store_test_results', 'setup_remote_docker']);
+
+function subst(text, params) {
+  return String(text).replace(/<<\s*parameters\.([A-Za-z0-9_-]+)\s*>>/g, (_, k) => { if (!(k in params)) throw new Unsupported(`parameter ${k}`); return String(params[k]); })
+    .replace(/<<\s*(pipeline|[^>\s]+)[^>]*>>/g, (m) => { throw new Unsupported(`template ${m.slice(0, 40)}`); });
+}
+
+function expand(steps, doc, params, depth, out) {
+  if (depth > 4) throw new Unsupported('command nesting');
+  for (const st of steps || []) {
+    if (typeof st === 'string') { if (NOOP.has(st)) continue; if (doc.commands && doc.commands[st]) { expand(doc.commands[st].steps, doc, defaultsOf(doc.commands[st], {}), depth + 1, out); continue; } throw new Unsupported(`step ${st}`); }
+    const [k] = Object.keys(st); const v = st[k];
+    if (NOOP.has(k)) continue;
+    if (k === 'run') {
+      const o = typeof v === 'string' ? { command: v } : v;
+      if (o.when === 'on_fail' || o.when === 'never') continue;
+      out.push({ name: o.name || 'run', command: subst(o.command, params), env: Object.fromEntries(Object.entries(o.environment || {}).map(([a, b]) => [a, subst(b, params)])), wd: o.working_directory ? subst(o.working_directory, params) : '.' });
+      continue;
+    }
+    if (doc.commands && doc.commands[k]) { expand(doc.commands[k].steps, doc, defaultsOf(doc.commands[k], v || {}), depth + 1, out); continue; }
+    throw new Unsupported(`step ${k}`);
+  }
+}
+function defaultsOf(def, args) { const p = {}; for (const [k, d] of Object.entries(def.parameters || {})) p[k] = k in args ? args[k] : d && d.default; for (const [k, v] of Object.entries(args)) p[k] = v; return p; }
+
+const DEFAULT_PORTS = { postgres: 5432, mysql: 3306, mariadb: 3306, redis: 6379, mongo: 27017, rabbitmq: 5672, memcached: 11211, elasticsearch: 9200 };
+export function declaredPorts(d) {
+  if (Array.isArray(d.ports) && d.ports.length) return d.ports.map((p) => (/^\d+$/.test(String(p)) ? `${p}:${p}` : String(p)));
+  const base = String(d.image || '').split('/').pop().split(':')[0].replace(/^cimg-/, '');
+  const hit = DEFAULT_PORTS[base] || DEFAULT_PORTS[String(d.image || '').split('/').pop().split(':')[0]];
+  return hit ? [`${hit}:${hit}`] : [];
+}
+
+export function plan(a) {
+  const out = []; const warnings = []; const used = [];
+  for (const file of a.defFiles(DEF_RE)) {
+    const doc = loadDoc(a, file, parseSafe);
+    if (doc === undefined) { out.push(deferred(jobId('circleci', 'config'), 'circleci', file, 'config YAML unreadable')); continue; }
+    if (!doc || !doc.jobs) continue;
+    used.push(file);
+    if (doc.orbs) warnings.push(`${file}: orbs declared; jobs/commands from orbs are deferred-to-ci`);
+    const entries = [];
+    const wfs = Object.entries(doc.workflows || {}).filter(([k]) => k !== 'version');
+    if (!wfs.length && doc.jobs.build) entries.push({ job: 'build', name: 'build', cfg: {} });
+    for (const [wn, wf] of wfs) {
+      if (wf.when !== undefined || wf.unless !== undefined) { out.push(deferred(jobId('circleci', wn), 'circleci', wn, 'workflow when/unless conditions are not evaluated')); continue; }
+      for (const item of wf.jobs || []) {
+        const [jn, cfg] = typeof item === 'string' ? [item, {}] : [Object.keys(item)[0], Object.values(item)[0] || {}];
+        entries.push({ job: jn, name: cfg.name || jn, cfg, wf: wn });
+      }
+    }
+    const nameToId = {}; for (const e of entries) nameToId[e.name] = jobId('circleci', e.wf || '', e.name);
+    for (const e of entries) {
+      const jid = nameToId[e.name]; const jobDef = doc.jobs[e.job];
+      const def = (r) => out.push(deferred(jid, 'circleci', e.name, r));
+      try {
+        if (!jobDef) { def(`job ${e.job} defined by an orb or missing`); continue; }
+        const f = e.cfg.filters && e.cfg.filters.branches;
+        if (f && ((f.only && !branchMatches(a.headBranch || '', [].concat(f.only))) || (f.ignore && branchMatches(a.headBranch || '', [].concat(f.ignore))))) { out.push(notApplicable(jid, 'circleci', e.name, 'branch filter excludes this branch')); continue; }
+        if (e.cfg.context) { def('uses a context (secrets)'); continue; }
+        if (e.cfg.type === 'approval') { def('approval gate'); continue; }
+        if (e.cfg.matrix) { def('matrix job'); continue; }
+        if (DEPLOY_NAME.test(e.job)) { def('deploy/publish job'); continue; }
+        let exec = jobDef; if (jobDef.executor) { const ex = typeof jobDef.executor === 'string' ? jobDef.executor : jobDef.executor.name; if (!doc.executors || !doc.executors[ex]) { def(`executor ${ex} is not defined in this file`); continue; } exec = { ...doc.executors[ex], ...jobDef }; }
+        if (exec.macos && hostClass() !== 'darwin') { def('macOS executor'); continue; }
+        if (exec.machine && exec.machine.image && /windows/i.test(exec.machine.image)) { def('Windows executor'); continue; }
+        const params = defaultsOf(jobDef, Object.fromEntries(Object.entries(e.cfg).filter(([k]) => !['requires', 'filters', 'context', 'name', 'type', 'matrix'].includes(k))));
+        const steps = []; expand(jobDef.steps, doc, params, 0, steps);
+        if (jobDef.steps.some((s) => s && typeof s === 'object' && Object.keys(s)[0] === 'add_ssh_keys')) { def('add_ssh_keys'); continue; }
+        const services = [];
+        // forge#4040 (SEC-6): secondary containers never use host networking. Each runs on a per-job docker network and only
+        // its declared ports (a `ports:` list, else the image's well-known default) are published on 127.0.0.1.
+        if (Array.isArray(exec.docker) && exec.docker.length > 1) for (const d of exec.docker.slice(1)) services.push({ name: String(d.name || d.image).replace(/[^A-Za-z0-9_.-]/g, '-'), image: d.image, env: d.environment || {}, ports: declaredPorts(d), options: '' });
+        { const blind = services.filter((sv) => !sv.ports.length); if (blind.length) { def(`unknown service ports: ${blind.map((sv) => sv.image).join(', ')} (no ports: declared and no well-known default; unreachable without host networking)`); continue; } }
+        const env = Object.fromEntries(Object.entries({ ...(exec.environment || {}), ...(exec.docker && exec.docker[0] ? exec.docker[0].environment || {} : {}) }).map(([k, v]) => [k, subst(v, params)]));
+        const defined = new Set(Object.keys(env));
+        const all = steps.map((s) => s.command).join('\n');
+        const sv = secretLikeVar(all, defined);
+        if (sv) { def(`references credential-like variable $${sv}`); continue; }
+        Object.assign(env, { CIRCLE_BRANCH: a.headBranch || '', CIRCLE_SHA1: a.headSha, CIRCLE_WORKING_DIRECTORY: a.repo, CIRCLE_JOB: e.job });
+        const built = []; let defer = '';
+        for (const s of steps) { const r = stepsFromScript(s.command, { name: s.name, cwd: s.wd, env: s.env, repo: a.repo }); if (r.defer) defer = defer || r.defer; built.push(...r.steps); }
+        if (defer) { def(defer); continue; }
+        out.push(baseJob({ id: jid, source: 'circleci', name: e.name, env, steps: built, services, needs: (e.cfg.requires || []).map((r) => nameToId[r]).filter(Boolean), timeout_s: parseDuration(jobDef.no_output_timeout) }));
+      } catch (err) { if (err instanceof Unsupported) { def(`unsupported construct: ${err.message}`); continue; } throw err; }
+    }
+  }
+  return { jobs: out, warnings, used };
+}

package/scripts/ci-local/adapters/common.mjs

@@ -0,0 +1,115 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// common.mjs - what every source adapter shares: turning a script into classified steps, and the rule that a step
+// which installs system packages, deploys, or talks to a remote service is never run locally.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { commandExists, shellSplit, parseYaml, Degraded } from '../lib.mjs';
+import { splitBlock } from '../scoping.mjs';
+
+const SYSTEM = /^(sudo|apt|apt-get|aptitude|yum|dnf|apk|brew|choco|snap|zypper|pacman|port|winget|scoop|systemctl|service)(\s|$)/;
+const REMOTE = /^(docker\s+(push|login)|podman\s+push|kubectl|helm|terraform\s+(apply|destroy)|pulumi|aws|gcloud|az|gsutil|doctl|flyctl|fly\s+deploy|vercel|netlify|wrangler|heroku|firebase|npm\s+(publish|login|whoami|token)|yarn\s+npm\s+publish|pnpm\s+publish|twine\s+upload|cargo\s+(publish|login)|gem\s+push|git\s+push|git\s+remote\s+add|gh(\s|$)|curl\s+.*\|\s*(ba)?sh|ssh(\s|$)|scp(\s|$)|rsync\s+.*:|sentry-cli|codecov|slack)/;
+const INSTALL = /^(npm\s+(ci|install|i)|pnpm\s+(install|i)|yarn(\s+install)?|bun\s+install|pip3?\s+install|python3?\s+-m\s+pip\s+install|poetry\s+install|uv\s+(sync|pip\s+install)|pipenv\s+install|bundle\s+install|composer\s+install|go\s+mod\s+(download|tidy)|cargo\s+fetch|mvn\s+.*dependency:|gradle(w)?\s+dependencies|dotnet\s+restore)(\s|$)/;
+const SHELL_BUILTIN = new Set(['cd', 'echo', 'test', '[', '[[', 'export', 'set', 'true', 'false', 'exit', 'printf', 'source', '.', 'read', 'eval', 'exec', 'trap', 'unset', 'command', 'type', ':', 'if', 'for', 'while', 'case', 'until', 'function', 'local', 'declare', 'shift', 'wait', 'return', 'mkdir', 'rm', 'cp', 'mv', 'cat', 'ls', 'pwd', 'sleep', 'grep', 'sed', 'awk', 'find', 'sort', 'head', 'tail', 'wc', 'tr', 'cut', 'xargs', 'diff', 'touch', 'chmod', 'env', 'bash', 'sh', 'git', 'tee', 'basename', 'dirname', 'date', 'uname', 'which', 'tar', 'unzip', 'jq', 'yq', 'make', 'node', 'npm', 'npx']);
+
+// Classify one command line. Returns { kind, defer?, noop?, note? }.
+// Every command segment of a line: split on `&&`, `||`, `;`, `|`, `&`, newlines, `$(`, backticks and grouping, then drop
+// leading env assignments and control words, so `cd x && aws s3 sync` is judged on `aws`, not on `cd`. A quoted separator
+// over-splits, which only ever defers more (the safe direction).
+const LEAD_WORD = /^(?:(?:then|do|else|elif|if|while|until|time|exec|command|env|nohup|!)\s+)+/;
+function segments(cmd) {
+  return cmd.split(/&&|\|\||[;|&\n`(){}]|\$\(/).map((p) => p.trim().replace(/^(?:[A-Za-z_][A-Za-z0-9_]*=\S*\s+)+/, '').replace(LEAD_WORD, '').replace(/^(?:[A-Za-z_][A-Za-z0-9_]*=\S*\s+)+/, '')).filter(Boolean);
+}
+
+export function classifyCommand(cmd, repo, cwdRel) {
+  const first = cmd.trim().replace(/^(?:[A-Za-z_][A-Za-z0-9_]*=\S*\s+)+/, '');
+  const head3 = (t) => t.split(/\s+/).slice(0, 3).join(' ');
+  for (const seg of segments(cmd)) {
+    if (SYSTEM.test(seg)) return { kind: 'system', defer: `system-level command: ${head3(seg)}` };
+    if (REMOTE.test(seg)) return { kind: 'remote', defer: `remote-mutating or service command: ${head3(seg)}` };
+  }
+  // `curl ... | sh` spans a pipe, so it is matched on the whole line as well as per segment.
+  if (/\b(curl|wget)\b[^\n]*\|\s*(ba)?sh\b/.test(cmd)) return { kind: 'remote', defer: `remote-mutating or service command: ${head3(first)}` };
+  if (INSTALL.test(first)) {
+    if (/^(npm|pnpm|yarn|bun)\b/.test(first)) {
+      const nm = path.join(repo, cwdRel || '.', 'node_modules');
+      return fs.existsSync(nm) ? { kind: 'install', noop: true, note: 'node_modules present' } : { kind: 'install', note: 'dependency install (offline failure defers the job)' };
+    }
+    if (/^(go|cargo|mvn|gradle|dotnet)/.test(first)) return { kind: 'install', noop: true, note: 'the build tool fetches dependencies on demand' };
+    return { kind: 'install', noop: true, note: 'python/ruby/php dependency install is not run on the host; assumes a provisioned environment' };
+  }
+  return { kind: 'check' };
+}
+
+// Missing local tool for a simple command: the job is deferred, never failed for an absent binary.
+export function missingTool(cmd) {
+  const toks = shellSplit(cmd);
+  if (!toks) return '';
+  const i = toks.findIndex((t) => !/^[A-Za-z_][A-Za-z0-9_]*=/.test(t));
+  const t = i >= 0 ? toks[i] : '';
+  if (!t || t.includes('/') || SHELL_BUILTIN.has(t) || t.startsWith('.')) return '';
+  return commandExists(t) ? '' : t;
+}
+
+// Build normalised step objects for one `run` script. Lines are split only when provably independent.
+export function stepsFromScript(script, o) {
+  const parts = splitBlock(script);
+  const out = []; let defer = '';
+  parts.forEach((cmd, idx) => {
+    const c = classifyCommand(cmd, o.repo, o.cwd);
+    if (c.defer && !defer) defer = c.defer;
+    // A multi-line block is also scanned line by line for deferrable commands.
+    if (parts.length === 1 && cmd.includes('\n')) for (const l of cmd.split('\n')) { const lc = classifyCommand(l, o.repo, o.cwd); if (lc.defer && !defer) defer = lc.defer; }
+    if (c.noop) { out.push({ name: `${o.name || 'run'}`, command: cmd, cwd: o.cwd, env: o.env || {}, shell: o.shell, kind: 'install', noop: true, note: c.note }); return; }
+    out.push({
+      name: parts.length > 1 ? `${o.name || 'run'} [${idx + 1}/${parts.length}]` : (o.name || 'run'),
+      group: parts.length > 1 ? (o.name || 'run') : undefined,
+      command: cmd, cwd: o.cwd, env: o.env || {}, shell: o.shell, kind: c.kind, note: c.note,
+      continue_on_error: !!o.continue_on_error, timeout_s: o.timeout_s,
+    });
+  });
+  return { steps: out, defer };
+}
+
+// Resolve the shell invocation (argv minus the script file) for a CI `shell:` value; null when it cannot run here.
+export function shellArgv(shell) {
+  const s = (shell || '').trim();
+  if (!s) return ['bash', '-e'];
+  if (s === 'bash') return ['bash', '--noprofile', '--norc', '-eo', 'pipefail'];
+  if (s === 'sh') return ['sh', '-e'];
+  if (/^bash\b/.test(s)) return s.replace(/\s*\{0\}\s*$/, '').split(/\s+/);
+  if (/^sh\b/.test(s)) return s.replace(/\s*\{0\}\s*$/, '').split(/\s+/);
+  if (s === 'python') return ['python3'];
+  return null;
+}
+
+export function deepMerge(a, b) {
+  if (b === undefined) return a;
+  if (a && b && typeof a === 'object' && typeof b === 'object' && !Array.isArray(a) && !Array.isArray(b)) {
+    const out = { ...a }; for (const [k, v] of Object.entries(b)) out[k] = k in a ? deepMerge(a[k], v) : v; return out;
+  }
+  return b;
+}
+
+// Trusted base definition merged with the PR's own edits (PR wins per key; a PR cannot delete a base job).
+export function loadDoc(a, file, parse) {
+  const { base, head } = a.readDefs(file);
+  const b = base != null ? parse(base) : null; const h = head != null ? parse(head) : null;
+  if (b === undefined || h === undefined) return undefined;
+  if (!b) return h; if (!h) return b;
+  return deepMerge(b, h);
+}
+
+export function parseSafe(text) {
+  // parseYaml throws Degraded when yq is absent (propagate) and Error on malformed YAML (undefined = unreadable).
+  try { return parseYaml(text); } catch (e) { if (e instanceof Degraded) throw e; return undefined; }
+}
+
+// Env and checkout conventions for jobs derived from non-GitHub CI systems.
+export function baseJob(o) {
+  return { id: o.id, source: o.source, name: o.name, cwd: '.', env: o.env || {}, services: o.services || [], steps: o.steps || [], needs: o.needs || [], timeout_s: o.timeout_s, deferred_reason: null, warnings: o.warnings || [], matrix: o.matrix };
+}
+
+export function globList(v) { return [].concat(v || []).filter((x) => typeof x === 'string'); }

package/scripts/ci-local/adapters/explicit.mjs

@@ -0,0 +1,21 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// explicit.mjs - commands the operator declared in forge.yaml -> verification.ci_local.extra_commands.
+// Always merged into the plan. verification.commands is NOT repeated here: validate V2 already runs it as the fast pre-step.
+
+import { jobId } from '../lib.mjs';
+import { baseJob, stepsFromScript } from './common.mjs';
+
+export const id = 'explicit';
+
+export function plan(a) {
+  const jobs = [];
+  (a.config.extra_commands || []).forEach((c, i) => {
+    const cmd = typeof c === 'string' ? c : c && c.command; if (!cmd) return;
+    const name = typeof c === 'object' && c.name ? String(c.name) : cmd.split('\n')[0].slice(0, 60);
+    const r = stepsFromScript(String(cmd), { name, cwd: '.', env: {}, repo: a.repo });
+    jobs.push(baseJob({ id: jobId('explicit', String(i + 1)), source: 'explicit', name, steps: r.steps }));
+  });
+  return { jobs, warnings: [], used: [] };
+}