forgedock 1.18.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +13 -0
- package/bin/engine/phases.mjs +54 -4
- package/bin/engine/runlog.mjs +10 -2
- package/bin/engine/telemetry-seam.mjs +3 -0
- package/bin/engine/telemetry.test.mjs +3 -2
- package/bin/engine-cli.mjs +32 -2
- package/bin/engine.mjs +139 -7
- package/bin/runner.mjs +69 -9
- package/bin/stats/aggregate.mjs +7 -3
- package/bin/stats/cache.mjs +7 -3
- package/bin/stats/github-source.mjs +136 -28
- package/bin/stats/index.mjs +46 -9
- package/bin/stats/render.mjs +1 -1
- package/bin/stats/stats.test.mjs +201 -40
- package/commands/review-pr-staging.md +4 -4
- package/commands/review-pr.md +25 -15
- package/commands/work-on/build/architect.md +53 -0
- package/commands/work-on/build/implement.md +19 -0
- package/commands/work-on/build/validate.md +320 -28
- package/commands/work-on/build.md +66 -9
- package/commands/work-on/investigate.md +7 -0
- package/commands/work-on/remediate.md +32 -13
- package/commands/work-on/review.md +32 -19
- package/docs/CONFIG.md +89 -2
- package/docs/OUTCOME-CODES.md +5 -1
- package/docs/STATS.md +11 -4
- package/docs/TELEMETRY.md +5 -2
- package/docs/WORK-ON-RUNTIME.md +18 -0
- package/package.json +2 -2
- package/packages/protocol/src/types.js +1 -1
- package/scripts/acceptance-red-green.sh +90 -0
- package/scripts/acceptance-red-green.test.sh +74 -0
- package/scripts/cap-findings.test.sh +2 -0
- package/scripts/check-contract-scope.test.sh +2 -0
- package/scripts/check-edge-cases.sh +143 -0
- package/scripts/check-edge-cases.test.sh +175 -0
- package/scripts/check-spec-markers.sh +2 -0
- package/scripts/check-test-tmp-cleanup.sh +56 -0
- package/scripts/check-test-tmp-cleanup.test.sh +69 -0
- package/scripts/ci-local/adapters/azure-pipelines.mjs +113 -0
- package/scripts/ci-local/adapters/bitbucket.mjs +64 -0
- package/scripts/ci-local/adapters/buildkite.mjs +58 -0
- package/scripts/ci-local/adapters/circleci.mjs +101 -0
- package/scripts/ci-local/adapters/common.mjs +115 -0
- package/scripts/ci-local/adapters/explicit.mjs +21 -0
- package/scripts/ci-local/adapters/github-actions.mjs +349 -0
- package/scripts/ci-local/adapters/gitlab-ci.mjs +184 -0
- package/scripts/ci-local/adapters/portability.mjs +104 -0
- package/scripts/ci-local/adapters/task-runners.mjs +67 -0
- package/scripts/ci-local/cli.mjs +50 -0
- package/scripts/ci-local/lib.mjs +371 -0
- package/scripts/ci-local/plan.mjs +186 -0
- package/scripts/ci-local/run.mjs +345 -0
- package/scripts/ci-local/scoping.mjs +508 -0
- package/scripts/ci-local.sh +97 -0
- package/scripts/ci-local.test.sh +919 -0
- package/scripts/classify-finding.sh +32 -6
- package/scripts/classify-finding.test.sh +23 -0
- package/scripts/classify-test-run.sh +69 -0
- package/scripts/classify-test-run.test.sh +90 -0
- package/scripts/close-parent-c3.test.sh +2 -0
- package/scripts/consumer-sweep.sh +569 -0
- package/scripts/consumer-sweep.test.sh +357 -0
- package/scripts/detect-append-registries.sh +106 -0
- package/scripts/detect-append-registries.test.sh +64 -0
- package/scripts/fixtures/ci-local/azure/azure-pipelines.yml +11 -0
- package/scripts/fixtures/ci-local/bitbucket/bitbucket-pipelines.yml +12 -0
- package/scripts/fixtures/ci-local/buildkite/.buildkite/pipeline.yml +10 -0
- package/scripts/fixtures/ci-local/circleci/.circleci/config.yml +29 -0
- package/scripts/fixtures/ci-local/gha-basic/.github/workflows/ci.yml +59 -0
- package/scripts/fixtures/ci-local/gha-basic/.github/workflows/docs.yml +10 -0
- package/scripts/fixtures/ci-local/gha-basic/.github/workflows/nightly.yml +10 -0
- package/scripts/fixtures/ci-local/gha-basic/docs/index.md +1 -0
- package/scripts/fixtures/ci-local/gha-basic/src/greet.sh +2 -0
- package/scripts/fixtures/ci-local/gha-basic/tests/greet.test.sh +3 -0
- package/scripts/fixtures/ci-local/gha-dorny/.github/workflows/ci.yml +51 -0
- package/scripts/fixtures/ci-local/gha-dorny/backend/app.py +1 -0
- package/scripts/fixtures/ci-local/gha-dorny/docs/a.md +1 -0
- package/scripts/fixtures/ci-local/gha-dorny/frontend/app.js +1 -0
- package/scripts/fixtures/ci-local/gha-run/.github/workflows/ci.yml +39 -0
- package/scripts/fixtures/ci-local/gha-run/src/x.txt +1 -0
- package/scripts/fixtures/ci-local/gitlab/.gitlab-ci.yml +43 -0
- package/scripts/fixtures/ci-local/gitlab/docs/d.md +1 -0
- package/scripts/fixtures/ci-local/gitlab/src/a.txt +1 -0
- package/scripts/fixtures/ci-local/monorepo/.github/workflows/ci.yml +13 -0
- package/scripts/fixtures/ci-local/monorepo/package-lock.json +1 -0
- package/scripts/fixtures/ci-local/monorepo/packages/a/src/a.sh +2 -0
- package/scripts/fixtures/ci-local/monorepo/packages/a/tests/a.test.sh +3 -0
- package/scripts/fixtures/ci-local/monorepo/packages/b/src/b.sh +2 -0
- package/scripts/fixtures/ci-local/monorepo/packages/b/tests/b.test.sh +3 -0
- package/scripts/fixtures/ci-local/no-ci/Makefile +6 -0
- package/scripts/fixtures/ci-local/no-ci/package.json +1 -0
- package/scripts/fixtures/ci-local/no-ci/src/a.js +1 -0
- package/scripts/fixtures/ci-local/no-ci/src/b.js +1 -0
- package/scripts/fixtures/ci-local/no-ci/tests/a.test.js +2 -0
- package/scripts/fixtures/ci-local/no-ci/tests/b.test.js +2 -0
- package/scripts/forge-root.test.sh +1 -1
- package/scripts/lib/telemetry.sh +1 -0
- package/scripts/remediate-approved-head.test.sh +2 -0
- package/scripts/remediate-basesync.test.sh +1 -1
- package/scripts/review-acceptance-gate.sh +2 -2
- package/scripts/review-acceptance-gate.test.sh +20 -0
- package/scripts/review-delta.sh +8 -1
- package/scripts/review-delta.test.sh +19 -0
- package/scripts/review-finalize.sh +193 -41
- package/scripts/review-finalize.test.sh +211 -4
- package/scripts/review-pr-delta-wiring.test.sh +13 -0
- package/scripts/review-prepare.sh +5 -3
- package/scripts/review-prepare.test.sh +18 -2
- package/scripts/run-shell-tests.sh +5 -0
- package/scripts/telemetry.test.sh +2 -0
- package/scripts/trusted-comments.test.sh +2 -1
- package/scripts/verify-phase-trail.test.sh +8 -0
- package/scripts/wait-ci-green.sh +61 -9
- package/scripts/wait-ci-green.test.sh +41 -3
- package/scripts/with-run-tmpdir.sh +20 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
//
|
|
4
|
+
// gitlab-ci.mjs - .gitlab-ci.yml (+ local includes) -> normalised jobs. Merge-request pipeline semantics:
|
|
5
|
+
// rules/only/except with changes: path filters, stages and needs ordering, extends, variables. Services, triggers,
|
|
6
|
+
// environments, manual jobs and credential-like variables are deferred-to-ci.
|
|
7
|
+
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import { Unsupported, matchesFilter, branchMatches, jobId, deferred, notApplicable, parseDuration, secretLikeVar, DEPLOY_NAME } from '../lib.mjs';
|
|
10
|
+
import { stepsFromScript, loadDoc, parseSafe, deepMerge, baseJob, globList } from './common.mjs';
|
|
11
|
+
|
|
12
|
+
export const id = 'gitlab-ci';
|
|
13
|
+
export const DEF_RE = /^\.gitlab-ci\.ya?ml$/;
|
|
14
|
+
const RESERVED = new Set(['stages', 'variables', 'default', 'include', 'workflow', 'image', 'services', 'before_script', 'after_script', 'cache', 'artifacts', 'pages', 'spec', 'inputs']);
|
|
15
|
+
|
|
16
|
+
// --- minimal GitLab rules:if evaluator ($VAR, strings, ==, !=, =~, !~, &&, ||, !, parens, null) ---
|
|
17
|
+
function evalRule(src, vars) {
|
|
18
|
+
const toks = []; let i = 0;
|
|
19
|
+
while (i < src.length) {
|
|
20
|
+
const c = src[i];
|
|
21
|
+
if (/\s/.test(c)) { i++; continue; }
|
|
22
|
+
if (c === '$') { const m = /^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?/.exec(src.slice(i)); if (!m) throw new Unsupported('rule syntax'); toks.push({ t: 'var', v: m[1] }); i += m[0].length; continue; }
|
|
23
|
+
if (c === '"' || c === "'") { const j = src.indexOf(c, i + 1); if (j < 0) throw new Unsupported('rule string'); toks.push({ t: 'str', v: src.slice(i + 1, j) }); i = j + 1; continue; }
|
|
24
|
+
if (c === '/') { const m = /^\/((?:\\.|[^/\\])*)\/([a-z]*)/.exec(src.slice(i)); if (!m) throw new Unsupported('rule regex'); toks.push({ t: 're', v: new RegExp(m[1], m[2].replace(/[^imsu]/g, '')) }); i += m[0].length; continue; }
|
|
25
|
+
const two = src.slice(i, i + 2);
|
|
26
|
+
if (['==', '!=', '=~', '!~', '&&', '||'].includes(two)) { toks.push({ t: 'op', v: two }); i += 2; continue; }
|
|
27
|
+
if ('()!'.includes(c)) { toks.push({ t: 'op', v: c }); i++; continue; }
|
|
28
|
+
if (/^null\b/.test(src.slice(i))) { toks.push({ t: 'null' }); i += 4; continue; }
|
|
29
|
+
throw new Unsupported('rule token ' + c);
|
|
30
|
+
}
|
|
31
|
+
let p = 0;
|
|
32
|
+
const val = (t) => (t.t === 'var' ? (t.v in vars ? vars[t.v] : (() => { throw new Unsupported('variable $' + t.v); })()) : t.t === 'null' ? null : t.t === 'str' ? t.v : t.t === 're' ? t.v : (() => { throw new Unsupported('rule operand'); })());
|
|
33
|
+
function prim() {
|
|
34
|
+
const t = toks[p++]; if (!t) throw new Unsupported('rule end');
|
|
35
|
+
if (t.t === 'op' && t.v === '(') { const v = or(); if (!(toks[p] && toks[p].v === ')')) throw new Unsupported('rule )'); p++; return v; }
|
|
36
|
+
if (t.t === 'op' && t.v === '!') return !prim();
|
|
37
|
+
const l = val(t); const n = toks[p];
|
|
38
|
+
if (n && n.t === 'op' && ['==', '!=', '=~', '!~'].includes(n.v)) {
|
|
39
|
+
p++; const r = val(toks[p++]);
|
|
40
|
+
if (n.v === '==') return (l === null ? null : l) === (r === null ? null : r);
|
|
41
|
+
if (n.v === '!=') return l !== r;
|
|
42
|
+
const hit = l !== null && r instanceof RegExp && r.test(String(l)); return n.v === '=~' ? hit : !hit;
|
|
43
|
+
}
|
|
44
|
+
return l !== null && l !== '';
|
|
45
|
+
}
|
|
46
|
+
function and() { let l = prim(); while (toks[p] && toks[p].v === '&&') { p++; const r = prim(); l = l && r; } return l; }
|
|
47
|
+
function or() { let l = and(); while (toks[p] && toks[p].v === '||') { p++; const r = and(); l = l || r; } return l; }
|
|
48
|
+
const v = or(); if (p < toks.length) throw new Unsupported('rule trailing'); return !!v;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function refMatches(ref, a) {
|
|
52
|
+
if (typeof ref !== 'string') return false;
|
|
53
|
+
if (ref === 'merge_requests' || ref === 'branches' || ref === 'pushes' || ref === 'external_pull_requests') return true;
|
|
54
|
+
if (['tags', 'schedules', 'api', 'triggers', 'web', 'pipelines', 'chat'].includes(ref)) return false;
|
|
55
|
+
const m = /^\/(.*)\/([a-z]*)$/.exec(ref);
|
|
56
|
+
return m ? new RegExp(m[1], m[2]).test(a.headBranch || '') : branchMatches(a.headBranch || '', [ref]);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function resolveExtends(name, jobs, depth = 0) {
|
|
60
|
+
const j = jobs[name]; if (!j || typeof j !== 'object') throw new Unsupported(`extends target ${name}`);
|
|
61
|
+
if (depth > 6) throw new Unsupported('extends depth');
|
|
62
|
+
let merged = {};
|
|
63
|
+
for (const e of [].concat(j.extends || [])) merged = deepMerge(merged, resolveExtends(e, jobs, depth + 1));
|
|
64
|
+
const own = { ...j }; delete own.extends;
|
|
65
|
+
return deepMerge(merged, own);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function loadWithIncludes(a, file, seen = new Set(), depth = 0) {
|
|
69
|
+
const doc = loadDoc(a, file, parseSafe);
|
|
70
|
+
if (doc === undefined) return undefined;
|
|
71
|
+
if (!doc || typeof doc !== 'object') return doc;
|
|
72
|
+
let merged = {};
|
|
73
|
+
const warn = [];
|
|
74
|
+
for (const inc of [].concat(doc.include || [])) {
|
|
75
|
+
const loc = typeof inc === 'string' ? inc : inc && inc.local;
|
|
76
|
+
if (!loc || /^https?:/.test(loc)) { warn.push(`include not resolved: ${typeof inc === 'string' ? inc : JSON.stringify(inc).slice(0, 60)}`); continue; }
|
|
77
|
+
const f = loc.replace(/^\//, '');
|
|
78
|
+
if (seen.has(f) || depth > 3) continue; seen.add(f);
|
|
79
|
+
const sub = loadWithIncludes(a, f, seen, depth + 1);
|
|
80
|
+
if (sub && typeof sub === 'object') { warn.push(...(sub.__warnings || [])); delete sub.__warnings; merged = deepMerge(merged, sub); }
|
|
81
|
+
}
|
|
82
|
+
const out = deepMerge(merged, doc); delete out.include; out.__warnings = warn; return out;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function plan(a) {
|
|
86
|
+
const out = []; const warnings = []; const used = [];
|
|
87
|
+
for (const file of a.defFiles(DEF_RE)) {
|
|
88
|
+
const doc = loadWithIncludes(a, file);
|
|
89
|
+
if (doc === undefined) { out.push(deferred(jobId('gitlab', 'pipeline'), 'gitlab-ci', file, 'pipeline YAML unreadable')); continue; }
|
|
90
|
+
if (!doc || typeof doc !== 'object') continue;
|
|
91
|
+
used.push(file);
|
|
92
|
+
for (const w of doc.__warnings || []) warnings.push(`${file}: ${w} (jobs from it are not in the plan)`);
|
|
93
|
+
const stages = Array.isArray(doc.stages) ? doc.stages : ['.pre', 'build', 'test', 'deploy', '.post'];
|
|
94
|
+
const gvars = vars(doc.variables); const defaults = doc.default || {};
|
|
95
|
+
const defined = new Set([...Object.keys(gvars)]);
|
|
96
|
+
const names = Object.keys(doc).filter((k) => !RESERVED.has(k) && !k.startsWith('.') && !k.startsWith('__') && doc[k] && typeof doc[k] === 'object' && !Array.isArray(doc[k]));
|
|
97
|
+
const results = {}; const byStage = {};
|
|
98
|
+
for (const name of names) {
|
|
99
|
+
const jid = jobId('gitlab', name);
|
|
100
|
+
let job;
|
|
101
|
+
try { job = doc[name].extends ? resolveExtends(name, doc) : doc[name]; } catch (e) { if (e instanceof Unsupported) { out.push(deferred(jid, 'gitlab-ci', name, e.message)); results[name] = 'deferred'; continue; } throw e; }
|
|
102
|
+
if (!job.script && !job.trigger && !job.run) continue;
|
|
103
|
+
const stage = job.stage || 'test';
|
|
104
|
+
const env = { ...gvars, ...vars(job.variables) };
|
|
105
|
+
const ctxVars = { CI_PIPELINE_SOURCE: 'merge_request_event', CI_MERGE_REQUEST_TARGET_BRANCH_NAME: a.baseBranch, CI_MERGE_REQUEST_SOURCE_BRANCH_NAME: a.headBranch || '', CI_COMMIT_BRANCH: null, CI_COMMIT_TAG: null, CI_COMMIT_REF_NAME: a.headBranch || '', CI_COMMIT_SHA: a.headSha, ...env };
|
|
106
|
+
const def = (r) => { out.push(deferred(jid, 'gitlab-ci', name, r)); results[name] = 'deferred'; };
|
|
107
|
+
const na = (r) => { out.push(notApplicable(jid, 'gitlab-ci', name, r)); results[name] = 'skipped'; };
|
|
108
|
+
try {
|
|
109
|
+
const gate = gating(job, ctxVars, a);
|
|
110
|
+
if (gate.na) { na(gate.na); continue; }
|
|
111
|
+
if (gate.defer) { def(gate.defer); continue; }
|
|
112
|
+
if (job.trigger) { def('trigger (child/multi-project pipeline)'); continue; }
|
|
113
|
+
if (job.environment) { def('deploy environment'); continue; }
|
|
114
|
+
if (job.secrets || job.id_tokens) { def('uses CI secrets / OIDC tokens'); continue; }
|
|
115
|
+
if (DEPLOY_NAME.test(name) || stage === 'deploy') { def('deploy/publish job'); continue; }
|
|
116
|
+
const services = job.services || defaults.services;
|
|
117
|
+
if (services && services.length) { def('service containers need docker network aliases (not reproduced on the host)'); continue; }
|
|
118
|
+
const lines = [].concat(defaults.before_script || [], job.before_script || [], job.script || []);
|
|
119
|
+
if (lines.some((l) => typeof l !== 'string')) { def('script uses !reference or nested lists'); continue; }
|
|
120
|
+
const sv = secretLikeVar(lines.join('\n'), defined) || secretLikeVar(JSON.stringify(env), defined);
|
|
121
|
+
if (/CI_JOB_TOKEN|CI_REGISTRY_PASSWORD/.test(lines.join('\n'))) { def('uses CI_JOB_TOKEN / registry credentials'); continue; }
|
|
122
|
+
if (sv) { def(`references credential-like variable $${sv}`); continue; }
|
|
123
|
+
const warn = []; const image = job.image || defaults.image;
|
|
124
|
+
if (image) warn.push(`image ${typeof image === 'string' ? image : image.name} ignored; running with the host toolchain`);
|
|
125
|
+
let matrix; let variants = 0;
|
|
126
|
+
if (job.parallel && typeof job.parallel === 'object' && job.parallel.matrix) {
|
|
127
|
+
const combos = job.parallel.matrix.flatMap((m) => Object.entries(m).reduce((acc, [k, v]) => acc.flatMap((c) => [].concat(v).map((x) => ({ ...c, [k]: String(x) }))), [{}]));
|
|
128
|
+
Object.assign(env, combos[0] || {}); matrix = Object.entries(combos[0] || {}).map(([k, v]) => `${k}=${v}`).join(','); variants = combos.length - 1;
|
|
129
|
+
}
|
|
130
|
+
Object.assign(env, { CI_PROJECT_DIR: a.repo, CI_COMMIT_SHA: a.headSha, CI_COMMIT_REF_NAME: a.headBranch || '', CI_MERGE_REQUEST_TARGET_BRANCH_NAME: a.baseBranch, CI_PIPELINE_SOURCE: 'merge_request_event', CI_JOB_NAME: name });
|
|
131
|
+
const r = stepsFromScript(lines.join('\n'), { name, cwd: '.', env: {}, repo: a.repo, continue_on_error: job.allow_failure === true, timeout_s: parseDuration(job.timeout) });
|
|
132
|
+
if (r.defer) { def(r.defer); continue; }
|
|
133
|
+
const needs = job.needs !== undefined ? [].concat(job.needs).map((n) => (typeof n === 'string' ? n : n && n.job)).filter(Boolean).map((n) => jobId('gitlab', n)) : null;
|
|
134
|
+
const j = baseJob({ id: jid, source: 'gitlab-ci', name, env: Object.fromEntries(Object.entries(env).map(([k, v]) => [k, String(v)])), steps: r.steps, warnings: warn, matrix, needs: needs || [], timeout_s: parseDuration(job.timeout) });
|
|
135
|
+
j._stage = stage; j._explicitNeeds = needs !== null;
|
|
136
|
+
(byStage[stage] = byStage[stage] || []).push(j);
|
|
137
|
+
results[name] = 'applicable'; out.push(j);
|
|
138
|
+
if (variants) out.push(deferred(jobId(jid, 'variants'), 'gitlab-ci', name, `${variants} parallel:matrix variant(s) not run locally`));
|
|
139
|
+
} catch (e) { if (e instanceof Unsupported) { def(`unsupported construct: ${e.message}`); continue; } throw e; }
|
|
140
|
+
}
|
|
141
|
+
// stage ordering: a job without explicit needs waits for every runnable job of earlier stages
|
|
142
|
+
for (const jobs of Object.values(byStage)) for (const j of jobs) {
|
|
143
|
+
if (j._explicitNeeds) { delete j._stage; delete j._explicitNeeds; continue; }
|
|
144
|
+
const idx = stages.indexOf(j._stage);
|
|
145
|
+
j.needs = Object.entries(byStage).filter(([s]) => stages.indexOf(s) < idx && stages.indexOf(s) >= 0).flatMap(([, js]) => js.map((x) => x.id));
|
|
146
|
+
delete j._stage; delete j._explicitNeeds;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return { jobs: out, warnings, used };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function vars(v) { const o = {}; for (const [k, x] of Object.entries(v || {})) o[k] = x && typeof x === 'object' ? String(x.value ?? '') : String(x ?? ''); return o; }
|
|
153
|
+
|
|
154
|
+
function changesOk(list, a) { const pats = globList(list && list.paths ? list.paths : list); return !a.changed.length || a.changed.some((f) => matchesFilter(f, pats)); }
|
|
155
|
+
|
|
156
|
+
// -> { na } | { defer } | {} (job is selected)
|
|
157
|
+
function gating(job, v, a) {
|
|
158
|
+
if (job.when === 'manual' || job.when === 'delayed') return { defer: `when: ${job.when}` };
|
|
159
|
+
if (job.when === 'never') return { na: 'when: never' };
|
|
160
|
+
if (Array.isArray(job.rules)) {
|
|
161
|
+
for (const r of job.rules) {
|
|
162
|
+
if (r.if !== undefined && !evalRule(String(r.if), v)) continue;
|
|
163
|
+
if (r.changes !== undefined && !changesOk(r.changes, a)) continue;
|
|
164
|
+
if (r.exists !== undefined) throw new Unsupported('rules:exists');
|
|
165
|
+
if (r.when === 'never') return { na: 'matched a when: never rule' };
|
|
166
|
+
if (r.when === 'manual' || r.when === 'delayed') return { defer: `rule when: ${r.when}` };
|
|
167
|
+
return {};
|
|
168
|
+
}
|
|
169
|
+
return { na: 'no rule selects this merge-request pipeline' };
|
|
170
|
+
}
|
|
171
|
+
if (job.only !== undefined) {
|
|
172
|
+
const o = Array.isArray(job.only) ? { refs: job.only } : job.only;
|
|
173
|
+
const refs = globList(o.refs || (Array.isArray(job.only) ? job.only : []));
|
|
174
|
+
if (refs.length && !refs.some((x) => refMatches(x, a))) return { na: 'only: refs do not select this pipeline' };
|
|
175
|
+
if (o.changes && !changesOk(o.changes, a)) return { na: 'only: changes selects none of the changed files' };
|
|
176
|
+
}
|
|
177
|
+
if (job.except !== undefined) {
|
|
178
|
+
const o = Array.isArray(job.except) ? { refs: job.except } : job.except;
|
|
179
|
+
const refs = globList(o.refs || (Array.isArray(job.except) ? job.except : []));
|
|
180
|
+
if (refs.some((x) => refMatches(x, a))) return { na: 'except: refs exclude this pipeline' };
|
|
181
|
+
if (o.changes && changesOk(o.changes, a)) return { na: 'except: changes excludes this pipeline' };
|
|
182
|
+
}
|
|
183
|
+
return {};
|
|
184
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
//
|
|
4
|
+
// portability.mjs - deterministic cross-platform checks that always run, because macOS/Windows CI jobs are deferred:
|
|
5
|
+
// Windows-invalid filename characters and reserved names, case-insensitive path collisions, CRLF in shell scripts,
|
|
6
|
+
// bash 3.2 constructs and GNU-only tool flags (added lines only), plus the repo's own check-shell-portability.sh
|
|
7
|
+
// when present. Findings are `PORTABILITY | <rule> | <file>[:line] | <detail>` lines.
|
|
8
|
+
|
|
9
|
+
import fs from 'node:fs';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
import { git, jobId } from '../lib.mjs';
|
|
12
|
+
import { baseJob } from './common.mjs';
|
|
13
|
+
|
|
14
|
+
export const id = 'portability';
|
|
15
|
+
const WIN_BAD = /[<>:"|?*\u0000-\u001f]/;
|
|
16
|
+
const WIN_RESERVED = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\.|$)/i;
|
|
17
|
+
const SH_EXT = /\.(sh|bash)$/;
|
|
18
|
+
|
|
19
|
+
const BASH32 = [
|
|
20
|
+
[/\b(mapfile|readarray)\b/, 'bash-4 builtin mapfile/readarray'],
|
|
21
|
+
[/\b(declare|local|typeset)\s+-[A-Za-z]*A\b/, 'bash-4 associative array'],
|
|
22
|
+
[/\$\{[A-Za-z_][A-Za-z0-9_]*(,,|\^\^)\}|\$\{[A-Za-z_][A-Za-z0-9_]*[,^]\}/, 'bash-4 case modification expansion'],
|
|
23
|
+
[/(^|[^|])\|&/, 'bash-4 |& redirection'],
|
|
24
|
+
[/&>>/, 'bash-4 &>> redirection'],
|
|
25
|
+
[/\bcoproc\b/, 'bash-4 coproc'],
|
|
26
|
+
[/\bwait\s+-n\b/, 'bash-4.3 wait -n'],
|
|
27
|
+
[/\[\[\s+-v\s/, 'bash-4.2 [[ -v ]]'],
|
|
28
|
+
[/\b(local|declare)\s+-n\b/, 'bash-4.3 namerefs'],
|
|
29
|
+
[/\$\{[A-Za-z_][A-Za-z0-9_]*@[AEPQUuLa]\}/, 'bash-4.4/5 parameter transformation'],
|
|
30
|
+
[/\bread\s+(-[A-Za-z]*i\b)/, 'bash-4 read -i'],
|
|
31
|
+
[/\$(EPOCHSECONDS|EPOCHREALTIME|SRANDOM|BASHPID)\b/, 'bash-4/5 special variable'],
|
|
32
|
+
[/;;&|;&(\s|$)/, 'bash-4 case fallthrough'],
|
|
33
|
+
[/\bshopt\s+-s\s+(globstar|lastpipe|inherit_errexit)\b/, 'bash-4 shopt option'],
|
|
34
|
+
];
|
|
35
|
+
const GNU = [
|
|
36
|
+
[/\bgrep\s+(-[A-Za-z]*P\b|--perl-regexp)/, 'grep -P (PCRE) is GNU-only'],
|
|
37
|
+
[/\bsed\s+-i\s+(?!['"])/, 'sed -i without a backup suffix argument is GNU-only'],
|
|
38
|
+
[/\bsed\s+(-[A-Za-z]*r\b|--regexp-extended)/, 'sed -r is GNU-only (use -E)'],
|
|
39
|
+
[/\bdate\s+(-d\b|--date)/, 'date -d is GNU-only'],
|
|
40
|
+
[/\breadlink\s+-f\b/, 'readlink -f is GNU-only'],
|
|
41
|
+
[/\bstat\s+-c\b/, 'stat -c is GNU-only'],
|
|
42
|
+
[/\bxargs\s+(-r\b|--no-run-if-empty)/, 'xargs -r is GNU-only'],
|
|
43
|
+
[/\bfind\b.*\s-printf\b/, 'find -printf is GNU-only'],
|
|
44
|
+
[/\bsort\s+(-[A-Za-z]*V\b|--version-sort)/, 'sort -V is not on older BSD sort'],
|
|
45
|
+
[/\bhead\s+-n\s*-[0-9]/, 'head -n -N is GNU-only'],
|
|
46
|
+
[/\bls\s+--color\b/, 'ls --color is GNU-only'],
|
|
47
|
+
[/\bcp\s+--parents\b/, 'cp --parents is GNU-only'],
|
|
48
|
+
];
|
|
49
|
+
|
|
50
|
+
export function plan(a) {
|
|
51
|
+
if (!a.changed.length) return { jobs: [], warnings: [], used: [] };
|
|
52
|
+
const steps = [{ name: 'portability checks (filenames, case collisions, CRLF, bash 3.2, GNU-only flags)', command: '(built-in portability checks)', cwd: '.', env: {}, kind: 'check', builtin: 'portability' }];
|
|
53
|
+
if (a.changed.some((f) => /^scripts\/.*\.sh$/.test(f)) && fs.existsSync(path.join(a.repo, 'scripts/check-shell-portability.sh')) && fs.existsSync(path.join(a.repo, 'scripts'))) {
|
|
54
|
+
steps.push({ name: 'repo check-shell-portability.sh', command: 'bash scripts/check-shell-portability.sh scripts', cwd: '.', env: {}, kind: 'check', prescoped: true });
|
|
55
|
+
}
|
|
56
|
+
return { jobs: [baseJob({ id: jobId('portability', 'builtin'), source: 'portability', name: 'portability checks', steps })], warnings: [], used: [] };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function addedLines(repo, from, f) {
|
|
60
|
+
const tracked = git(repo, ['ls-files', '--error-unmatch', '--', f]).code === 0;
|
|
61
|
+
if (!tracked) { try { return fs.readFileSync(path.join(repo, f), 'utf8').split('\n').map((t, i) => ({ n: i + 1, t })); } catch { return []; } }
|
|
62
|
+
const d = git(repo, ['diff', '-U0', '--no-color', '--no-ext-diff', from, '--', f]).out;
|
|
63
|
+
const out = []; let n = 0;
|
|
64
|
+
for (const l of d.split('\n')) {
|
|
65
|
+
const h = /^@@ -\d+(?:,\d+)? \+(\d+)/.exec(l);
|
|
66
|
+
if (h) { n = Number(h[1]); continue; }
|
|
67
|
+
if (l.startsWith('+') && !l.startsWith('+++')) { out.push({ n, t: l.slice(1) }); n++; }
|
|
68
|
+
}
|
|
69
|
+
return out;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Run the built-in checks; returns an array of finding strings.
|
|
73
|
+
export function checkPortability({ repo, from, changed, tracked }) {
|
|
74
|
+
const findings = [];
|
|
75
|
+
const lower = new Map();
|
|
76
|
+
for (const f of tracked) { const k = f.toLowerCase(); lower.set(k, (lower.get(k) || []).concat(f)); }
|
|
77
|
+
for (const f of changed) {
|
|
78
|
+
for (const seg of f.split('/')) {
|
|
79
|
+
if (WIN_BAD.test(seg)) { findings.push(`PORTABILITY | windows-filename | ${f} | invalid character in "${seg}"`); break; }
|
|
80
|
+
if (/[. ]$/.test(seg)) { findings.push(`PORTABILITY | windows-filename | ${f} | segment "${seg}" ends with a dot or space`); break; }
|
|
81
|
+
if (WIN_RESERVED.test(seg)) { findings.push(`PORTABILITY | windows-filename | ${f} | reserved device name "${seg}"`); break; }
|
|
82
|
+
}
|
|
83
|
+
if (f.length > 240) findings.push(`PORTABILITY | windows-path-length | ${f} | path longer than 240 characters`);
|
|
84
|
+
const same = lower.get(f.toLowerCase()) || [];
|
|
85
|
+
if (same.length > 1) findings.push(`PORTABILITY | case-collision | ${f} | collides with ${same.filter((x) => x !== f).join(', ')} on case-insensitive filesystems`);
|
|
86
|
+
const abs = path.join(repo, f);
|
|
87
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) continue;
|
|
88
|
+
let head = ''; try { head = fs.readFileSync(abs, 'utf8').slice(0, 200); } catch { continue; }
|
|
89
|
+
const isShell = SH_EXT.test(f) || /^#!.*\b(ba)?sh\b/.test(head);
|
|
90
|
+
if (!isShell && !/\.bats$/.test(f)) continue;
|
|
91
|
+
const lines = addedLines(repo, from, f);
|
|
92
|
+
for (const { n, t } of lines) {
|
|
93
|
+
if (/\r$/.test(t)) { findings.push(`PORTABILITY | crlf | ${f}:${n} | CRLF line ending in a shell script`); break; }
|
|
94
|
+
}
|
|
95
|
+
if (/\.bats$/.test(f)) continue;
|
|
96
|
+
for (const { n, t } of lines) {
|
|
97
|
+
const s = t.replace(/\r$/, '');
|
|
98
|
+
if (/^\s*#/.test(s) || /allowlist:/.test(s)) continue;
|
|
99
|
+
for (const [re, why] of BASH32) if (re.test(s)) findings.push(`PORTABILITY | bash-3.2 | ${f}:${n} | ${why}`);
|
|
100
|
+
for (const [re, why] of GNU) if (re.test(s)) findings.push(`PORTABILITY | gnu-only | ${f}:${n} | ${why}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return findings;
|
|
104
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
//
|
|
4
|
+
// task-runners.mjs - repos with no CI definition that gates the change: derive the check set from the repo's own
|
|
5
|
+
// task runners and tool configuration (package.json scripts, Makefile, justfile, Taskfile, tox/nox, pre-commit,
|
|
6
|
+
// pyproject tools, Cargo, Go, Maven, Gradle). Only lint/test/check style tasks; never build/deploy/publish.
|
|
7
|
+
|
|
8
|
+
import fs from 'node:fs';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import { jobId } from '../lib.mjs';
|
|
11
|
+
import { parseSafe, baseJob } from './common.mjs';
|
|
12
|
+
import { stepsFromScript } from './common.mjs';
|
|
13
|
+
|
|
14
|
+
export const id = 'task-runners';
|
|
15
|
+
const NODE_SCRIPTS = ['lint', 'test', 'typecheck', 'type-check', 'check', 'tsc', 'format:check', 'test:unit'];
|
|
16
|
+
const TASK_NAMES = ['test', 'lint', 'check', 'ci', 'verify'];
|
|
17
|
+
|
|
18
|
+
const read = (a, f) => { try { return fs.readFileSync(path.join(a.repo, f), 'utf8'); } catch { return null; } };
|
|
19
|
+
const has = (a, f) => fs.existsSync(path.join(a.repo, f));
|
|
20
|
+
|
|
21
|
+
function mk(a, kind, name, command, extra = {}) {
|
|
22
|
+
const r = stepsFromScript(command, { name, cwd: '.', env: {}, repo: a.repo });
|
|
23
|
+
const steps = [...(extra.pre || []), ...r.steps];
|
|
24
|
+
return baseJob({ id: jobId('task', kind, name), source: 'task-runners', name: `${kind}: ${name}`, steps, warnings: [] });
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function plan(a) {
|
|
28
|
+
const jobs = []; const used = [];
|
|
29
|
+
const add = (j) => jobs.push(j);
|
|
30
|
+
|
|
31
|
+
const pjText = read(a, 'package.json');
|
|
32
|
+
if (pjText) {
|
|
33
|
+
let pj = null; try { pj = JSON.parse(pjText); } catch { /* malformed */ }
|
|
34
|
+
const pm = has(a, 'pnpm-lock.yaml') ? 'pnpm' : has(a, 'yarn.lock') ? 'yarn' : (has(a, 'bun.lock') || has(a, 'bun.lockb')) ? 'bun' : 'npm';
|
|
35
|
+
if (pj && pj.scripts) {
|
|
36
|
+
used.push('package.json');
|
|
37
|
+
const install = !has(a, 'node_modules') ? [{ name: 'install dependencies', command: pm === 'npm' ? (has(a, 'package-lock.json') ? 'npm ci' : 'npm install') : `${pm} install`, cwd: '.', env: {}, kind: 'install' }] : [];
|
|
38
|
+
for (const s of NODE_SCRIPTS) if (typeof pj.scripts[s] === 'string' && !/^\s*echo\b.*(no test|not specified)/i.test(pj.scripts[s]) && !/Error: no test specified/.test(pj.scripts[s])) add(mk(a, pm, s, s === 'test' && pm === 'npm' ? 'npm test' : `${pm} run ${s}`, { pre: install }));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const mf = read(a, 'Makefile') || read(a, 'makefile') || read(a, 'GNUmakefile');
|
|
42
|
+
if (mf) { used.push('Makefile'); for (const t of TASK_NAMES) if (new RegExp(`^${t}\\s*:`, 'm').test(mf)) add(mk(a, 'make', t, `make ${t}`)); }
|
|
43
|
+
const jf = read(a, 'justfile') || read(a, 'Justfile');
|
|
44
|
+
if (jf) { used.push('justfile'); for (const t of TASK_NAMES) if (new RegExp(`^${t}\\b[^\\n:]*:`, 'm').test(jf)) add(mk(a, 'just', t, `just ${t}`)); }
|
|
45
|
+
const tf = read(a, 'Taskfile.yml') || read(a, 'Taskfile.yaml');
|
|
46
|
+
if (tf) { const d = parseSafe(tf); used.push('Taskfile'); if (d && d.tasks) for (const t of TASK_NAMES) if (d.tasks[t]) add(mk(a, 'task', t, `task ${t}`)); }
|
|
47
|
+
if (has(a, 'tox.ini')) { used.push('tox.ini'); add(mk(a, 'python', 'tox', 'tox -q')); }
|
|
48
|
+
if (has(a, 'noxfile.py')) { used.push('noxfile.py'); add(mk(a, 'python', 'nox', 'nox')); }
|
|
49
|
+
const py = read(a, 'pyproject.toml');
|
|
50
|
+
if (py) {
|
|
51
|
+
used.push('pyproject.toml');
|
|
52
|
+
if (/^\[tool\.ruff/m.test(py)) add(mk(a, 'python', 'ruff', 'ruff check .'));
|
|
53
|
+
if (/^\[tool\.black/m.test(py)) add(mk(a, 'python', 'black', 'black --check .'));
|
|
54
|
+
if (/^\[tool\.pytest/m.test(py) || has(a, 'pytest.ini')) add(mk(a, 'python', 'pytest', 'pytest'));
|
|
55
|
+
if (/^\[tool\.mypy/m.test(py)) add(mk(a, 'python', 'mypy', 'mypy .'));
|
|
56
|
+
} else if (has(a, 'pytest.ini')) add(mk(a, 'python', 'pytest', 'pytest'));
|
|
57
|
+
if (has(a, '.pre-commit-config.yaml')) {
|
|
58
|
+
used.push('.pre-commit-config.yaml');
|
|
59
|
+
const files = a.changed.filter((f) => has(a, f));
|
|
60
|
+
if (files.length) { const j = mk(a, 'pre-commit', 'run', 'pre-commit run --files ' + files.map((f) => `'${f.replace(/'/g, "'\\''")}'`).join(' ')); j.steps.forEach((s) => { s.prescoped = true; }); add(j); }
|
|
61
|
+
}
|
|
62
|
+
if (has(a, 'Cargo.toml')) { used.push('Cargo.toml'); add(mk(a, 'cargo', 'test', 'cargo test')); }
|
|
63
|
+
if (has(a, 'go.mod')) { used.push('go.mod'); add(mk(a, 'go', 'vet', 'go vet ./...')); add(mk(a, 'go', 'test', 'go test ./...')); }
|
|
64
|
+
if (has(a, 'pom.xml')) { used.push('pom.xml'); add(mk(a, 'maven', 'test', 'mvn -q -B test')); }
|
|
65
|
+
if (has(a, 'build.gradle') || has(a, 'build.gradle.kts')) { used.push('gradle'); add(mk(a, 'gradle', 'test', has(a, 'gradlew') ? './gradlew test' : 'gradle test')); }
|
|
66
|
+
return { jobs, warnings: [], used };
|
|
67
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
//
|
|
4
|
+
// cli.mjs - node entry behind scripts/ci-local.sh. Subcommands: plan, run.
|
|
5
|
+
// Exit codes: 0 pass, 1 a runnable job failed, 2 degraded or usage error (the gate could not run; never a pass).
|
|
6
|
+
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import { Degraded } from './lib.mjs';
|
|
9
|
+
import { buildPlan, summarizePlan } from './plan.mjs';
|
|
10
|
+
|
|
11
|
+
function parseArgs(argv) {
|
|
12
|
+
const o = { _: [] };
|
|
13
|
+
for (let i = 0; i < argv.length; i++) {
|
|
14
|
+
const t = argv[i];
|
|
15
|
+
const take = () => { if (i + 1 >= argv.length) throw new Error(`${t} needs a value`); return argv[++i]; };
|
|
16
|
+
if (t === '--repo') o.repo = take(); else if (t === '--base') o.base = take(); else if (t === '--base-branch') o.baseBranch = take();
|
|
17
|
+
else if (t === '--head-branch') o.headBranch = take(); else if (t === '--files') o.files = take().split(/[\n ]+/).filter(Boolean);
|
|
18
|
+
else if (t === '--files-from') o.files = fs.readFileSync(take(), 'utf8').split('\n').filter(Boolean);
|
|
19
|
+
else if (t === '--out') o.out = take(); else if (t === '--plan') o.planFile = take(); else if (t === '--no-cache') o.noCache = true;
|
|
20
|
+
else if (t === '--budget-minutes') o.budgetMinutes = Number(take()); else if (t === '--max-parallel') o.maxParallel = Number(take());
|
|
21
|
+
else if (t === '--json') o.json = true; else if (t === '--result-file') o.resultFile = take(); else if (t === '--iteration') o.iteration = Number(take());
|
|
22
|
+
else if (t === '--cache-dir') o.cacheDir = take(); else if (t === '--result-cache-dir') o.resultCacheDir = take();
|
|
23
|
+
else if (t.startsWith('-')) throw new Error(`unknown option ${t}`); else o._.push(t);
|
|
24
|
+
}
|
|
25
|
+
return o;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function main() {
|
|
29
|
+
const [sub, ...rest] = process.argv.slice(2);
|
|
30
|
+
const o = parseArgs(rest);
|
|
31
|
+
if (!['plan', 'run'].includes(sub)) { console.error('usage: ci-local.sh plan|run --repo PATH --base REF [--files "a b"] [--out FILE]'); return 2; }
|
|
32
|
+
if (!o.repo) { console.error('ci-local: --repo is required'); return 2; }
|
|
33
|
+
let plan;
|
|
34
|
+
if (sub === 'run' && o.planFile) plan = JSON.parse(fs.readFileSync(o.planFile, 'utf8'));
|
|
35
|
+
else plan = buildPlan(o);
|
|
36
|
+
if (sub === 'plan') {
|
|
37
|
+
if (o.out) fs.writeFileSync(o.out, JSON.stringify(plan, null, 2));
|
|
38
|
+
if (o.json) console.log(JSON.stringify(plan, null, 2)); else for (const l of summarizePlan(plan)) console.log(l);
|
|
39
|
+
return 0;
|
|
40
|
+
}
|
|
41
|
+
const { runPlan } = await import('./run.mjs');
|
|
42
|
+
return runPlan(plan, o);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// process.exitCode (not process.exit) so a large stdout pipe is flushed before the process ends.
|
|
46
|
+
main().then((c) => { process.exitCode = c; }, (e) => {
|
|
47
|
+
const msg = String((e && e.message) || e).split('\n')[0];
|
|
48
|
+
if (!(e instanceof Degraded)) console.error(`ci-local: ${msg}`);
|
|
49
|
+
console.log(`CI_LOCAL_RESULT: DEGRADED reason=${msg}`); process.exitCode = 2;
|
|
50
|
+
});
|