forgedock 1.35.0 → 1.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -324
- package/bin/engine/events.mjs +152 -0
- package/bin/engine/events.test.mjs +128 -0
- package/bin/engine/telemetry-seam.mjs +4 -3
- package/bin/engine-cli.mjs +113 -18
- package/bin/experiment.mjs +55 -0
- package/bin/forgedock.mjs +1 -1
- package/bin/hooks/reread-nudge.mjs +15 -4
- package/bin/lean/batch.mjs +42 -2
- package/bin/lean/batch.test.mjs +101 -1
- package/bin/lean/conformance.mjs +14 -3
- package/bin/lean/conformance.test.mjs +26 -0
- package/bin/lean/effects.mjs +3 -0
- package/bin/lean/execution.mjs +2 -0
- package/bin/lean/i18n.mjs +65 -0
- package/bin/lean/i18n.test.mjs +47 -0
- package/bin/lean/lane.mjs +5 -2
- package/bin/lean/lean.mjs +37 -4
- package/bin/lean/lean.test.mjs +124 -5
- package/bin/lean/orchestrate.mjs +49 -5
- package/bin/lean/postdeploy-check.mjs +149 -0
- package/bin/lean/postdeploy-check.test.mjs +146 -0
- package/bin/lean/postdeploy-monitor.mjs +87 -0
- package/bin/lean/postdeploy-monitor.test.mjs +121 -0
- package/bin/lean/precheck.mjs +6 -4
- package/bin/lean/precheck.test.mjs +17 -2
- package/bin/lean/prefilter.mjs +29 -4
- package/bin/lean/prefilter.test.mjs +13 -5
- package/bin/lean/probe.mjs +2 -1
- package/bin/lean/prompts.mjs +10 -2
- package/bin/lean/sandbox-provider.contract.test.mjs +28 -0
- package/bin/lean/sandbox-remote.mjs +7 -0
- package/bin/lean/schedd.mjs +24 -4
- package/bin/lean/schedd.test.mjs +54 -0
- package/bin/stats/aggregate.mjs +1 -1
- package/bin/stats/experiment.mjs +44 -0
- package/bin/stats/github-source.mjs +2 -1
- package/bin/stats/index.mjs +42 -2
- package/bin/stats/release.mjs +75 -0
- package/bin/stats/release.test.mjs +68 -0
- package/commands/orchestrate.md +8 -3
- package/docs/CONFIG.md +49 -0
- package/docs/EVENTS.md +68 -0
- package/docs/STATS.md +31 -0
- package/docs/TELEMETRY.md +3 -1
- package/docs/site/install.md +89 -0
- package/package.json +1 -1
- package/scripts/bench-lightlane.mjs +2 -1
- package/scripts/eval-gate-scorecard.mjs +7 -0
- package/scripts/file-deferred-followups.sh +1 -0
- package/scripts/file-deferred-followups.test.sh +8 -0
package/bin/lean/lean.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { createHash } from "node:crypto";
|
|
|
16
16
|
import { TRACKED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, applyAmendments, precheck, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, pendingTracking, unnamedEdges, structureNote } from "./conformance.mjs";
|
|
17
17
|
import { checkAdmission, hostParallelism, ADMISSION_RETRY_SECONDS } from "./admission.mjs";
|
|
18
18
|
import { namedPaths, isTrustedIssueComment } from "./prefilter.mjs";
|
|
19
|
+
import { checksAt } from "./postdeploy-check.mjs";
|
|
19
20
|
import { preflightProbes, preflightProblems, unsetEnvVars, runPreflight, knownToContract } from "./preflight.mjs";
|
|
20
21
|
import { resolveLane, isDefaultLane, isShippedLane, DEFAULT_STAGING_BRANCH } from "./lane.mjs";
|
|
21
22
|
import { splitGate, rationaleLine } from "./decompose.mjs";
|
|
@@ -30,7 +31,7 @@ import { renderInvestigation, renderImplementation, renderReview, renderPrBody,
|
|
|
30
31
|
import { parseVerifierConfig, pickVerifierModel, changedSymbols, callersOf, verifierBudget, verifierPrompt, parseVerifierFindings, findingsToFailures, renderVerifierNote, VERIFIER_SYSTEM } from "./verifier.mjs";
|
|
31
32
|
import { computeTrigger, riskPathHits, TRIGGER_MODES } from "./review-trigger.mjs";
|
|
32
33
|
import { hostCapabilityBlock } from "./execution.mjs";
|
|
33
|
-
import { REVIEWER_SYSTEM, LEAN_DIR, RESULT_FILE, INVESTIGATION_FILE, IMPLEMENTATION_FILE, CONTRACT_FILE, builderPlanPrompt, builderImplementPrompt, contractCorrectionPrompt, conformanceResumePrompt, reviewerPrompt, reviewerRepairPrompt, REVIEW_DIFF_MAX_BYTES, blockCorrectionPrompt, HUMAN_BLOCK_REASONS, structurePrompt, hotspotPlanNote, hotspotBuildRule, reproPlanNote, contextPackNote, docsPrompt, ciFailurePrompt, ciUnavailablePrompt, precheckFailurePrompt, contextRestartPrompt, turnBudgetWrapupPrompt, syncChecks, GROUPING_SYSTEM, enumeratingPlanNote, enumeratingBuildRule, invariantsPlanNote, invariantsBuildRule, missingEntriesPrompt, sequencePrompt, sequenceSyncNote, fixSessionPrompt, postMergeFailurePrompt, duplicateExportsPrompt, migrationGeneratorPrompt } from "./prompts.mjs";
|
|
34
|
+
import { REVIEWER_SYSTEM, LEAN_DIR, RESULT_FILE, INVESTIGATION_FILE, IMPLEMENTATION_FILE, CONTRACT_FILE, builderPlanPrompt, builderImplementPrompt, contractCorrectionPrompt, conformanceResumePrompt, reviewerPrompt, reviewerRepairPrompt, REVIEW_DIFF_MAX_BYTES, blockCorrectionPrompt, HUMAN_BLOCK_REASONS, structurePrompt, hotspotPlanNote, hotspotBuildRule, reproPlanNote, contextPackNote, docsPrompt, ciFailurePrompt, ciUnavailablePrompt, precheckFailurePrompt, contextRestartPrompt, turnBudgetWrapupPrompt, syncChecks, GROUPING_SYSTEM, enumeratingPlanNote, enumeratingBuildRule, invariantsPlanNote, invariantsBuildRule, missingEntriesPrompt, sequencePrompt, sequenceSyncNote, fixSessionPrompt, postMergeFailurePrompt, duplicateExportsPrompt, migrationGeneratorPrompt, i18nCopiesPrompt } from "./prompts.mjs";
|
|
34
35
|
import { postMergeCommands } from "./postmerge.mjs";
|
|
35
36
|
import { freshFix, issueSummary, firstPassOf, sessionCostDelta } from "./fixround.mjs";
|
|
36
37
|
import { parseBuilderRouting, routeBuilder, shouldEscalate } from "./routing.mjs";
|
|
@@ -44,6 +45,7 @@ import { laneBackendId } from "./limits.mjs";
|
|
|
44
45
|
import { SPLIT_LABEL } from "./refactor.mjs";
|
|
45
46
|
import { writeHeavyShims, sumWaits, isHeavyProcess } from "./heavy.mjs";
|
|
46
47
|
import { priorityFromLabels } from "./sched.mjs";
|
|
48
|
+
import { assignArm } from "../experiment.mjs";
|
|
47
49
|
import { parseExploreConfig, explorerAgents, explorePlanRule, exploreBuildRule } from "./explore.mjs";
|
|
48
50
|
import { normaliseFacts, renderFactsRecord, FACTS_MARKER } from "./facts.mjs";
|
|
49
51
|
import { parseContextPackConfig, contextArm, contextPackFor, renderContextRecord, laneType, priorAttempts, linkStateStamp, stateOf } from "./context.mjs";
|
|
@@ -221,6 +223,7 @@ export async function runCore(o) {
|
|
|
221
223
|
let sessionId = null;
|
|
222
224
|
|
|
223
225
|
const ev = (e) => appendEvent(dir, issue, e, { repo }); // #4432: the run log is keyed by repo + issue
|
|
226
|
+
const expTag = o.experiment ? { experiment: o.experiment.id, arm: assignArm(o.experiment, issue) } : {}; // #4629: same arm for the whole run
|
|
224
227
|
const emit = async (rec) => { try { await emitTelemetry?.({ ...rec, issue }); } catch { /* never fails a run */ } };
|
|
225
228
|
let ghCache = null; // #4507: the run's own gh HTTP cache (set once the run tmp dir exists)
|
|
226
229
|
const ghc = (args) => ghCacheRetry((env) => io.gh(args, env ? { env } : undefined), ghCache);
|
|
@@ -238,7 +241,7 @@ export async function runCore(o) {
|
|
|
238
241
|
const c = sessionCostDelta(sessionCosts, { resume, sessionId: r?.sessionId ?? null, cost: num(r?.costUsd) });
|
|
239
242
|
const rec = { event: "PHASE_ATTEMPT_USAGE", phase, attempt: n, trigger, ...(restartOf ? { restart_of: restartOf } : {}), usage: r?.usage ?? null, duration_ms: num(r?.durationMs) ?? 0,
|
|
240
243
|
cost_usd: num(r?.costUsd), cost_usd_delta: c.delta, cost_usd_session: c.session, model: r?.model ?? null, turns: num(r?.turns), ...(mode ? { mode } : {}),
|
|
241
|
-
...(num(r?.baselineTokens) != null ? { baseline_tokens: num(r.baselineTokens) } : {}), ...(r?.tools ? { tools: r.tools } : {}), lane_profile: laneProfile.name }; // #4410
|
|
244
|
+
...(num(r?.baselineTokens) != null ? { baseline_tokens: num(r.baselineTokens) } : {}), ...(r?.tools ? { tools: r.tools } : {}), lane_profile: laneProfile.name, ...expTag }; // #4410, #4629
|
|
242
245
|
if (phase === "implement") { builderTriggers.push({ trigger, restart_of: restartOf }); impl.cost += c.delta ?? 0; if (r?.model && r.model !== "default") impl.models.add(r.model); tb.note(num(r?.turns), stage === "implement"); }
|
|
243
246
|
ev(rec); await emit(rec);
|
|
244
247
|
modelMs += rec.duration_ms; // #4451: model time next to the lane's wall time
|
|
@@ -417,7 +420,7 @@ export async function runCore(o) {
|
|
|
417
420
|
// forge#4165: pr, milestone and the conformance verdict ride on the existing RUN_TERMINAL write (verdict-mix queries).
|
|
418
421
|
const term = { event: "RUN_TERMINAL", reason, code, ...(extra.pr ? { pr: extra.pr } : {}), ...(milestone ? { milestone } : {}),
|
|
419
422
|
conformance_mode: conformance === "enforce" || riskEnforced ? "enforce" : "shadow", ...(shadowWould ? { shadow_would: shadowWould } : {}),
|
|
420
|
-
context_pack: ctxCfg.mode, ...ctxAb, explore: exploreCfg.mode, wall_ms: now() - t0, model_ms: modelMs, light, ...(ctxLane ? { lane_type: ctxLane } : {}), ...(pitMatched != null ? { pitfalls_matched: pitMatched } : {}), ...(tb.record() ? { turns: tb.record() } : {}),
|
|
423
|
+
context_pack: ctxCfg.mode, ...ctxAb, ...expTag, explore: exploreCfg.mode, wall_ms: now() - t0, model_ms: modelMs, light, ...(ctxLane ? { lane_type: ctxLane } : {}), ...(pitMatched != null ? { pitfalls_matched: pitMatched } : {}), ...(tb.record() ? { turns: tb.record() } : {}),
|
|
421
424
|
...(route ? { builder_tier: route.tier, builder_class: route.class, builder_escalated: escalated, builder_model: route.model ?? "default" } : {}) }; // forge#4275: the on/off A/B groups runs by these; #4394: per-tier stats
|
|
422
425
|
ev(term);
|
|
423
426
|
await emit(term);
|
|
@@ -910,6 +913,7 @@ export async function runCore(o) {
|
|
|
910
913
|
}
|
|
911
914
|
ghCache = io.ghCache?.(runTmp) ?? null;
|
|
912
915
|
ev({ event: "RUN_START", run: `core_${issue}_${t0}`, issue, lane, path: "core", version: ENGINE_VERSION, forge_version: ENGINE_VERSION, repo });
|
|
916
|
+
await emit({ event: "RUN_START", run: `core_${issue}_${t0}`, lane, path: "core" }); // #4630: lifecycle event stream
|
|
913
917
|
try {
|
|
914
918
|
iss = JSON.parse(await ghc(["issue", "view", String(issue), "-R", repo, "--json", "number,title,body,state,comments,labels,updatedAt"]));
|
|
915
919
|
} catch (e) { return finish("engine-error", `cannot read issue: ${e.message}`); }
|
|
@@ -1426,6 +1430,7 @@ export async function runCore(o) {
|
|
|
1426
1430
|
if (body.length > MAX_COMMENT_CHARS - 500) body = renderContract(frozen, hash, { compact: true, scope: scopeHash });
|
|
1427
1431
|
await post(["FORGE:CONTRACT", "FORGE:PLAN"], body);
|
|
1428
1432
|
ev({ event: "CONTRACT_FROZEN", hash, requirements: frozen.requirements.length, deferred: 0, derived: "light" });
|
|
1433
|
+
await emit({ event: "CONTRACT_FROZEN", hash, requirements: frozen.requirements.length, deferred: 0, derived: "light" });
|
|
1429
1434
|
ev({ event: "LIGHT_DONE", files: lr.files.length, test_failures: lr.testFailures, repro: lr.repro, cost_usd: usage.cost_usd });
|
|
1430
1435
|
T.decisions.push(`Light lane: one builder session, contract ${hash} derived from the diff and the test`);
|
|
1431
1436
|
try { await post("FORGE:LITE", renderLiteReceipt({ lr, eligibility: elig, costUsd: usage.cost_usd, contractHash: hash })); } catch { /* #4452: the receipt is best effort */ }
|
|
@@ -1507,6 +1512,7 @@ export async function runCore(o) {
|
|
|
1507
1512
|
await post(["FORGE:CONTRACT", "FORGE:PLAN"], body);
|
|
1508
1513
|
const deferredN = frozen.requirements.filter((r) => TRACKED.includes(r.disposition)).length;
|
|
1509
1514
|
ev({ event: "CONTRACT_FROZEN", hash, requirements: frozen.requirements.length, deferred: deferredN });
|
|
1515
|
+
await emit({ event: "CONTRACT_FROZEN", hash, requirements: frozen.requirements.length, deferred: deferredN });
|
|
1510
1516
|
T.decisions.push(`Contract frozen (${hash}): ${frozen.requirements.length} requirement(s), ${deferredN} deferred pending review`);
|
|
1511
1517
|
} else {
|
|
1512
1518
|
T.validate = lightDone ? "light lane (no plan session)" : "resumed (existing branch)";
|
|
@@ -1748,6 +1754,28 @@ export async function runCore(o) {
|
|
|
1748
1754
|
else await gatePassed("sequence");
|
|
1749
1755
|
}
|
|
1750
1756
|
await migrationGate("hand-off");
|
|
1757
|
+
/** #4596: keys added to a non-en locale catalog as English copies: one translate round, then an anomaly. True when a fix round ran. */
|
|
1758
|
+
async function i18nGate(trigger) {
|
|
1759
|
+
if (!io.englishCopies || isSplit) return false;
|
|
1760
|
+
const found = async () => (await io.englishCopies({ cwd: wt, base }).catch(() => null)) ?? [];
|
|
1761
|
+
let cp = await found();
|
|
1762
|
+
const brief = () => cp.slice(0, 5).map((d) => `${d.path}:${d.key}`).join("; ");
|
|
1763
|
+
if (cp.length && sessionId) {
|
|
1764
|
+
ev({ event: "I18N_ENGLISH_COPIES", trigger, count: cp.length });
|
|
1765
|
+
T.decisions.push(`English copies in locale catalogs (${trigger}): ${brief()}; builder resumed once`);
|
|
1766
|
+
await fixRound("duplicate", i18nCopiesPrompt(cp, vcfg.mode), [], "duplicate", cp.map((d) => `english copy \`${d.key}\` in ${d.path}`).join("\n"));
|
|
1767
|
+
result = (await readResult()) ?? result;
|
|
1768
|
+
files = await changedFiles();
|
|
1769
|
+
const before = cp;
|
|
1770
|
+
cp = await found();
|
|
1771
|
+
if (cp.length) T.anomalies.push(`English copies remain in locale catalogs (${trigger}): ${brief()}`);
|
|
1772
|
+
else await gatePassed("i18n");
|
|
1773
|
+
return before.length > 0;
|
|
1774
|
+
}
|
|
1775
|
+
if (cp.length) T.anomalies.push(`English copies in locale catalogs (${trigger}): ${brief()}`);
|
|
1776
|
+
return false;
|
|
1777
|
+
}
|
|
1778
|
+
await i18nGate("hand-off");
|
|
1751
1779
|
/** #4406: verify.post_merge (else the scoped typecheck) on the merged tree before the base-sync push; fix rounds up to verify.max_rounds. */
|
|
1752
1780
|
async function postMergeGate(trigger) {
|
|
1753
1781
|
const commands = io.runCommand && !isSplit ? postMergeCommands({ configured: vcfg.postMerge, scoped: (await io.postMergeChecks?.({ cwd: wt, base }).catch(() => [])) ?? [], mode: vcfg.mode }) : [];
|
|
@@ -1786,6 +1814,7 @@ export async function runCore(o) {
|
|
|
1786
1814
|
async function ciVerify(trigger) {
|
|
1787
1815
|
let fixes = 0;
|
|
1788
1816
|
for (;;) {
|
|
1817
|
+
await untrackToolState({ git, base, ev }); // #4621: gitPush's untrack commit would move HEAD after `head` was read; do it first so head is the pushed tip
|
|
1789
1818
|
const head = String(await git(["rev-parse", "HEAD"])).trim();
|
|
1790
1819
|
if (head === ciVerifiedHead || ciFellBack) return { fixes };
|
|
1791
1820
|
files = await changedFiles();
|
|
@@ -1998,7 +2027,9 @@ export async function runCore(o) {
|
|
|
1998
2027
|
const hotspots = await hotspotsOf({ base, paths: files }); // #4502
|
|
1999
2028
|
const validateRuns = await rerunFailedValidates(effective);
|
|
2000
2029
|
const repoHas = validateRuns && Object.keys(validateRuns).length ? repoTracks(String(await git(["ls-files"]).catch(() => "")).split("\n").map((x) => x.trim()).filter(Boolean)) : undefined;
|
|
2001
|
-
|
|
2030
|
+
// #4626: post-deploy check ids resolve against the lane base's checks file or this PR's
|
|
2031
|
+
const checkIds = effective.requirements?.some((r) => r.disposition === "post-deploy") ? new Set(Object.keys(await checksAt(git, [base, "HEAD"], wt))) : null;
|
|
2032
|
+
const pc = precheck({ contract: effective, result, changedFiles: files, ignore: IGNORE, ciVerified, hotspots, repro: await reproOf(effective), validateRuns, repoHas, frozen, checkIds });
|
|
2002
2033
|
// #4350: a scope-check validate stale only by amended deliverables is judged by the engine's scope gate instead
|
|
2003
2034
|
if (pc.superseded?.length) {
|
|
2004
2035
|
ev({ event: "VALIDATE_SUPERSEDED", runs: pc.superseded });
|
|
@@ -2170,6 +2201,7 @@ export async function runCore(o) {
|
|
|
2170
2201
|
const pr = Number(/\/pull\/(\d+)/.exec(prOut)?.[1] ?? /(\d+)\s*$/.exec(prOut)?.[1]);
|
|
2171
2202
|
if (!Number.isInteger(pr)) return await needsHuman(`Could not parse the PR number from: ${prOut.slice(0, 200)}`);
|
|
2172
2203
|
ev({ event: "PR_OPENED", pr });
|
|
2204
|
+
await emit({ event: "PR_OPENED", pr });
|
|
2173
2205
|
if (resumed?.pr && closing.tracked.length) { // an existing PR still has to link the tracking issues visibly
|
|
2174
2206
|
try { await ghw(["pr", "edit", String(pr), "-R", repo, "--body", renderPrBody({ issue, investigation: await readStage(STAGES[0]), contract: effective, implementation: await readStage(STAGES[1]), result, gitFiles: files, tracked: closing.tracked })]); }
|
|
2175
2207
|
catch (e) { e.deferCode = OUTCOME_CODES.FOLLOWUP_UNAVAILABLE; e.deferDetail = `could not link the tracking issues on PR #${pr}; nothing merged, rerun to retry`; throw e; }
|
|
@@ -2290,6 +2322,7 @@ export async function runCore(o) {
|
|
|
2290
2322
|
if (pm.fixed) how += "; builder fixed post-merge check failures";
|
|
2291
2323
|
dupFixed = await duplicateGate(`base-sync ${syncs}`);
|
|
2292
2324
|
if (dupFixed) how += "; builder resolved duplicate exports";
|
|
2325
|
+
if (await i18nGate(`base-sync ${syncs}`)) how += "; builder translated English-copy keys";
|
|
2293
2326
|
}
|
|
2294
2327
|
const after = await gitOut(["rev-parse", "HEAD"]);
|
|
2295
2328
|
if (soft && clean && after === before) { syncs--; sameBaseSyncs--; return null; } // UNKNOWN state and nothing to merge: not a conflict
|
package/bin/lean/lean.test.mjs
CHANGED
|
@@ -124,6 +124,8 @@ function world(o = {}) {
|
|
|
124
124
|
if (a[0] === "diff" && a[1] === "--check") return "";
|
|
125
125
|
if (a[0] === "grep") return w.markers ? "src/foo.js:3:<<<<<<< HEAD" : "";
|
|
126
126
|
if (a[0] === "merge" && a[1] === "--abort") { w.mergeInProgress = false; return ""; }
|
|
127
|
+
if (a[0] === "rev-parse" && a.length === 2 && (o.shipped ?? []).includes(String(a[1]).replace(/^origin\//, ""))) return `tip-${a[1]}`; // #4623
|
|
128
|
+
if (a[0] === "rev-list" && a.includes("--merges")) { const l = String(a.at(-1)).slice(1).replace(/^origin\//, ""); return (o.shipped ?? []).includes(l) ? `m1 s1 tip-origin/${l}\n` : ""; }
|
|
127
129
|
if (a[0] === "rev-parse" && a.length === 2 && /^origin\//.test(String(a[1]))) return w.baseHeads.length > 1 ? w.baseHeads.shift() : w.baseHeads[0]; // #4386: the base head moves only when a test says so
|
|
128
130
|
if (a[0] === "rev-parse") return w.head;
|
|
129
131
|
if (a[0] === "diff" && a[1] === "--name-only" && !a.some((x) => String(x).includes("..."))) return w.changed.filter((f) => !(o.equalToBase ?? []).includes(f)).join("\n");
|
|
@@ -131,7 +133,7 @@ function world(o = {}) {
|
|
|
131
133
|
if (a[0] === "diff" && a[1] === "--numstat") return w.numstat ?? "";
|
|
132
134
|
if (a[0] === "ls-files") return [...new Set(["src/foo.js", "src/bar.js", "test/foo.test.js", ...w.changed])].join("\n");
|
|
133
135
|
if (a[0] === "diff") return "diff --git a/src/foo.js b/src/foo.js\n+fix";
|
|
134
|
-
if (a[0] === "merge-base" && a[1] === "--is-ancestor") { if ((o.shipped ?? []).includes(String(a[2]).replace(/^origin\//, ""))) return ""; throw new Error("not an ancestor"); } // #4593
|
|
136
|
+
if (a[0] === "merge-base" && a[1] === "--is-ancestor") { if ([...(o.shipped ?? []), ...(o.unusedAncestors ?? [])].includes(String(a[2]).replace(/^origin\//, ""))) return ""; throw new Error("not an ancestor"); } // #4593
|
|
135
137
|
if (a[0] === "ls-remote" && String(a.at(-1)).startsWith("refs/heads/")) { const b = a.at(-1).slice("refs/heads/".length); return (o.branchesOnOrigin ?? ["staging", "main"]).includes(b) ? `abc\trefs/heads/${b}\n` : ""; }
|
|
136
138
|
if (a[0] === "ls-remote") return w.remote;
|
|
137
139
|
if (a[0] === "show" && w.baseFiles) { if (!(a[1] in w.baseFiles)) throw new Error(`fatal: path not in ${a[1]}`); return w.baseFiles[a[1]]; }
|
|
@@ -794,8 +796,9 @@ describe("conformance gate (frozen contract, precheck, structured review)", () =
|
|
|
794
796
|
|
|
795
797
|
for (const d of ["defer", "post-deploy"]) {
|
|
796
798
|
it(`a contracted ${d} is filed only after the reviewer accepts it, then linked in the PR and on close`, async () => {
|
|
797
|
-
const k = two(d, { why: d === "defer" ? "separate module" : "only measurable after deploy" });
|
|
799
|
+
const k = two(d, { why: d === "defer" ? "separate module" : "only measurable after deploy", ...(d === "post-deploy" ? { check: "login-smoke" } : {}) });
|
|
798
800
|
const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, { ...k, requirements: [k.requirements[0], { ...k.requirements[1], tracked: "#900" }] })] });
|
|
801
|
+
if (d === "post-deploy") w.baseFiles = { "origin/staging:.forge/post-deploy-checks.json": JSON.stringify({ "login-smoke": "true" }) };
|
|
799
802
|
const r = await run(w);
|
|
800
803
|
assert.equal(r.terminalReason, "merged");
|
|
801
804
|
assert.equal(w.followups.length, 1); assert.equal(w.followups[0].parent, 7);
|
|
@@ -1623,7 +1626,7 @@ describe("hotspot prevention (forge#4195): plan facts, build rule, unexempted gr
|
|
|
1623
1626
|
const plan = builderPlanPrompt(7) + hotspotPlanNote(facts([HOT, { ...HOT, path: "a/b.py" }, { ...HOT, path: "c/d.py" }]));
|
|
1624
1627
|
// 1850: #4404 (verify out_of_band) and #4396 (split size rule) grew the plan prompt to ~1803 tokens on CI paths; 1900: #4589 (check timeout is not a block, one block-rule line).
|
|
1625
1628
|
assert.ok(approxTokens(plan) <= 1900, `plan prompt with hotspots is ${approxTokens(plan)} tokens`);
|
|
1626
|
-
assert.ok(approxTokens(builderImplementPrompt() + hotspotBuildRule(facts([]))) <=
|
|
1629
|
+
assert.ok(approxTokens(builderImplementPrompt() + hotspotBuildRule(facts([]))) <= 1600); // 1600: #4591 (never hand-name migration files) and #4596 (i18n translate rule), one line each
|
|
1627
1630
|
});
|
|
1628
1631
|
|
|
1629
1632
|
it("#4519 generated_paths: both prompts name the globs (even with the gate off), and an amendment adding one blocks", async () => {
|
|
@@ -2041,7 +2044,7 @@ describe("comment templates", () => {
|
|
|
2041
2044
|
it("plan and implement prompts carry the stage templates and the contract schema within budget", () => {
|
|
2042
2045
|
const plan = builderPlanPrompt(7), impl = builderImplementPrompt();
|
|
2043
2046
|
assert.ok(approxTokens(plan) <= 1800, `plan prompt is ${approxTokens(plan)} tokens`);
|
|
2044
|
-
assert.ok(approxTokens(impl) <=
|
|
2047
|
+
assert.ok(approxTokens(impl) <= 1550, `implement prompt is ${approxTokens(impl)} tokens`); // 1550: #4596 i18n translate rule
|
|
2045
2048
|
for (const h of ["### What Was Claimed", "### What We Found", "### Root Cause", "### Affected Files", "### Sibling Sweep", "### History", "### Recommendation", "### Related Issues",
|
|
2046
2049
|
"investigation.md", "contract.json", "PLANNING ONLY", '"requirements"', '"deliverables"', '"preserve"', '"validate"', "verbatim", "defer", "post-deploy"]) assert.ok(plan.includes(h), h);
|
|
2047
2050
|
for (const h of ["### Approach", "### Changes", "### Acceptance Criteria Status", "### Verification", "### Notes", "implementation.md", "result.json",
|
|
@@ -2074,6 +2077,83 @@ describe("merge_wait policy", () => {
|
|
|
2074
2077
|
const w2 = world(); w2.io.requiredChecks = async () => ["build"]; w2.io.checkStates = async () => states({ build: "pending" });
|
|
2075
2078
|
assert.equal((await run(w2, { ciMaxMinutes: 5 })).terminalReason, "ci-pending");
|
|
2076
2079
|
});
|
|
2080
|
+
describe("#4624 merge-time rerun / wait / flaky branch of mergeFixRound", () => {
|
|
2081
|
+
const RUN = (n, attempt) => `https://github.com/o/r/actions/runs/${n}/job/${attempt}`;
|
|
2082
|
+
const NO_FIX = { ...parseVerify({}), maxRounds: 0 };
|
|
2083
|
+
const WAIT = 20000;
|
|
2084
|
+
const waits = (w) => { const n = { n: 0 }; const s = w.io.sleep; w.io.sleep = async (ms) => { if (ms === WAIT) n.n++; return s(ms); }; return n; };
|
|
2085
|
+
const fixPrompts = (w) => { const p = []; const inner = w.io.runner; w.io.runner = async (c) => { if (/CI failed on the pushed branch/.test(c.prompt)) p.push(c.prompt); return inner(c); }; return p; };
|
|
2086
|
+
const events = (dir, name) => readLog(dir, 7).filter((e) => e.event === name);
|
|
2087
|
+
/** `build` (run 77) is red; `urlAfter` is its URL once the rerun was triggered, `stateAfter` its state then. */
|
|
2088
|
+
const redBuild = (stateAfter, urlAfter) => {
|
|
2089
|
+
const w = world(); w.reran = [];
|
|
2090
|
+
w.io.requiredChecks = async () => ["build"];
|
|
2091
|
+
w.io.rerunFailedJobs = async (id) => { w.reran.push(id); };
|
|
2092
|
+
w.io.checkStates = async () => (w.reran.length ? [{ name: "build", state: stateAfter, url: urlAfter }] : [{ name: "build", state: "failure", url: RUN(77, 1) }]);
|
|
2093
|
+
w.io.failedLogs = async () => "boom";
|
|
2094
|
+
return w;
|
|
2095
|
+
};
|
|
2096
|
+
|
|
2097
|
+
it("a failed required check is rerun once, waited on while its URL is unchanged, and merges when the new attempt is green without a fix round", async () => {
|
|
2098
|
+
const w = redBuild("success", RUN(77, 2)); const p = fixPrompts(w), dir = tmpDir("core-");
|
|
2099
|
+
// the rerun was triggered, but the check still reports the old attempt for two more polls
|
|
2100
|
+
const states = w.io.checkStates; let stale = 2;
|
|
2101
|
+
w.io.checkStates = async () => (w.reran.length && stale-- > 0 ? [{ name: "build", state: "failure", url: RUN(77, 1) }] : states());
|
|
2102
|
+
const n = waits(w);
|
|
2103
|
+
const r = await run(w, { dir });
|
|
2104
|
+
assert.equal(r.terminalReason, "merged"); assert.deepEqual(w.reran, ["77"]); assert.equal(p.length, 0);
|
|
2105
|
+
assert.equal(n.n, 3, "one wait after the rerun, then two while the URL is unchanged");
|
|
2106
|
+
assert.deepEqual(events(dir, "CI_FLAKE_RERUN").map((e) => [e.trigger, e.result, e.runs]), [["merge", "rerun-started", ["77"]]]);
|
|
2107
|
+
assert.equal(events(dir, "CI_FIX_ROUND").length, 0);
|
|
2108
|
+
});
|
|
2109
|
+
|
|
2110
|
+
it("a rerun whose check URL never changes is waited on 20 times, then falls through to the fix round; the head is rerun only once", async () => {
|
|
2111
|
+
const w = redBuild("failure", RUN(77, 1)); const n = waits(w), dir = tmpDir("core-");
|
|
2112
|
+
const r = await run(w, { dir, verify: NO_FIX });
|
|
2113
|
+
assert.equal(r.terminalReason, "needs-human"); assert.equal(r.code, OUTCOME_CODES.CI_RED); assert.equal(w.merges.length, 0);
|
|
2114
|
+
assert.equal(n.n, 21, "the wait after the rerun plus 20 bounded waits");
|
|
2115
|
+
assert.deepEqual(w.reran, ["77"]);
|
|
2116
|
+
assert.equal(events(dir, "CI_FLAKE_RERUN").length, 1);
|
|
2117
|
+
});
|
|
2118
|
+
|
|
2119
|
+
it("a rerun that reports a new attempt still red goes to the fix round and reaches the builder", async () => {
|
|
2120
|
+
const w = redBuild("failure", RUN(77, 2)); const p = fixPrompts(w), n = waits(w), dir = tmpDir("core-");
|
|
2121
|
+
await run(w, { dir, verify: { ...parseVerify({}), maxRounds: 1 } });
|
|
2122
|
+
assert.ok(n.n >= 1); assert.equal(p.length, 1); assert.equal(w.reran[0], "77");
|
|
2123
|
+
assert.deepEqual(events(dir, "CI_FIX_ROUND").map((e) => [e.trigger, e.jobs]), [["merge", ["build"]]]);
|
|
2124
|
+
});
|
|
2125
|
+
|
|
2126
|
+
it("a rerun that cannot start is logged rerun-failed and the lane does not wait", async () => {
|
|
2127
|
+
const w = redBuild("failure", RUN(77, 1)); const n = waits(w), dir = tmpDir("core-");
|
|
2128
|
+
w.io.rerunFailedJobs = async () => { throw new Error("HTTP 403"); };
|
|
2129
|
+
const r = await run(w, { dir, verify: NO_FIX });
|
|
2130
|
+
assert.equal(r.code, OUTCOME_CODES.CI_RED); assert.equal(n.n, 0);
|
|
2131
|
+
assert.deepEqual(events(dir, "CI_FLAKE_RERUN").map((e) => e.result), ["rerun-failed"]);
|
|
2132
|
+
});
|
|
2133
|
+
|
|
2134
|
+
it("verify.flaky: a required check still red after the rerun is reported and ends needs-human; nothing merged, builder not resumed", async () => {
|
|
2135
|
+
const w = redBuild("failure", RUN(77, 2)); const p = fixPrompts(w), dir = tmpDir("core-");
|
|
2136
|
+
const r = await run(w, { dir, verify: { ...parseVerify({}), flaky: ["build"] } });
|
|
2137
|
+
assert.equal(r.terminalReason, "needs-human"); assert.equal(r.code, OUTCOME_CODES.CI_RED);
|
|
2138
|
+
assert.equal(w.merges.length, 0); assert.equal(p.length, 0); assert.deepEqual(w.reran, ["77"]);
|
|
2139
|
+
assert.deepEqual(events(dir, "CI_FLAKY_REPORTED").map((e) => [e.trigger, e.jobs]), [["merge", ["build"]]]);
|
|
2140
|
+
assert.ok(JSON.stringify(w.comments).includes("still failed on PR #"), "needs-human message names the rerun");
|
|
2141
|
+
});
|
|
2142
|
+
|
|
2143
|
+
it("verify.flaky with a mix: only the non-flaky required check reaches the builder", async () => {
|
|
2144
|
+
const w = world(); w.reran = [];
|
|
2145
|
+
w.io.requiredChecks = async () => ["build", "lint"];
|
|
2146
|
+
w.io.rerunFailedJobs = async (id) => { w.reran.push(id); };
|
|
2147
|
+
w.io.checkStates = async () => [{ name: "build", state: "failure", url: RUN(77, w.reran.length + 1) }, { name: "lint", state: "failure", url: RUN(78, w.reran.length + 1) }];
|
|
2148
|
+
w.io.failedLogs = async () => "boom";
|
|
2149
|
+
const dir = tmpDir("core-");
|
|
2150
|
+
await run(w, { dir, verify: { ...parseVerify({}), flaky: ["build"], maxRounds: 1 } });
|
|
2151
|
+
assert.deepEqual(w.reran.slice(0, 2).sort(), ["77", "78"], "both red runs rerun on the first head");
|
|
2152
|
+
assert.deepEqual(events(dir, "CI_FLAKY_REPORTED")[0].jobs, ["build"]);
|
|
2153
|
+
assert.deepEqual(events(dir, "CI_FIX_ROUND").map((e) => e.jobs), [["lint"]]);
|
|
2154
|
+
});
|
|
2155
|
+
});
|
|
2156
|
+
|
|
2077
2157
|
it("all waits for every check (waitCiGreen); none merges immediately", async () => {
|
|
2078
2158
|
const w = world({ waits: [{ rc: 2, text: "" }] }); let req = 0; w.io.requiredChecks = async () => { req++; return []; };
|
|
2079
2159
|
assert.equal((await run(w, { mergeWait: "all" })).terminalReason, "merged");
|
|
@@ -3785,6 +3865,12 @@ describe("shipped lane follows the open PR base (#4593)", () => {
|
|
|
3785
3865
|
assert.equal(r.terminalReason, "merged");
|
|
3786
3866
|
assert.ok(w.calls.some((c) => c[0] === "git" && c.includes("origin/milestone/new..refs/heads/feat/issue-7-lean")), "work is compared against the PR base");
|
|
3787
3867
|
});
|
|
3868
|
+
it("unused lane at the staging tip is not shipped (#4623)", async () => {
|
|
3869
|
+
// ancestor of staging but no merge commit took it: the hand-retargeted PR must stop, not be followed
|
|
3870
|
+
const w = world({ milestone: "Old", branchesOnOrigin: ["staging", "milestone/old", "milestone/new"], unusedAncestors: ["milestone/old"], prList: pr("milestone/new"), remote: "abc\trefs/heads/feat/issue-7-lean" });
|
|
3871
|
+
const r = await run(w, { lane: undefined });
|
|
3872
|
+
assert.deepEqual([r.terminalReason, r.code], ["needs-human", "lane.mismatch"]);
|
|
3873
|
+
});
|
|
3788
3874
|
it("stays needs-human when the lane is not shipped, the base is shipped, missing, or the default branch", async () => {
|
|
3789
3875
|
for (const [o, code] of [
|
|
3790
3876
|
[{ prList: pr("milestone/new") }, "lane.mismatch"],
|
|
@@ -3949,6 +4035,17 @@ describe("verify-by-CI (#4167): ci mode runs the heavy checks on CI and feeds fa
|
|
|
3949
4035
|
const ciValidation = [{ id: "V1", command: "node --test test/foo.test.js", ci: true }];
|
|
3950
4036
|
const events = (dir) => readLog(dir, 7).filter((e) => e.event === "CI_VERIFY");
|
|
3951
4037
|
|
|
4038
|
+
it("untrack commit before CI verify: a tool cache the builder committed does not move HEAD under the CI lookup (#4621)", async () => {
|
|
4039
|
+
const w = withCi(world({ validation: ciValidation })); w.added = ["src/foo.js", "apps/worker/.eslintcache"];
|
|
4040
|
+
const inner = w.io.git;
|
|
4041
|
+
w.io.git = async (a, o) => { const out = await inner(a, o); if (a[0] === "commit" && a.some((x) => /untrack lane tool state/.test(x))) w.head = `dddd${w.calls.length}`.padEnd(16, "0"); return out; };
|
|
4042
|
+
const dir = tmpDir("core-ci-");
|
|
4043
|
+
const r = await run(w, { verify: CI, dir });
|
|
4044
|
+
assert.equal(r.terminalReason, "merged");
|
|
4045
|
+
assert.deepEqual(events(dir).map((e) => e.result), ["pass"], "CI found the run for the pushed tip");
|
|
4046
|
+
assert.ok(!builders(w).some((b) => /CI (is )?unavailable|cannot run/i.test(String(b[4]))), "no ci_unavailable builder round");
|
|
4047
|
+
});
|
|
4048
|
+
|
|
3952
4049
|
it("green CI: the builder is told not to run test suites, the branch is pushed and verified before review, validate left to CI passes enforce", async () => {
|
|
3953
4050
|
const w = withCi(world({ validation: ciValidation }));
|
|
3954
4051
|
const dir = tmpDir("core-ci-");
|
|
@@ -5189,6 +5286,17 @@ describe("enumerating tests + sequence numbers at hand-off (#4322)", () => {
|
|
|
5189
5286
|
assert.equal(w.dupArgs.cwd, WT);
|
|
5190
5287
|
});
|
|
5191
5288
|
|
|
5289
|
+
it("#4596 English copies in a locale catalog resume the builder once to translate; a leftover is an anomaly", async () => {
|
|
5290
|
+
const w = hworld(); const c = [[{ path: "messages/ta.json", key: "calendarView.day", value: "Day view", source: "messages/en.json" }], []]; w.cpCalls = 0;
|
|
5291
|
+
w.io.englishCopies = async (x) => { w.cpCalls++; w.cpArgs = x; return c.length > 1 ? c.shift() : c[0]; };
|
|
5292
|
+
assert.equal((await run(w)).terminalReason, "merged");
|
|
5293
|
+
const b = builders(w);
|
|
5294
|
+
assert.equal(b.length, 3);
|
|
5295
|
+
assert.match(b[2][4], /messages\/ta\.json: calendarView\.day = "Day view" \(same as messages\/en\.json\)[\s\S]*Translate each/);
|
|
5296
|
+
assert.ok(w.seq.indexOf("pr") > w.seq.lastIndexOf("builder-resume"), "fixed before the PR");
|
|
5297
|
+
assert.equal(w.cpArgs.cwd, WT);
|
|
5298
|
+
});
|
|
5299
|
+
|
|
5192
5300
|
it("#4406 a duplicate that remains after the round is an anomaly; verify.duplicate_exports false skips the check", async () => {
|
|
5193
5301
|
let w = hworld(); w.io.duplicateExports = async () => DUP;
|
|
5194
5302
|
assert.equal((await run(w)).terminalReason, "merged");
|
|
@@ -5838,6 +5946,17 @@ describe("e2e merge gate (#4403)", () => {
|
|
|
5838
5946
|
assert.equal(r2.terminalReason, "needs-human"); assert.equal(w2.merges.length, 0);
|
|
5839
5947
|
});
|
|
5840
5948
|
|
|
5949
|
+
it("#4624 verify.flaky: a flaky-listed own e2e failure is reported as an anomaly, never reaches the builder, and the lane merges; a non-listed one still does", async () => {
|
|
5950
|
+
const w = withE2e(world({ changed: SPECS }), ["failure"]);
|
|
5951
|
+
const fixes = []; const inner = w.io.runner; w.io.runner = async (c) => { if (/CI failed on the pushed branch/.test(c.prompt)) fixes.push(c.prompt); return inner(c); };
|
|
5952
|
+
const r = await go(w, { verify: { ...cfg(), flaky: ["e2e"] } });
|
|
5953
|
+
assert.equal(r.terminalReason, "merged"); assert.equal(fixes.length, 0);
|
|
5954
|
+
assert.ok(traj(w).some((t) => /Known-flaky e2e failure\(s\) reported, not blocking \(verify\.flaky\): e2e/.test(t)));
|
|
5955
|
+
const w2 = withE2e(world({ changed: SPECS }), ["failure"]);
|
|
5956
|
+
const r2 = await go(w2, { verify: { ...cfg(), flaky: ["other"], maxRounds: 0 } });
|
|
5957
|
+
assert.equal(r2.code, OUTCOME_CODES.CI_RED); assert.equal(w2.merges.length, 0);
|
|
5958
|
+
});
|
|
5959
|
+
|
|
5841
5960
|
it("verify.e2e: never (or no label/workflow configured) keeps today's behaviour: no label, no e2e wait, no event", async () => {
|
|
5842
5961
|
for (const verify of [cfg({ e2e: "never" }), parseVerify({})]) {
|
|
5843
5962
|
const w = withE2e(world({ changed: SPECS }), ["failure"]);
|
|
@@ -6174,7 +6293,7 @@ describe("#4395 lane profile", () => {
|
|
|
6174
6293
|
assert.match(p, /Issue independent read-only calls \(reads, greps, git log\/blame, gh\) in the same turn or one shell call\./);
|
|
6175
6294
|
assert.match(p, /Read each file's ranges once .* and keep notes; don't re-read it\./);
|
|
6176
6295
|
}
|
|
6177
|
-
assert.ok(approxTokens(builderImplementPrompt()) <=
|
|
6296
|
+
assert.ok(approxTokens(builderImplementPrompt()) <= 1550 && approxTokens(builderPlanPrompt(7)) <= 1800);
|
|
6178
6297
|
});
|
|
6179
6298
|
|
|
6180
6299
|
it("runCliBackendAsync reports the first stream turn's context as baselineTokens; makeLeanRunner drops it on resume", async () => {
|
package/bin/lean/orchestrate.mjs
CHANGED
|
@@ -104,6 +104,7 @@ import { recoveryEvidence } from "./render.mjs";
|
|
|
104
104
|
import { renderAlerts } from "../stats/render.mjs";
|
|
105
105
|
import { formatParkedLine, postDeployOnly } from "./prefilter.mjs";
|
|
106
106
|
import { laneSize, SIZES } from "./budget.mjs";
|
|
107
|
+
import { requeueKey, REQUEUABLE } from "./batch.mjs";
|
|
107
108
|
|
|
108
109
|
// A dependency that finished like this no longer blocks its dependents.
|
|
109
110
|
const SATISFIED = new Set(["merged", "invalid"]);
|
|
@@ -234,7 +235,7 @@ const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k),
|
|
|
234
235
|
* @returns {Promise<Array<{issue:number, outcome:string, pr:?number, cost_usd:number, minutes:number, detail?:string}>>}
|
|
235
236
|
*/
|
|
236
237
|
export async function orchestrate({ issues, parallel = 3, runOne, readBody, prefilter = null, resolveLane = null, now = () => Date.now(), onUpdate = () => {}, onCrossLane = () => {},
|
|
237
|
-
store = null, adopt = null, getParallel = null, getAdded = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
|
|
238
|
+
store = null, adopt = null, getParallel = null, getAdded = null, getRequeued = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
|
|
238
239
|
classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
|
|
239
240
|
recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, getHeld = null, stopLane = null,
|
|
240
241
|
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS }) {
|
|
@@ -301,6 +302,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
301
302
|
const recoveryMax = Number.isInteger(recovery?.maxAttempts) && recovery.maxAttempts > 0 ? recovery.maxAttempts : 0;
|
|
302
303
|
const recoveries = toMap(saved?.recoveries); // issue -> recovery lanes launched (#4267)
|
|
303
304
|
const recovering = toMap(saved?.recovering); // issue -> {code, attempt, detail, outcome, ...} of the recovery lane queued or running
|
|
305
|
+
const requeueApplied = new Set(saved?.requeueApplied ?? []); // #4550: `--requeue` entries already applied (batch `requeue` file)
|
|
306
|
+
const requeuedInfo = toMap(saved?.requeued); // #4550: issue -> {reason, by, at, prior: [earlier rows]}
|
|
304
307
|
const relabelled = new Set(saved?.relabelled ?? []); // #4315: queued recoveries relabelled needs-human when a batch stopped
|
|
305
308
|
// #4267: decisions are recorded, never awaited: a failing or slow recorder cannot block the batch.
|
|
306
309
|
const decided = (d) => { try { Promise.resolve(decide?.(d)).catch(() => {}); } catch { /* best effort */ } };
|
|
@@ -314,6 +317,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
314
317
|
prefilter: toObj(checked), errorTimes, breaker: tripped, ...(budget ? { classes: toObj(classes), dispatch: { ...dispatch } } : {}),
|
|
315
318
|
...(reg ? { priorities: toObj(priorities) } : {}), ...(sandboxUse.size ? { sandbox: toObj(sandboxUse) } : {}),
|
|
316
319
|
...(recoveries.size || recovering.size ? { recoveries: toObj(recoveries), recovering: toObj(recovering) } : {}),
|
|
320
|
+
...(requeueApplied.size ? { requeueApplied: [...requeueApplied], requeued: toObj(requeuedInfo) } : {}),
|
|
317
321
|
...(relabelled.size ? { relabelled: [...relabelled] } : {}), ...(cancelled ? { cancelled } : {}),
|
|
318
322
|
...(estimate ? { estimates: toObj(estimates), enqueued: toObj(enqueued) } : {}), ...(heads.size ? { heads: [...heads] } : {}) });
|
|
319
323
|
};
|
|
@@ -782,10 +786,46 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
782
786
|
update({ issue: null, outcome: "added", detail: list });
|
|
783
787
|
return fresh.length;
|
|
784
788
|
};
|
|
785
|
-
// #
|
|
789
|
+
// #4550: `--requeue` entries (batch `requeue` file) not applied yet: a terminal issue and the dependents skipped behind it
|
|
790
|
+
// return to pending, the prior rows kept in history; edges stay, so a dependent still waits for the issue to merge.
|
|
791
|
+
const drainRequeues = async () => {
|
|
792
|
+
if (!getRequeued) return 0;
|
|
793
|
+
let got = [];
|
|
794
|
+
try { got = getRequeued() ?? []; } catch { got = []; }
|
|
795
|
+
const fresh = (Array.isArray(got) ? got : []).filter((e) => Number.isSafeInteger(e?.issue) && !requeueApplied.has(requeueKey(e)));
|
|
796
|
+
let n = 0;
|
|
797
|
+
for (const e of fresh) {
|
|
798
|
+
requeueApplied.add(requeueKey(e));
|
|
799
|
+
const row = results.get(e.issue);
|
|
800
|
+
if (!row || !REQUEUABLE(row.outcome) || running.has(e.issue)) continue;
|
|
801
|
+
const moved = [e.issue], priors = new Map([[e.issue, row]]);
|
|
802
|
+
results.delete(e.issue);
|
|
803
|
+
for (let i = 0; i < moved.length; i++) { // dependents skipped behind a requeued issue (fixpoint)
|
|
804
|
+
for (const [m, r] of [...results]) {
|
|
805
|
+
if (r?.outcome !== "skipped" || Number(/^dependency #(\d+)/.exec(r.detail ?? "")?.[1]) !== moved[i]) continue;
|
|
806
|
+
priors.set(m, r); results.delete(m); moved.push(m);
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
for (const m of moved) {
|
|
810
|
+
for (const x of [requeues, deferrals, engineErrors, recoveries, recovering]) x.delete(m);
|
|
811
|
+
relabelled.delete(m);
|
|
812
|
+
if (m === e.issue) requeuedInfo.set(m, { reason: e.reason, by: e.by, at: e.at, prior: [...(requeuedInfo.get(m)?.prior ?? []), priors.get(m)] });
|
|
813
|
+
if (!pending.includes(m)) requeue(m, 0);
|
|
814
|
+
}
|
|
815
|
+
save();
|
|
816
|
+
const list = moved.filter((m) => m !== e.issue).map((m) => `#${m}`);
|
|
817
|
+
decided({ issue: e.issue, what: `requeued by ${e.by ?? "operator"} (forgedock orchestrate --requeue); prior outcome ${row.outcome}${list.length ? `; dependents back to pending: ${list.join(", ")}` : ""}`,
|
|
818
|
+
why: e.reason || "operator decision", alternative: "start a second batch" });
|
|
819
|
+
update({ issue: e.issue, outcome: "requeued", detail: `${e.reason || "operator decision"} (was ${row.outcome})` });
|
|
820
|
+
n++;
|
|
821
|
+
}
|
|
822
|
+
if (fresh.length) save();
|
|
823
|
+
return n;
|
|
824
|
+
};
|
|
825
|
+
// #4376: one SIGTERM per running local lane (their teardown saves the work); remote lanes run to their result unless the cancel carries `stopRemote` (#4612).
|
|
786
826
|
const stopSent = new Set();
|
|
787
827
|
function stop(n, rec) {
|
|
788
|
-
if (!stopLane || rec?.remote || stopSent.has(n)) return;
|
|
828
|
+
if (!stopLane || (rec?.remote && cancelled?.stopRemote !== true) || stopSent.has(n)) return; // #4612: remote lanes only when the cancel asked for it
|
|
789
829
|
stopSent.add(n);
|
|
790
830
|
try { Promise.resolve(stopLane(n, rec)).catch(() => {}); } catch { /* best effort */ }
|
|
791
831
|
}
|
|
@@ -879,6 +919,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
879
919
|
for (;;) {
|
|
880
920
|
if (signal?.aborted) throw halt();
|
|
881
921
|
await drainAdds();
|
|
922
|
+
await drainRequeues();
|
|
882
923
|
checkCancel();
|
|
883
924
|
await watchBlockers(!pending.length && !running.size);
|
|
884
925
|
if (!pending.length && !running.size) break;
|
|
@@ -970,8 +1011,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
970
1011
|
if (holdUntil && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4446: a cleared hold (forgedock limits clear) admits on the next poll
|
|
971
1012
|
if ((getParallel || getHeld) && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
|
|
972
1013
|
if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
|
|
1014
|
+
if ((getAdded || getRequeued) && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408/#4550: an add or requeue while only lanes run
|
|
973
1015
|
if (getHeld && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4592: a release while only lanes run
|
|
974
|
-
if (getAdded && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408: an add while only lanes run
|
|
975
1016
|
if (syncParked && running.size && parkedPrs().length) wake = Math.min(wake ?? syncEveryMs, Math.max(0, syncEveryMs - (t - (lastParkedSync ?? t)))); // #4525
|
|
976
1017
|
if (running.size && watchable()) wake = Math.min(wake ?? WATCH_EVERY_MS, WATCH_EVERY_MS); // #4595: re-check the blockers of skipped rows
|
|
977
1018
|
if (tripped) wake = null;
|
|
@@ -998,7 +1039,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
998
1039
|
if (reg) withdrawExcept(new Set());
|
|
999
1040
|
await relabelQueued();
|
|
1000
1041
|
save();
|
|
1001
|
-
|
|
1042
|
+
const withRequeue = (r) => (r && requeuedInfo.has(r.issue) ? { ...r, requeued: { reason: requeuedInfo.get(r.issue).reason, by: requeuedInfo.get(r.issue).by, prior: requeuedInfo.get(r.issue).prior.map((p) => ({ outcome: p.outcome, detail: p.detail ?? null, pr: p.pr ?? null })) } } : r);
|
|
1043
|
+
return order.map((n) => withRequeue(results.get(n)) ?? { issue: n, outcome: "held", pr: null, cost_usd: 0, minutes: 0, detail: heldDetail(n),
|
|
1002
1044
|
...(recovering.has(n) ? { recovery: { code: recovering.get(n).code, attempts: recoveries.get(n) ?? recovering.get(n).attempt, outcome: recovering.get(n).outcome ?? "needs-human",
|
|
1003
1045
|
...(recovering.get(n).result_code ? { result_code: recovering.get(n).result_code } : {}), detail: firstLine(recovering.get(n).detail ?? recovering.get(n).code) } } : {}) });
|
|
1004
1046
|
}
|
|
@@ -1027,6 +1069,8 @@ export const OUTCOME_LINE_MAX = 200;
|
|
|
1027
1069
|
export function formatOutcomeLists(rows) {
|
|
1028
1070
|
const out = [];
|
|
1029
1071
|
const one = (x) => String(x ?? "").split("\n")[0].slice(0, OUTCOME_LINE_MAX);
|
|
1072
|
+
const requeued = (rows ?? []).filter((r) => r?.requeued); // #4550: operator --requeue after a decision; the row is the latest outcome
|
|
1073
|
+
if (requeued.length) out.push(`requeued: ${requeued.map((r) => `#${r.issue} (${one(r.requeued.reason)})`).join(" ")}`);
|
|
1030
1074
|
const rec = (rows ?? []).filter((r) => r?.recovered);
|
|
1031
1075
|
if (rec.length) out.push(`recovered: ${rec.map((r) => `#${r.issue} (${r.recovered})`).join(" ")}`);
|
|
1032
1076
|
for (const r of rows ?? []) {
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
/**
|
|
4
|
+
* Named post-deploy checks (#4626). A post-deploy requirement names a check id (`[check:<id>]` in its follow-up item);
|
|
5
|
+
* the command lives in the checks file on the default branch, `.forge/post-deploy-checks.json`:
|
|
6
|
+
* { "<id>": { "run": "sh command", "pass": "exit 0 | /regex/flags", "timeout": 120 } } (a bare string is `run`)
|
|
7
|
+
* and is read with `git show origin/<default>:<file>`: reviewed default-branch code, never an issue body or comment
|
|
8
|
+
* (#4468). When the #4389 gate reports a follow-up ready, `measureFollowup` runs every item's check (no model call, the
|
|
9
|
+
* probe runner's minimal env): all pass -> a FORGE:PROBE record and the follow-up closes; a fail -> the record, and the
|
|
10
|
+
* parent reopens with the output. An item without a runnable check is `awaiting-measurement`: no lane, ever.
|
|
11
|
+
*/
|
|
12
|
+
import { noComments } from "./render.mjs";
|
|
13
|
+
import { MAX_TIMEOUT_S, OUTPUT_TAIL, PROBE_MARKER, probeEnv, probeResultOf, readProbeConfig, runProbe } from "./probe.mjs";
|
|
14
|
+
import { parseMinAge } from "./prefilter.mjs";
|
|
15
|
+
|
|
16
|
+
export const CHECKS_FILE = ".forge/post-deploy-checks.json";
|
|
17
|
+
export const CHECK_ID_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
18
|
+
export const AWAITING_MEASUREMENT_LABEL = "awaiting-measurement";
|
|
19
|
+
export const MONITORING_LABEL = "post-deploy-monitoring";
|
|
20
|
+
export const DEFAULT_EVERY_MS = 6 * 3600e3;
|
|
21
|
+
const ITEM_RE = /^\s*[-*]\s+\[[ xX]\]\s+(.*)$/;
|
|
22
|
+
const CHECK_TOKEN_RE = /^`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:\s*\[check:\s*([^\]\s]*)\s*\]/;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* The checks file's runnable entries `{id: {run, pass, timeoutS, forMs?, everyMs?}}`; malformed input or entries are dropped.
|
|
26
|
+
* `for` (a window such as "7d") makes the check windowed (#4627); `every` (default 6h) is its cadence and needs a `for`.
|
|
27
|
+
*/
|
|
28
|
+
export function parseChecks(text) {
|
|
29
|
+
let j;
|
|
30
|
+
try { j = JSON.parse(String(text ?? "")); } catch { return {}; }
|
|
31
|
+
if (!j || typeof j !== "object" || Array.isArray(j)) return {};
|
|
32
|
+
const out = {};
|
|
33
|
+
for (const [id, v] of Object.entries(j)) {
|
|
34
|
+
const spec = typeof v === "string" ? { run: v } : v;
|
|
35
|
+
if (!CHECK_ID_RE.test(id) || !spec || typeof spec.run !== "string" || !spec.run.trim()) continue;
|
|
36
|
+
let pass = { kind: "exit" };
|
|
37
|
+
const p = String(spec.pass ?? "").trim();
|
|
38
|
+
if (p && !/^exit\s+0$/i.test(p)) {
|
|
39
|
+
const r = /^\/(.+)\/([a-z]*)$/.exec(p);
|
|
40
|
+
try { if (!r) continue; new RegExp(r[1], r[2]); } catch { continue; }
|
|
41
|
+
pass = { kind: "regex", re: r[1], flags: r[2] };
|
|
42
|
+
}
|
|
43
|
+
const t = spec.timeout == null ? null : Number(spec.timeout);
|
|
44
|
+
if (t != null && !(Number.isFinite(t) && t > 0)) continue;
|
|
45
|
+
const win = spec.for == null ? null : parseMinAge(spec.for);
|
|
46
|
+
const cad = spec.every == null ? null : parseMinAge(spec.every);
|
|
47
|
+
if ((spec.for != null && !win) || (spec.every != null && !cad) || (spec.every != null && spec.for == null)) continue;
|
|
48
|
+
out[id] = { run: spec.run.trim(), pass, timeoutS: t == null ? null : Math.min(Math.floor(t) || 1, MAX_TIMEOUT_S), ...(win ? { forMs: win, everyMs: cad ?? DEFAULT_EVERY_MS } : {}) };
|
|
49
|
+
}
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Per checklist item of a follow-up body: its `[check:<id>]` id, or null when it names none (or a malformed one). */
|
|
54
|
+
export function itemCheckIds(body) {
|
|
55
|
+
return noComments(String(body ?? ""), " ").split("\n").map((l) => ITEM_RE.exec(l)?.[1]).filter((t) => t != null)
|
|
56
|
+
.map((t) => { const m = CHECK_TOKEN_RE.exec(t); return m && CHECK_ID_RE.test(m[1]) ? m[1] : null; });
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** The checks file at each git ref (`git show <ref>:<file>`); a missing file or ref contributes nothing. */
|
|
60
|
+
export async function checksAt(git, refs, cwd) {
|
|
61
|
+
const out = {};
|
|
62
|
+
for (const ref of refs) {
|
|
63
|
+
try { Object.assign(out, parseChecks(String(await git(["show", `${ref}:${CHECKS_FILE}`], { cwd })))); } catch { /* no file at this ref */ }
|
|
64
|
+
}
|
|
65
|
+
return out;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const fenced = (s) => { const f = "`".repeat(Math.max(3, ...[...String(s).matchAll(/`+/g)].map((m) => m[0].length + 1))); return `${f}\n${String(s).trim() || "(no output)"}\n${f}`; };
|
|
69
|
+
|
|
70
|
+
/** Run each distinct check id with the probe runner (minimal env, timeout, masked output): `[{id, result, exit, ms, output}]`. */
|
|
71
|
+
export async function runNamedChecks({ ids, checks, cwd, config, env, spawn }) {
|
|
72
|
+
const rows = [];
|
|
73
|
+
for (const id of [...new Set(ids)]) {
|
|
74
|
+
const r = await runProbe(checks[id], { cwd, env: probeEnv(config.env, env), mask: config.env.map((k) => env[k]), timeoutS: config.timeoutS, ...(spawn ? { spawn } : {}) });
|
|
75
|
+
rows.push({ id, ...r });
|
|
76
|
+
}
|
|
77
|
+
return rows;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The windowed-monitoring fields of a FORGE:PROBE record body (`at`, `window_until`, `every_ms`, `regression`, ...), or null. */
|
|
81
|
+
export function monitorRecord(body) {
|
|
82
|
+
const base = probeResultOf(body);
|
|
83
|
+
if (!base) return null;
|
|
84
|
+
const line = String(body).split("\n").find((l) => /^\s*`?\{.*"result"/.test(l));
|
|
85
|
+
try {
|
|
86
|
+
const j = JSON.parse(line.trim().replace(/^`|`$/g, ""));
|
|
87
|
+
const at = Date.parse(j.at), until = Date.parse(j.window_until);
|
|
88
|
+
return Number.isFinite(at) ? { ...base, at, defaultSha: j.default_sha ?? null, everyMs: Number(j.every_ms) || DEFAULT_EVERY_MS, until: Number.isFinite(until) ? until : null, regression: j.regression ?? null } : null;
|
|
89
|
+
} catch { return null; }
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** The post-deploy-monitoring label, created on first use; best effort. */
|
|
93
|
+
export async function addMonitoringLabel({ io, repo, issue }) {
|
|
94
|
+
const edit = () => io.gh(["issue", "edit", String(issue), "-R", repo, "--add-label", MONITORING_LABEL]);
|
|
95
|
+
try { await edit(); } catch {
|
|
96
|
+
try { await io.gh(["label", "create", MONITORING_LABEL, "-R", repo, "--force", "--color", "C5DEF5", "--description", "Post-deploy follow-up whose windowed check re-runs on later releases. Managed by ForgeDock."]); await edit(); } catch { /* the record still shows it */ }
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Measure a ready post-deploy follow-up. Never runs a command that is not in the default branch's checks file.
|
|
102
|
+
* @returns {Promise<{state:"awaiting-measurement"|"pass"|"fail"|"recorded-fail"|"dry-run"|"monitoring", detail:string}>}
|
|
103
|
+
* When every item's check declares a window (#4627) a pass records the release and the window end and the follow-up stays
|
|
104
|
+
* open (`monitoring`); `sweepDue` (postdeploy-monitor.mjs) re-runs it.
|
|
105
|
+
*/
|
|
106
|
+
export async function measureFollowup({ iss, repo, parent, def, shas = [], cwd, io, dryRun = false, isTrusted = () => false, config = readProbeConfig(cwd), env = process.env, spawn, now = () => Date.now() }) {
|
|
107
|
+
const ids = itemCheckIds(iss.body);
|
|
108
|
+
const checks = await checksAt(io.git, [`origin/${def}`], cwd);
|
|
109
|
+
const missing = ids.filter((id) => !id || !checks[id]);
|
|
110
|
+
if (!ids.length || missing.length) return { state: "awaiting-measurement", detail: `no runnable check on ${def} for ${ids.length ? `${missing.map((m) => m ?? "an item without a check id").join(", ")}` : "any item"} (${CHECKS_FILE})` };
|
|
111
|
+
const fix = [...shas].sort().join(",");
|
|
112
|
+
const prior = (iss.comments ?? []).filter((c) => isTrusted(c)).map((c) => probeResultOf(c?.body)).filter(Boolean).reverse().find((p) => p.fix === fix);
|
|
113
|
+
if (prior?.result === "fail") return { state: "recorded-fail", detail: `check failed for ${fix.slice(0, 12)} (FORGE:PROBE); waiting for a new fix of #${parent}` };
|
|
114
|
+
const windowed = ids.every((id) => checks[id].forMs);
|
|
115
|
+
const monitored = (iss.comments ?? []).filter((c) => isTrusted(c)).map((c) => monitorRecord(c?.body)).filter(Boolean).find((p) => p.fix === fix && p.result === "pass" && p.until);
|
|
116
|
+
if (windowed && monitored) return { state: "monitoring", detail: `windowed checks ${[...new Set(ids)].join(", ")} monitored until ${new Date(monitored.until).toISOString()}` };
|
|
117
|
+
if (dryRun) return { state: "dry-run", detail: `checks ${[...new Set(ids)].join(", ")} not run (dry run)` };
|
|
118
|
+
let defaultSha = null;
|
|
119
|
+
try { defaultSha = String(await io.git(["rev-parse", `origin/${def}`], { cwd })).trim() || null; } catch { /* recorded as null */ }
|
|
120
|
+
const rows = [];
|
|
121
|
+
if (prior?.result !== "pass") {
|
|
122
|
+
rows.push(...await runNamedChecks({ ids, checks, cwd, config, env, spawn }));
|
|
123
|
+
const ok = rows.every((r) => r.result === "pass");
|
|
124
|
+
const at = now();
|
|
125
|
+
const win = windowed && ok ? { at: new Date(at).toISOString(), window_until: new Date(at + Math.max(...ids.map((id) => checks[id].forMs))).toISOString(), every_ms: Math.min(...ids.map((id) => checks[id].everyMs)) } : {};
|
|
126
|
+
const body = [
|
|
127
|
+
`<!-- ${PROBE_MARKER} -->`, `## Post-deploy check: ${ok ? "pass" : "fail"}`,
|
|
128
|
+
JSON.stringify({ fix, default_sha: defaultSha, result: ok ? "pass" : "fail", ...win, checks: rows.map((r) => ({ id: r.id, result: r.result, exit: r.exit, ms: r.ms })) }), "",
|
|
129
|
+
`Ran after the fix of #${parent} reached \`${def}\`; commands come from \`${CHECKS_FILE}\` on \`${def}\`.${ok ? (win.at ? ` Monitored until \`${win.window_until}\`, re-run on later releases.` : " The follow-up is closed.") : " The parent is reopened."}`,
|
|
130
|
+
...rows.filter((r) => r.result === "fail").flatMap((r) => ["", `**${r.id}** (exit ${r.exit ?? "timeout"})`, fenced(r.output)]),
|
|
131
|
+
].join("\n").slice(0, OUTPUT_TAIL * 4);
|
|
132
|
+
await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
|
|
133
|
+
if (!ok) {
|
|
134
|
+
const bad = rows.filter((r) => r.result === "fail");
|
|
135
|
+
let note = "";
|
|
136
|
+
try { await io.gh(["issue", "reopen", String(parent), "-R", repo]); } catch (e) { note = ` (reopen failed: ${String(e?.message ?? e).slice(0, 120)})`; }
|
|
137
|
+
try {
|
|
138
|
+
await io.gh(["issue", "comment", String(parent), "-R", repo, "--body", `Post-deploy check ${bad.map((r) => `\`${r.id}\``).join(", ")} failed for the fix on \`${def}\` (follow-up #${iss.number}); reopened.\n\n${bad.map((r) => fenced(r.output)).join("\n\n")}`.slice(0, OUTPUT_TAIL * 4)]);
|
|
139
|
+
} catch { /* the follow-up comment carries the output */ }
|
|
140
|
+
return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; #${parent} reopened${note}` };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
if (windowed) {
|
|
144
|
+
await addMonitoringLabel({ io, repo, issue: iss.number });
|
|
145
|
+
return { state: "monitoring", detail: `checks ${[...new Set(ids)].join(", ")} passed; monitored on later releases` };
|
|
146
|
+
}
|
|
147
|
+
await io.gh(["issue", "close", String(iss.number), "-R", repo, "--reason", "completed"]);
|
|
148
|
+
return { state: "pass", detail: `checks ${[...new Set(ids)].join(", ")} passed` };
|
|
149
|
+
}
|