mandrel 2.84.0 → 2.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/docs/agentrc-reference.json +2 -1
- package/.agents/docs/configuration.md +4 -2
- package/.agents/schemas/agentrc.schema.json +41 -2
- package/.agents/schemas/story-deliver-terminal.schema.json +2 -2
- package/.agents/scripts/coverage-capture.js +1 -0
- package/.agents/scripts/evidence-gate.js +3 -2
- package/.agents/scripts/lib/close-lock-events.js +29 -9
- package/.agents/scripts/lib/close-validation/process.js +6 -0
- package/.agents/scripts/lib/close-validation/runner.js +22 -6
- package/.agents/scripts/lib/config-settings-schema-delivery.js +41 -2
- package/.agents/scripts/lib/full-suite-lock.js +100 -38
- package/.agents/scripts/lib/full-suite-queue.js +104 -37
- package/.agents/scripts/lib/generated/agentrc-validator.js +1 -1
- package/.agents/scripts/lib/orchestration/review-providers/claude-verdict.js +242 -0
- package/.agents/scripts/lib/orchestration/review-providers/code-review.js +151 -111
- package/.agents/scripts/lib/orchestration/single-story-close/runner.js +30 -4
- package/.agents/scripts/lib/orchestration/story-handoff.js +103 -11
- package/.agents/scripts/lib/suite-admission.js +410 -0
- package/.agents/workflows/helpers/deliver-story-reference.md +6 -0
- package/docs/CHANGELOG.md +15 -0
- package/package.json +1 -1
|
@@ -135,8 +135,10 @@ Everything `/mandrel-deliver` and `single-story-close` consume: worktree isolati
|
|
|
135
135
|
|
|
136
136
|
| Key | Required | Type | Default | Description |
|
|
137
137
|
| --- | --- | --- | --- | --- |
|
|
138
|
-
| `execution` | No | `object` | — | Serialization of the
|
|
139
|
-
| `execution.fullSuiteLock` | No | `boolean` | `true` | Serialize full-suite spawns (`npm test` / `npm run test:coverage`) behind a
|
|
138
|
+
| `execution` | No | `object` | — | Serialization and capacity admission of the suite spawns delivery drives. |
|
|
139
|
+
| `execution.fullSuiteLock` | No | `boolean` | `true` | Serialize full-suite spawns (`npm test` / `npm run test:coverage`) behind a repository-level advisory lock shared by every worktree, so two concurrent deliveries do not run two suites against the same cores (see `capacity` / `workloads` for declared runs that may overlap). The lock queues, it never overlaps: a wait that expires with a live holder spawns nothing and exits 75 (resumable), naming the holder. A dead or non-heartbeating holder is taken over, and a broken lockfile proceeds unserialized, so the lock can never fail a delivery. Set false — or export `MANDREL_FULL_SUITE_LOCK=0` for one invocation — to disable. |
|
|
140
|
+
| `execution.capacity` | No | `integer` | — | Story #5668. The admission budget, in weight units (CPU workers), shared by every suite spawn across this repository's worktrees and sessions — runs in other repositories on the host are not counted. Unset → the host's `os.availableParallelism()`. A declared workload is admitted while the live weights plus its own fit, strictly first-in-first-out: nothing overtakes an earlier live waiter, so a queued full suite is never starved. |
|
|
141
|
+
| `execution.workloads[]` | No | `array<object>` | `[]` | Story #5668. Suite commands declared safe to run alongside others. A spawn whose argv (whitespace-normalized) EXACTLY equals `command` takes that `weight` and `resources`; anything else — and every spawn when this list is empty, the default — runs exclusively (full capacity, today's serialization). Isolation is never inferred from a flag, prefix or diff size. `weight` must count the run's nested worker pool (`--workers 4` → 4), and a full suite needs its own entry before anything can overlap it. Two runs naming a common `resources` entry (a database fixture, a shared coverage file) never overlap; worktree-local outputs need none. A weight above `capacity` refuses the run with exit 78 — nothing spawns, no credit is recorded. An expired wait still exits 75, resumably. Scheduling never changes a run's credit identity. `MANDREL_FULL_SUITE_LOCK=0` (one invocation) or `fullSuiteLock: false` turns admission off entirely; reach for the env hatch only when you have checked the runs share nothing. Each item has: command, weight, resources. |
|
|
140
142
|
| `docsFreshness` | No | `object` | — | Documentation-freshness scope: the files a change of consequence is expected to touch. Read by the audit-documentation lens to seed its target set; no delivery gate enforces it. |
|
|
141
143
|
| `docsFreshness.paths` | No | `array<string>` | `["README.md"]` | Repo-relative documentation paths the audit-documentation lens adds to its target set. |
|
|
142
144
|
| `tempRetention` | No | `object` | — | Story #4794. Auto-purge of spent temp artifacts once their Story lands. Classification is an allowlist: only the declared classes below are ever deleted, so unrecognized files under tempRoot are reported with their size and left alone (`/clean-temp` is the operator path for them). signals.ndjson is never purged by any path. |
|
|
@@ -355,12 +355,51 @@
|
|
|
355
355
|
"properties": {
|
|
356
356
|
"execution": {
|
|
357
357
|
"type": "object",
|
|
358
|
-
"description": "Serialization of the
|
|
358
|
+
"description": "Serialization and capacity admission of the suite spawns delivery drives.",
|
|
359
359
|
"properties": {
|
|
360
360
|
"fullSuiteLock": {
|
|
361
361
|
"type": "boolean",
|
|
362
|
-
"description": "Serialize full-suite spawns (`npm test` / `npm run test:coverage`) behind a
|
|
362
|
+
"description": "Serialize full-suite spawns (`npm test` / `npm run test:coverage`) behind a repository-level advisory lock shared by every worktree, so two concurrent deliveries do not run two suites against the same cores (see `capacity` / `workloads` for declared runs that may overlap). The lock queues, it never overlaps: a wait that expires with a live holder spawns nothing and exits 75 (resumable), naming the holder. A dead or non-heartbeating holder is taken over, and a broken lockfile proceeds unserialized, so the lock can never fail a delivery. Set false — or export `MANDREL_FULL_SUITE_LOCK=0` for one invocation — to disable.",
|
|
363
363
|
"default": true
|
|
364
|
+
},
|
|
365
|
+
"capacity": {
|
|
366
|
+
"type": "integer",
|
|
367
|
+
"minimum": 1,
|
|
368
|
+
"description": "Story #5668. The admission budget, in weight units (CPU workers), shared by every suite spawn across this repository's worktrees and sessions — runs in other repositories on the host are not counted. Unset → the host's `os.availableParallelism()`. A declared workload is admitted while the live weights plus its own fit, strictly first-in-first-out: nothing overtakes an earlier live waiter, so a queued full suite is never starved."
|
|
369
|
+
},
|
|
370
|
+
"workloads": {
|
|
371
|
+
"type": "array",
|
|
372
|
+
"description": "Story #5668. Suite commands declared safe to run alongside others. A spawn whose argv (whitespace-normalized) EXACTLY equals `command` takes that `weight` and `resources`; anything else — and every spawn when this list is empty, the default — runs exclusively (full capacity, today's serialization). Isolation is never inferred from a flag, prefix or diff size. `weight` must count the run's nested worker pool (`--workers 4` → 4), and a full suite needs its own entry before anything can overlap it. Two runs naming a common `resources` entry (a database fixture, a shared coverage file) never overlap; worktree-local outputs need none. A weight above `capacity` refuses the run with exit 78 — nothing spawns, no credit is recorded. An expired wait still exits 75, resumably. Scheduling never changes a run's credit identity. `MANDREL_FULL_SUITE_LOCK=0` (one invocation) or `fullSuiteLock: false` turns admission off entirely; reach for the env hatch only when you have checked the runs share nothing.",
|
|
373
|
+
"items": {
|
|
374
|
+
"type": "object",
|
|
375
|
+
"required": ["command", "weight"],
|
|
376
|
+
"properties": {
|
|
377
|
+
"command": {
|
|
378
|
+
"type": "string",
|
|
379
|
+
"not": {
|
|
380
|
+
"pattern": "([;&|`]|\\$\\()"
|
|
381
|
+
},
|
|
382
|
+
"minLength": 1,
|
|
383
|
+
"description": "The exact suite argv, as `project.commands.test` spells it (or `npm run test:coverage` for coverage capture)."
|
|
384
|
+
},
|
|
385
|
+
"weight": {
|
|
386
|
+
"type": "integer",
|
|
387
|
+
"minimum": 1,
|
|
388
|
+
"description": "CPU workers the run occupies, including its nested pool."
|
|
389
|
+
},
|
|
390
|
+
"resources": {
|
|
391
|
+
"type": "array",
|
|
392
|
+
"uniqueItems": true,
|
|
393
|
+
"items": {
|
|
394
|
+
"type": "string",
|
|
395
|
+
"pattern": "^[a-z0-9][a-z0-9-]*$"
|
|
396
|
+
},
|
|
397
|
+
"description": "Names of shared resources this run needs exclusively."
|
|
398
|
+
}
|
|
399
|
+
},
|
|
400
|
+
"additionalProperties": false
|
|
401
|
+
},
|
|
402
|
+
"default": []
|
|
364
403
|
}
|
|
365
404
|
},
|
|
366
405
|
"additionalProperties": false
|
|
@@ -77,10 +77,10 @@
|
|
|
77
77
|
},
|
|
78
78
|
"gates": {
|
|
79
79
|
"type": "object",
|
|
80
|
-
"description": "Outcome of each close gate this invocation was responsible for. A gate the run skipped (--skip-validation, or a phase it never reached) reports \"skipped\" rather than being omitted, so a missing gate is never mistaken for a passing one. \"overridden\" is distinct from both: the gate ran, it FAILED, and an operator authorized delivery anyway via --override-review-block with a recorded reason. Reporting that as \"passed\" would erase the one fact a reader of this envelope most needs, and reporting it as \"skipped\" would claim the gate never ran.",
|
|
80
|
+
"description": "Outcome of each close gate this invocation was responsible for. A gate the run skipped (--skip-validation, or a phase it never reached) reports \"skipped\" rather than being omitted, so a missing gate is never mistaken for a passing one. \"overridden\" is distinct from both: the gate ran, it FAILED, and an operator authorized delivery anyway via --override-review-block with a recorded reason. Reporting that as \"passed\" would erase the one fact a reader of this envelope most needs, and reporting it as \"skipped\" would claim the gate never ran. \"degraded\" (codeReview only, Story #5663) means a review surface ran but produced no verdict (CLI not logged in, out of quota, unparseable output); it is reported, not blocking, and ranks below \"overridden\".",
|
|
81
81
|
"additionalProperties": {
|
|
82
82
|
"type": "string",
|
|
83
|
-
"enum": ["passed", "failed", "skipped", "overridden"]
|
|
83
|
+
"enum": ["passed", "failed", "skipped", "overridden", "degraded"]
|
|
84
84
|
}
|
|
85
85
|
},
|
|
86
86
|
"tail": {
|
|
@@ -107,6 +107,7 @@ export async function runCoverageCapture(argv = process.argv, deps = {}) {
|
|
|
107
107
|
const locked = lockedCapture(runCaptureImpl, config, env, {
|
|
108
108
|
rerunCommand: rerunCommandFor(argv),
|
|
109
109
|
onLockOutcome: lockHooks.onLockOutcome,
|
|
110
|
+
onLockProgress: lockHooks.onLockProgress,
|
|
110
111
|
...lockOptions,
|
|
111
112
|
});
|
|
112
113
|
const credited = creditedCapture(locked, {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* passed for the current HEAD and tree, and recording a pass for the next
|
|
6
6
|
* caller. `--standalone` is required: it keys evidence by Story id, the same
|
|
7
7
|
* keyspace close consults, so worker-side verify[] runs credit the close.
|
|
8
|
-
* `--gate test`
|
|
8
|
+
* `--gate test` is admitted through the full-suite lock.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { spawnSync } from 'node:child_process';
|
|
@@ -232,11 +232,12 @@ export function runLockedSuite(
|
|
|
232
232
|
const lockHooks = lockEventHooks(claimLockEventSink(env));
|
|
233
233
|
return withFullSuiteLockAsync(
|
|
234
234
|
{
|
|
235
|
-
...fullSuiteLockPolicy(config, env),
|
|
235
|
+
...fullSuiteLockPolicy(config, env, [cmd, ...args]),
|
|
236
236
|
cwd,
|
|
237
237
|
log,
|
|
238
238
|
rerunCommand: ['node', ...process.argv.slice(1)].join(' '),
|
|
239
239
|
onLockOutcome: lockHooks.onLockOutcome,
|
|
240
|
+
onLockProgress: lockHooks.onLockProgress,
|
|
240
241
|
...lockOptions,
|
|
241
242
|
},
|
|
242
243
|
({ lockWaitMs }) =>
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
* (never gate output, which tests can forge). A child claims
|
|
4
|
-
* before spawning, so no suite process can write to it.
|
|
2
|
+
* Lock-wait, suite-timing and live lock-progress events, as NDJSON in a
|
|
3
|
+
* per-run file (never gate output, which tests can forge). A child claims
|
|
4
|
+
* the env var before spawning, so no suite process can write to it.
|
|
5
5
|
*/
|
|
6
6
|
import crypto from 'node:crypto';
|
|
7
7
|
import fs from 'node:fs';
|
|
@@ -10,7 +10,7 @@ import path from 'node:path';
|
|
|
10
10
|
|
|
11
11
|
const CLOSE_LOCK_EVENTS_FILE_ENV = 'MANDREL_CLOSE_LOCK_EVENTS_FILE';
|
|
12
12
|
|
|
13
|
-
const EVENT_KINDS = new Set(['lock-wait', 'suite-timings']);
|
|
13
|
+
const EVENT_KINDS = new Set(['lock-wait', 'suite-timings', 'lock-progress']);
|
|
14
14
|
|
|
15
15
|
function appendEvent(file, event, fsImpl) {
|
|
16
16
|
try {
|
|
@@ -39,6 +39,7 @@ export function createLockEventChannel({
|
|
|
39
39
|
env: { [CLOSE_LOCK_EVENTS_FILE_ENV]: file },
|
|
40
40
|
emit: (event) => appendEvent(file, event, fsImpl),
|
|
41
41
|
read: () => readEvents(file, fsImpl),
|
|
42
|
+
drain: createDrain(file, fsImpl),
|
|
42
43
|
dispose() {
|
|
43
44
|
try {
|
|
44
45
|
fsImpl.rmSync(file, { force: true });
|
|
@@ -47,16 +48,34 @@ export function createLockEventChannel({
|
|
|
47
48
|
};
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
function
|
|
51
|
-
let text;
|
|
51
|
+
function readText(file, fsImpl) {
|
|
52
52
|
try {
|
|
53
|
-
|
|
53
|
+
return fsImpl.readFileSync(file, 'utf8');
|
|
54
54
|
} catch {
|
|
55
|
-
return
|
|
55
|
+
return '';
|
|
56
56
|
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function parseEvents(text) {
|
|
57
60
|
return text.split('\n').filter(Boolean).map(parseEventLine).filter(Boolean);
|
|
58
61
|
}
|
|
59
62
|
|
|
63
|
+
function readEvents(file, fsImpl) {
|
|
64
|
+
return parseEvents(readText(file, fsImpl));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Each call returns only complete lines appended since the last one. */
|
|
68
|
+
function createDrain(file, fsImpl) {
|
|
69
|
+
let consumed = 0;
|
|
70
|
+
return () => {
|
|
71
|
+
const text = readText(file, fsImpl);
|
|
72
|
+
const end = Math.max(consumed, text.lastIndexOf('\n') + 1);
|
|
73
|
+
const fresh = text.slice(consumed, end);
|
|
74
|
+
consumed = end;
|
|
75
|
+
return parseEvents(fresh);
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
60
79
|
/**
|
|
61
80
|
* Remove the channel from `env` and return its emitter (a no-op without one).
|
|
62
81
|
*
|
|
@@ -80,7 +99,7 @@ export function claimLockEventSink(env = process.env, { fsImpl = fs } = {}) {
|
|
|
80
99
|
export function summarizeLockEvents(events) {
|
|
81
100
|
let lockWait = null;
|
|
82
101
|
let suiteTimings = null;
|
|
83
|
-
for (const event of events) {
|
|
102
|
+
for (const event of events.filter((e) => e.kind !== 'lock-progress')) {
|
|
84
103
|
if (event.kind === 'lock-wait') lockWait = mergeLockWait(lockWait, event);
|
|
85
104
|
else suiteTimings = timingsOf(event);
|
|
86
105
|
}
|
|
@@ -104,6 +123,7 @@ function timingsOf({ lockWaitMs, hostWaitMs, testRunMs }) {
|
|
|
104
123
|
export function lockEventHooks(emit) {
|
|
105
124
|
return {
|
|
106
125
|
onLockOutcome: (outcome) => emit({ kind: 'lock-wait', ...outcome }),
|
|
126
|
+
onLockProgress: (progress) => emit({ kind: 'lock-progress', ...progress }),
|
|
107
127
|
onTimings: (timings) => emit({ kind: 'suite-timings', ...timings }),
|
|
108
128
|
};
|
|
109
129
|
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { spawn } from 'node:child_process';
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
|
+
ADMISSION_REFUSED_EXIT_CODE,
|
|
6
7
|
LOCK_WAIT_EXPIRED_EXIT_CODE,
|
|
7
8
|
resolveFullSuiteLockBudget,
|
|
8
9
|
withFullSuiteLockAsync,
|
|
@@ -106,6 +107,7 @@ function gateLockOptions(opts) {
|
|
|
106
107
|
log: opts.log,
|
|
107
108
|
skipIfSatisfied: opts.skipIfSatisfied,
|
|
108
109
|
onWaitExpired: deferredGateStatus,
|
|
110
|
+
onAdmissionRefused: refusedGateStatus,
|
|
109
111
|
onLockOutcome: opts.onLockEvent
|
|
110
112
|
? (outcome) => opts.onLockEvent({ kind: 'lock-wait', ...outcome })
|
|
111
113
|
: undefined,
|
|
@@ -117,6 +119,10 @@ function deferredGateStatus() {
|
|
|
117
119
|
return { status: LOCK_WAIT_EXPIRED_EXIT_CODE };
|
|
118
120
|
}
|
|
119
121
|
|
|
122
|
+
function refusedGateStatus() {
|
|
123
|
+
return { status: ADMISSION_REFUSED_EXIT_CODE };
|
|
124
|
+
}
|
|
125
|
+
|
|
120
126
|
/**
|
|
121
127
|
* @param {string} cmd
|
|
122
128
|
* @param {string[]} args
|
|
@@ -5,7 +5,10 @@
|
|
|
5
5
|
|
|
6
6
|
import { getQuality } from '../config/quality.js';
|
|
7
7
|
import { COVERAGE_TIMEOUT_EXIT_CODE } from '../coverage-capture.js';
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
fullSuiteLockPolicy,
|
|
10
|
+
LOCK_WAIT_EXPIRED_EXIT_CODE,
|
|
11
|
+
} from '../full-suite-lock.js';
|
|
9
12
|
import { gitSpawn } from '../git-utils.js';
|
|
10
13
|
import {
|
|
11
14
|
recordPass as defaultRecordPass,
|
|
@@ -372,7 +375,9 @@ function fullSuiteLockOptions({ config, lockEvents }) {
|
|
|
372
375
|
return {
|
|
373
376
|
...(env ? { env } : {}),
|
|
374
377
|
...(lockEvents ? { onLockEvent: lockEvents.emit } : {}),
|
|
375
|
-
...(gate.fullSuiteLock
|
|
378
|
+
...(gate.fullSuiteLock
|
|
379
|
+
? fullSuiteGateOptions(gate, timeoutMs, config)
|
|
380
|
+
: {}),
|
|
376
381
|
};
|
|
377
382
|
},
|
|
378
383
|
};
|
|
@@ -385,14 +390,25 @@ function gateEnv(gate, lockEvents) {
|
|
|
385
390
|
}
|
|
386
391
|
|
|
387
392
|
/**
|
|
388
|
-
* A full-suite gate's kill bound,
|
|
393
|
+
* A full-suite gate's kill bound, the command its lock-expiry line names, and
|
|
394
|
+
* the admission policy evidence-gate and coverage-capture resolve too — so
|
|
395
|
+
* `fullSuiteLock: false` / `MANDREL_FULL_SUITE_LOCK=0` reach close as well.
|
|
389
396
|
*
|
|
390
397
|
* @param {{ cmd: string, args?: string[] }} gate
|
|
391
398
|
* @param {number} timeoutMs
|
|
399
|
+
* @param {object|null} config
|
|
392
400
|
*/
|
|
393
|
-
function fullSuiteGateOptions(gate, timeoutMs) {
|
|
394
|
-
const
|
|
395
|
-
|
|
401
|
+
function fullSuiteGateOptions(gate, timeoutMs, config) {
|
|
402
|
+
const argv = [gate.cmd, ...(gate.args ?? [])];
|
|
403
|
+
const { enabled, capacity, workload } = fullSuiteLockPolicy(
|
|
404
|
+
config,
|
|
405
|
+
process.env,
|
|
406
|
+
argv,
|
|
407
|
+
);
|
|
408
|
+
return {
|
|
409
|
+
timeoutMs,
|
|
410
|
+
lockOptions: { rerunCommand: argv.join(' '), enabled, capacity, workload },
|
|
411
|
+
};
|
|
396
412
|
}
|
|
397
413
|
|
|
398
414
|
/**
|
|
@@ -24,14 +24,53 @@ const SAFE_STRING = {
|
|
|
24
24
|
|
|
25
25
|
const EXECUTION_SCHEMA = {
|
|
26
26
|
type: 'object',
|
|
27
|
-
description:
|
|
27
|
+
description:
|
|
28
|
+
'Serialization and capacity admission of the suite spawns delivery drives.',
|
|
28
29
|
properties: {
|
|
29
30
|
fullSuiteLock: {
|
|
30
31
|
type: 'boolean',
|
|
31
32
|
description:
|
|
32
|
-
'Serialize full-suite spawns (`npm test` / `npm run test:coverage`) behind a
|
|
33
|
+
'Serialize full-suite spawns (`npm test` / `npm run test:coverage`) behind a repository-level advisory lock shared by every worktree, so two concurrent deliveries do not run two suites against the same cores (see `capacity` / `workloads` for declared runs that may overlap). The lock queues, it never overlaps: a wait that expires with a live holder spawns nothing and exits 75 (resumable), naming the holder. A dead or non-heartbeating holder is taken over, and a broken lockfile proceeds unserialized, so the lock can never fail a delivery. Set false — or export `MANDREL_FULL_SUITE_LOCK=0` for one invocation — to disable.',
|
|
33
34
|
default: true,
|
|
34
35
|
},
|
|
36
|
+
capacity: {
|
|
37
|
+
type: 'integer',
|
|
38
|
+
minimum: 1,
|
|
39
|
+
description:
|
|
40
|
+
"Story #5668. The admission budget, in weight units (CPU workers), shared by every suite spawn across this repository's worktrees and sessions — runs in other repositories on the host are not counted. Unset → the host's `os.availableParallelism()`. A declared workload is admitted while the live weights plus its own fit, strictly first-in-first-out: nothing overtakes an earlier live waiter, so a queued full suite is never starved.",
|
|
41
|
+
},
|
|
42
|
+
workloads: {
|
|
43
|
+
type: 'array',
|
|
44
|
+
description:
|
|
45
|
+
"Story #5668. Suite commands declared safe to run alongside others. A spawn whose argv (whitespace-normalized) EXACTLY equals `command` takes that `weight` and `resources`; anything else — and every spawn when this list is empty, the default — runs exclusively (full capacity, today's serialization). Isolation is never inferred from a flag, prefix or diff size. `weight` must count the run's nested worker pool (`--workers 4` → 4), and a full suite needs its own entry before anything can overlap it. Two runs naming a common `resources` entry (a database fixture, a shared coverage file) never overlap; worktree-local outputs need none. A weight above `capacity` refuses the run with exit 78 — nothing spawns, no credit is recorded. An expired wait still exits 75, resumably. Scheduling never changes a run's credit identity. `MANDREL_FULL_SUITE_LOCK=0` (one invocation) or `fullSuiteLock: false` turns admission off entirely; reach for the env hatch only when you have checked the runs share nothing.",
|
|
46
|
+
items: {
|
|
47
|
+
type: 'object',
|
|
48
|
+
required: ['command', 'weight'],
|
|
49
|
+
properties: {
|
|
50
|
+
command: {
|
|
51
|
+
...SAFE_STRING,
|
|
52
|
+
minLength: 1,
|
|
53
|
+
description:
|
|
54
|
+
'The exact suite argv, as `project.commands.test` spells it (or `npm run test:coverage` for coverage capture).',
|
|
55
|
+
},
|
|
56
|
+
weight: {
|
|
57
|
+
type: 'integer',
|
|
58
|
+
minimum: 1,
|
|
59
|
+
description:
|
|
60
|
+
'CPU workers the run occupies, including its nested pool.',
|
|
61
|
+
},
|
|
62
|
+
resources: {
|
|
63
|
+
type: 'array',
|
|
64
|
+
uniqueItems: true,
|
|
65
|
+
items: { type: 'string', pattern: '^[a-z0-9][a-z0-9-]*$' },
|
|
66
|
+
description:
|
|
67
|
+
'Names of shared resources this run needs exclusively.',
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
additionalProperties: false,
|
|
71
|
+
},
|
|
72
|
+
default: [],
|
|
73
|
+
},
|
|
35
74
|
},
|
|
36
75
|
additionalProperties: false,
|
|
37
76
|
};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* full-suite-lock.js —
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* full-suite-lock.js — admit suite spawns across one repository's worktrees
|
|
3
|
+
* (they contend for cores and shared artifacts), over the `sweep-lock.js`
|
|
4
|
+
* primitive. An undeclared spawn runs exclusively; a declared one is admitted
|
|
5
|
+
* against a capacity budget by `suite-admission.js`.
|
|
5
6
|
*
|
|
6
7
|
* It queues, never overlaps: an expired wait behind a live holder spawns
|
|
7
8
|
* nothing ({@link LOCK_WAIT_EXPIRED_EXIT_CODE}). A dead holder is taken over
|
|
@@ -16,6 +17,17 @@ import { getQuality } from './config/quality.js';
|
|
|
16
17
|
import { mainCheckoutRoot } from './config/temp-paths.js';
|
|
17
18
|
import { isFirstInLine, waitInLine } from './full-suite-queue.js';
|
|
18
19
|
import { acquireSweepLock } from './single-story-sweep/sweep-lock.js';
|
|
20
|
+
import {
|
|
21
|
+
ADMISSION_REFUSED_EXIT_CODE,
|
|
22
|
+
completedProgress,
|
|
23
|
+
createAttempt,
|
|
24
|
+
describeRefusal,
|
|
25
|
+
FULL_SUITE_STALE_MS,
|
|
26
|
+
resolveAdmission,
|
|
27
|
+
resolveSuiteCapacity,
|
|
28
|
+
} from './suite-admission.js';
|
|
29
|
+
|
|
30
|
+
export { ADMISSION_REFUSED_EXIT_CODE };
|
|
19
31
|
|
|
20
32
|
/** Environment escape hatch: set to `0`/`false`/`off`/`no` to disable. */
|
|
21
33
|
export const FULL_SUITE_LOCK_ENV = 'MANDREL_FULL_SUITE_LOCK';
|
|
@@ -38,6 +50,9 @@ const FALSEY = /^(0|false|off|no)$/i;
|
|
|
38
50
|
|
|
39
51
|
const LOCK_TAG = '[full-suite-lock]';
|
|
40
52
|
|
|
53
|
+
/** The argv coverage capture spawns — its workload identity. */
|
|
54
|
+
const COVERAGE_CAPTURE_ARGV = Object.freeze(['npm', 'run', 'test:coverage']);
|
|
55
|
+
|
|
41
56
|
const LOCK_DEFAULTS = Object.freeze({
|
|
42
57
|
enabled: true,
|
|
43
58
|
log: () => {},
|
|
@@ -49,23 +64,26 @@ const LOCK_DEFAULTS = Object.freeze({
|
|
|
49
64
|
sleepFn: (ms) => defaultSleep(ms),
|
|
50
65
|
acquireOnceFn: (opts) => acquireSweepLock(opts),
|
|
51
66
|
onWaitExpired: () => LOCK_WAIT_EXPIRED_EXIT_CODE,
|
|
67
|
+
onAdmissionRefused: () => ADMISSION_REFUSED_EXIT_CODE,
|
|
68
|
+
workload: Object.freeze({ kind: 'exclusive' }),
|
|
69
|
+
capacity: resolveSuiteCapacity(),
|
|
52
70
|
onLockOutcome: () => {},
|
|
71
|
+
onLockProgress: () => {},
|
|
53
72
|
rerunCommand: 'the same command',
|
|
54
73
|
});
|
|
55
74
|
|
|
56
75
|
/**
|
|
57
|
-
* The one wait budget every
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
* holder (heartbeating at a third of its own threshold) is never read stale.
|
|
76
|
+
* The one wait budget every lock taker uses: a holder's suite is killed at
|
|
77
|
+
* its supervisor's `killBoundMs` (the coverage gate's `timeoutMs`). Staleness
|
|
78
|
+
* is fixed, never derived from the waiter's budget — a waiter with a shorter
|
|
79
|
+
* budget than the holder's timeout must not read a live holder stale.
|
|
62
80
|
*
|
|
63
81
|
* @param {number} [killBoundMs]
|
|
64
82
|
* @returns {{ waitMs: number, staleMs: number }}
|
|
65
83
|
*/
|
|
66
84
|
export function resolveFullSuiteLockBudget(killBoundMs) {
|
|
67
85
|
const waitMs = Math.max(MIN_WAIT_MS, Number(killBoundMs) || 0);
|
|
68
|
-
return { waitMs, staleMs:
|
|
86
|
+
return { waitMs, staleMs: FULL_SUITE_STALE_MS };
|
|
69
87
|
}
|
|
70
88
|
|
|
71
89
|
/**
|
|
@@ -100,28 +118,34 @@ export function resolveFullSuiteLockPath({
|
|
|
100
118
|
/**
|
|
101
119
|
* Uncontended fast path, skipped when waiters are queued (no overtaking).
|
|
102
120
|
*
|
|
103
|
-
* @returns {{ lock: object|null, lockPath: string|null
|
|
104
|
-
* `lockPath` means wait;
|
|
121
|
+
* @returns {{ attempt: object|null, lock: object|null, lockPath: string|null,
|
|
122
|
+
* blocking?: object }} null `lock` with a `lockPath` means wait; a null
|
|
123
|
+
* `lockPath` means no lock.
|
|
105
124
|
*/
|
|
106
|
-
function beginLock(
|
|
107
|
-
const
|
|
108
|
-
if (
|
|
109
|
-
|
|
110
|
-
|
|
125
|
+
function beginLock(opts) {
|
|
126
|
+
const lockPath = lockHome(opts);
|
|
127
|
+
if (lockPath === null) return NO_LOCK;
|
|
128
|
+
const attempt = createAttempt({ ...opts, lockPath });
|
|
129
|
+
return { attempt, lockPath, ...firstTry(opts, attempt, lockPath) };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const NO_LOCK = Object.freeze({ attempt: null, lock: null, lockPath: null });
|
|
133
|
+
|
|
134
|
+
const QUEUED = Object.freeze({ reason: 'queue' });
|
|
135
|
+
|
|
136
|
+
function firstTry({ staleMs, fsImpl }, attempt, lockPath) {
|
|
137
|
+
if (!isFirstInLine({ lockPath, ticket: null, staleMs, fsImpl })) {
|
|
138
|
+
return { lock: null, blocking: QUEUED };
|
|
111
139
|
}
|
|
112
|
-
|
|
113
|
-
lockPath: resolved,
|
|
114
|
-
timeoutMs: staleMs,
|
|
115
|
-
fsImpl,
|
|
116
|
-
});
|
|
117
|
-
if (first.acquired) return { lock: first, lockPath: resolved };
|
|
118
|
-
// A hard I/O error will not resolve by waiting — proceed unserialized.
|
|
119
|
-
return first.reason === 'error'
|
|
120
|
-
? NO_LOCK
|
|
121
|
-
: { lock: null, lockPath: resolved };
|
|
140
|
+
return afterFirst(attempt.tryOnce());
|
|
122
141
|
}
|
|
123
142
|
|
|
124
|
-
|
|
143
|
+
/** A hard I/O error will not resolve by waiting — proceed unserialized. */
|
|
144
|
+
function afterFirst(first) {
|
|
145
|
+
if (first.acquired) return { lock: first };
|
|
146
|
+
if (first.reason === 'error') return { lock: null, lockPath: null };
|
|
147
|
+
return { lock: null, blocking: first.blocking };
|
|
148
|
+
}
|
|
125
149
|
|
|
126
150
|
/**
|
|
127
151
|
* @param {{ enabled: boolean, cwd: string, lockPath?: string }} where
|
|
@@ -170,6 +194,7 @@ function consult(probe, applies) {
|
|
|
170
194
|
* skipIfSatisfied?: () => T|undefined,
|
|
171
195
|
* onWaitExpired?: (holder: import('./full-suite-queue.js').LockHolder) => T,
|
|
172
196
|
* onLockOutcome?: (outcome: object) => void,
|
|
197
|
+
* onLockProgress?: (progress: import('./full-suite-queue.js').LockProgress) => void,
|
|
173
198
|
* rerunCommand?: string,
|
|
174
199
|
* }} opts A non-`undefined` `skipIfSatisfied` return stands in for the spawn.
|
|
175
200
|
* @param {(timing: { lockWaitMs: number }) => Promise<T>} spawn
|
|
@@ -177,19 +202,33 @@ function consult(probe, applies) {
|
|
|
177
202
|
*/
|
|
178
203
|
export async function withFullSuiteLockAsync(options, spawn) {
|
|
179
204
|
const opts = withDefaults(options);
|
|
180
|
-
|
|
181
|
-
const
|
|
182
|
-
lock === null && lockPath !== null
|
|
183
|
-
? await waitInLine({ ...opts, lockPath })
|
|
184
|
-
: { held: lock, expired: false, waited: false, waitedMs: 0 };
|
|
185
|
-
reportOutcome(opts.onLockOutcome, wait);
|
|
205
|
+
if (isRefused(opts)) return refuse(opts);
|
|
206
|
+
const begun = beginLock(opts);
|
|
186
207
|
try {
|
|
208
|
+
const wait = await awaitTurn(opts, begun);
|
|
209
|
+
reportOutcome(opts.onLockOutcome, wait);
|
|
187
210
|
return await spawnOrStandIn(opts, wait, spawn);
|
|
188
211
|
} finally {
|
|
189
|
-
|
|
212
|
+
begun.attempt?.release();
|
|
190
213
|
}
|
|
191
214
|
}
|
|
192
215
|
|
|
216
|
+
function isRefused({ enabled, workload }) {
|
|
217
|
+
return enabled && workload?.kind === 'refused';
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function refuse(opts) {
|
|
221
|
+
opts.log(describeRefusal(opts.workload));
|
|
222
|
+
return opts.onAdmissionRefused(opts.workload);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function awaitTurn(opts, { attempt, lock, lockPath, blocking }) {
|
|
226
|
+
if (lock !== null || lockPath === null) {
|
|
227
|
+
return { held: lock, expired: false, waited: false, waitedMs: 0 };
|
|
228
|
+
}
|
|
229
|
+
return waitInLine({ ...opts, lockPath, attempt, blocking });
|
|
230
|
+
}
|
|
231
|
+
|
|
193
232
|
/** An I/O-error wait is no outcome; a throwing callback never costs the spawn. */
|
|
194
233
|
function reportOutcome(onLockOutcome, wait) {
|
|
195
234
|
if (!isFinishedWait(wait)) return;
|
|
@@ -223,8 +262,23 @@ async function spawnOrStandIn(opts, wait, spawn) {
|
|
|
223
262
|
);
|
|
224
263
|
return probe.value;
|
|
225
264
|
}
|
|
226
|
-
|
|
227
|
-
|
|
265
|
+
return wait.expired
|
|
266
|
+
? opts.onWaitExpired(wait.holder)
|
|
267
|
+
: await timedSpawn(opts, wait, spawn);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Wait and execution reach progress as separate figures. */
|
|
271
|
+
async function timedSpawn(opts, wait, spawn) {
|
|
272
|
+
const startedAt = opts.nowFn();
|
|
273
|
+
const result = await spawn({ lockWaitMs: wait.waitedMs ?? 0 });
|
|
274
|
+
if (wait.waited) reportCompleted(opts, wait, opts.nowFn() - startedAt);
|
|
275
|
+
return result;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function reportCompleted(opts, wait, ranMs) {
|
|
279
|
+
try {
|
|
280
|
+
opts.onLockProgress(completedProgress(wait.waitedMs, opts.waitMs, ranMs));
|
|
281
|
+
} catch {}
|
|
228
282
|
}
|
|
229
283
|
|
|
230
284
|
/**
|
|
@@ -238,14 +292,19 @@ function defaultSleep(ms) {
|
|
|
238
292
|
}
|
|
239
293
|
|
|
240
294
|
/**
|
|
295
|
+
* The one admission policy every caller resolves: enablement, budget,
|
|
296
|
+
* capacity, and the workload `argv` classifies to.
|
|
297
|
+
*
|
|
241
298
|
* @param {object} [config]
|
|
242
299
|
* @param {Record<string, string|undefined>} [env]
|
|
300
|
+
* @param {string[]} [argv]
|
|
243
301
|
*/
|
|
244
|
-
export function fullSuiteLockPolicy(config, env = process.env) {
|
|
302
|
+
export function fullSuiteLockPolicy(config, env = process.env, argv) {
|
|
245
303
|
return {
|
|
246
304
|
...resolveFullSuiteLockBudget(getQuality(config).coverage?.timeoutMs),
|
|
247
305
|
enabled: isFullSuiteLockEnabled({ config, env }),
|
|
248
306
|
onWaitExpired: () => LOCK_WAIT_EXPIRED_EXIT_CODE,
|
|
307
|
+
...resolveAdmission(config, argv),
|
|
249
308
|
};
|
|
250
309
|
}
|
|
251
310
|
|
|
@@ -265,7 +324,10 @@ export function lockedCapture(
|
|
|
265
324
|
env = process.env,
|
|
266
325
|
lockOptions = {},
|
|
267
326
|
) {
|
|
268
|
-
const policy = {
|
|
327
|
+
const policy = {
|
|
328
|
+
...fullSuiteLockPolicy(config, env, COVERAGE_CAPTURE_ARGV),
|
|
329
|
+
...lockOptions,
|
|
330
|
+
};
|
|
269
331
|
return (captureOpts = {}) =>
|
|
270
332
|
withFullSuiteLockAsync(
|
|
271
333
|
{
|