instar 1.3.735 → 1.3.737
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/devPreflight.d.ts +21 -0
- package/dist/commands/devPreflight.d.ts.map +1 -1
- package/dist/commands/devPreflight.js +36 -2
- package/dist/commands/devPreflight.js.map +1 -1
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +30 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +13 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/PostUpdateMigrator.d.ts +1 -0
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +73 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/SessionManager.d.ts.map +1 -1
- package/dist/core/SessionManager.js +12 -0
- package/dist/core/SessionManager.js.map +1 -1
- package/dist/core/WriteDomainRegistry.d.ts.map +1 -1
- package/dist/core/WriteDomainRegistry.js +22 -0
- package/dist/core/WriteDomainRegistry.js.map +1 -1
- package/dist/core/hostSemaphoreCore.d.ts +115 -0
- package/dist/core/hostSemaphoreCore.d.ts.map +1 -0
- package/dist/core/hostSemaphoreCore.js +215 -0
- package/dist/core/hostSemaphoreCore.js.map +1 -0
- package/dist/core/hostSpawnSemaphore.d.ts +8 -9
- package/dist/core/hostSpawnSemaphore.d.ts.map +1 -1
- package/dist/core/hostSpawnSemaphore.js +48 -126
- package/dist/core/hostSpawnSemaphore.js.map +1 -1
- package/dist/core/hostTestRunnerSemaphore.d.ts +492 -0
- package/dist/core/hostTestRunnerSemaphore.d.ts.map +1 -0
- package/dist/core/hostTestRunnerSemaphore.js +1648 -0
- package/dist/core/hostTestRunnerSemaphore.js.map +1 -0
- package/dist/core/testRunnerRunClassifier.d.ts +128 -0
- package/dist/core/testRunnerRunClassifier.d.ts.map +1 -0
- package/dist/core/testRunnerRunClassifier.js +382 -0
- package/dist/core/testRunnerRunClassifier.js.map +1 -0
- package/dist/core/types.d.ts +21 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/monitoring/guardManifest.d.ts.map +1 -1
- package/dist/monitoring/guardManifest.js +26 -0
- package/dist/monitoring/guardManifest.js.map +1 -1
- package/dist/scaffold/templates.d.ts.map +1 -1
- package/dist/scaffold/templates.js +7 -0
- package/dist/scaffold/templates.js.map +1 -1
- package/dist/server/CapabilityIndex.d.ts.map +1 -1
- package/dist/server/CapabilityIndex.js +9 -0
- package/dist/server/CapabilityIndex.js.map +1 -1
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +150 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/scripts/lib/test-runner-selfdisable-patterns.mjs +326 -0
- package/scripts/pre-push-test-runner-selfdisable.mjs +81 -0
- package/skills/spec-converge/SKILL.md +2 -1
- package/skills/spec-converge/templates/reviewer-integration.md +8 -6
- package/src/data/builtin-manifest.json +64 -64
- package/src/scaffold/templates.ts +7 -0
- package/upgrades/1.3.737.md +140 -0
- package/upgrades/side-effects/test-runner-concurrency-bound.md +173 -0
- package/upgrades/side-effects/three-standards-enforcement.md +131 -0
|
@@ -0,0 +1,1648 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hostTestRunnerSemaphore — the host-wide vitest concurrency bound (test lane).
|
|
3
|
+
*
|
|
4
|
+
* Spec: docs/specs/test-runner-concurrency-bound.md (converged round 10,
|
|
5
|
+
* operator-ratified: suite cap 1, ship posture dry-run/watch-only).
|
|
6
|
+
* Constitutional standard: Bounded Blast Radius (the 2nd instance of the
|
|
7
|
+
* unbounded-per-actor-spawn class; the 1st is hostSpawnSemaphore).
|
|
8
|
+
*
|
|
9
|
+
* THE LOAD-BEARING DESIGN DECISION (§1.1): the fail-direction INVERTS from the
|
|
10
|
+
* spawn cap. This lane fails OPEN — toward ADMITTING a run — on every
|
|
11
|
+
* *provable, persistent* uncertainty (corrupt holders file, df-unconfirmed
|
|
12
|
+
* disk, provably-wedged lock, unresolvable ancestry). A false BLOCK wedges
|
|
13
|
+
* every `git push` and `/build` gate host-wide with no degradation path; a
|
|
14
|
+
* false PASS is one extra concurrent suite. A single missed lock attempt is
|
|
15
|
+
* "keep polling", never "admit" (§2.4 fail-open granularity).
|
|
16
|
+
*
|
|
17
|
+
* TWO SYMMETRIC LANES (§2.3): suite-class (cap INSTAR_HOST_TEST_MAX, default
|
|
18
|
+
* 1) and targeted-run (cap INSTAR_HOST_TEST_TARGETED_MAX, default 6). A full
|
|
19
|
+
* targeted lane THROWS the typed capacity-timeout — it never fail-open-admits
|
|
20
|
+
* (fail-open-admit is lock-wedge-only).
|
|
21
|
+
*
|
|
22
|
+
* RECLAIM POLICY (§2.4, NOT the spawn defaults): immediate reclaim of a
|
|
23
|
+
* provably-dead pid; start-time corroboration reclaims a REUSED pid pre-TTL;
|
|
24
|
+
* a max-hold TTL frees even a pid-alive holder's slot (capacity-reclaim-ONLY
|
|
25
|
+
* by default — NO process is ever signaled unless the separate, opt-in,
|
|
26
|
+
* tuning-file-armed INSTAR_HOST_TEST_TTL_SIGNAL arm is on AND posture is
|
|
27
|
+
* enforcing, and even then only after four mandatory gates).
|
|
28
|
+
*
|
|
29
|
+
* All cross-actor levers (posture, both caps, the signal arm) take their
|
|
30
|
+
* host-uniform authority from ~/.instar/host-test-runner-tuning.json; env
|
|
31
|
+
* vars are per-process overrides that are honored but LOUD when divergent
|
|
32
|
+
* (§2.9). Every consequential decision is appended to the durable event
|
|
33
|
+
* ledger (§2.8) — the soak's evidence store.
|
|
34
|
+
*/
|
|
35
|
+
import fs from 'node:fs';
|
|
36
|
+
import os from 'node:os';
|
|
37
|
+
import path from 'node:path';
|
|
38
|
+
import crypto from 'node:crypto';
|
|
39
|
+
import { execFileSync } from 'node:child_process';
|
|
40
|
+
import { SafeFsExecutor } from './SafeFsExecutor.js';
|
|
41
|
+
import { isAlive } from './ProjectRoundLock.js';
|
|
42
|
+
import { atomicWriteFileSync, probeDfHostLocalDetailed, releaseLock, tryTakeLockOnce, } from './hostSemaphoreCore.js';
|
|
43
|
+
// ── Pinned code constants (frozen by the spec — NOT tunable) ──────────────
|
|
44
|
+
/** Suite-lane cap code-default (operator-ratified: full suites one-at-a-time). */
|
|
45
|
+
export const HOST_TEST_SUITE_CAP_DEFAULT = 1;
|
|
46
|
+
/** Targeted-lane cap code-default (§2.3). */
|
|
47
|
+
export const HOST_TEST_TARGETED_CAP_DEFAULT = 6;
|
|
48
|
+
/** Tuning-file cap sanity ceilings — 4× each code-default (§2.9, pinned). */
|
|
49
|
+
export const HOST_TEST_SUITE_CAP_CEILING = 4;
|
|
50
|
+
export const HOST_TEST_TARGETED_CAP_CEILING = 24;
|
|
51
|
+
/** Max-hold TTL code-default (1h). */
|
|
52
|
+
export const HOST_TEST_TTL_DEFAULT_MS = 3_600_000;
|
|
53
|
+
/** ttlMs sanity RANGE — pinned code constants, deliberately NOT tunable (§2.4). */
|
|
54
|
+
export const HOST_TEST_TTL_MIN_MS = 300_000; // 5 min floor
|
|
55
|
+
export const HOST_TEST_TTL_MAX_MS = 14_400_000; // 4h = 4× default
|
|
56
|
+
/** Poison sanity ceiling on holders-file row count (§2.4, pinned round 6). */
|
|
57
|
+
export const HOST_TEST_POISON_CEILING = 64;
|
|
58
|
+
/** Lockless fail-open storm ceiling — O_EXCL witness slots (§2.4, pinned). */
|
|
59
|
+
export const WEDGE_STORM_CEILING = 8;
|
|
60
|
+
/** Background-class suite-lane wait budget default (2 min, fail-loud). */
|
|
61
|
+
export const HOST_TEST_ACQUIRE_MS_DEFAULT = 120_000;
|
|
62
|
+
/** Targeted-lane wait budget default (1 min, fail-loud THROW). */
|
|
63
|
+
export const HOST_TEST_TARGETED_ACQUIRE_MS_DEFAULT = 60_000;
|
|
64
|
+
/** A lock older than this is provably wedged (critical section is sub-ms). */
|
|
65
|
+
export const LOCK_WEDGE_AGE_MS = 10_000;
|
|
66
|
+
/** Per-attempt lock-acquire deadline (≫ the sub-ms critical section). */
|
|
67
|
+
export const LOCK_ACQUIRE_DEADLINE_MS = 250;
|
|
68
|
+
/** Acquisition poll cadence (async yielding wait — NEVER a busy spin). */
|
|
69
|
+
export const POLL_INTERVAL_MS = 5_000;
|
|
70
|
+
export const POLL_JITTER_MS = 1_000;
|
|
71
|
+
/** SIGTERM→SIGKILL grace window under the (opt-in) signal arm. */
|
|
72
|
+
export const TOMBSTONE_GRACE_MS = 30_000;
|
|
73
|
+
/** Ledger rotation threshold (~5MB; segments retained per §2.8). */
|
|
74
|
+
export const LEDGER_ROTATE_BYTES = 5 * 1024 * 1024;
|
|
75
|
+
/** Segment retention floor AFTER the enforce-flip decision is recorded. */
|
|
76
|
+
export const LEDGER_SEGMENT_FLOOR = 10;
|
|
77
|
+
/** Clock-skew allowance for start-time corroboration. */
|
|
78
|
+
export const START_TIME_SKEW_MS = 120_000;
|
|
79
|
+
/** 80%-of-TTL warning fraction (§2.4). */
|
|
80
|
+
export const TTL_WARN_FRACTION = 0.8;
|
|
81
|
+
/** Targeted classification file-count limit K (§2.3). */
|
|
82
|
+
export const TARGETED_FILE_LIMIT = 5;
|
|
83
|
+
/**
|
|
84
|
+
* Resolve the rendezvous base dir. `INSTAR_HOST_TEST_BASE_DIR` is an INTERNAL
|
|
85
|
+
* test seam (meta-tests isolate their spawned roots into a temp universe) —
|
|
86
|
+
* it is NOT a public lever and is deliberately undocumented in §2.9.
|
|
87
|
+
*/
|
|
88
|
+
export function resolveTestRunnerPaths(env = process.env) {
|
|
89
|
+
const override = env['INSTAR_HOST_TEST_BASE_DIR'];
|
|
90
|
+
const baseDir = override && override.trim() ? override.trim() : path.join(os.homedir(), '.instar');
|
|
91
|
+
return {
|
|
92
|
+
baseDir,
|
|
93
|
+
holders: path.join(baseDir, 'host-test-runner-holders.json'),
|
|
94
|
+
lock: path.join(baseDir, 'host-test-runner-holders.lock'),
|
|
95
|
+
witnessDir: path.join(baseDir, 'host-test-runner-witness'),
|
|
96
|
+
tuning: path.join(baseDir, 'host-test-runner-tuning.json'),
|
|
97
|
+
tuningBaseline: path.join(baseDir, 'host-test-runner-tuning-baseline.json'),
|
|
98
|
+
dfMarker: path.join(baseDir, 'host-test-runner-dflocal.json'),
|
|
99
|
+
ledger: path.join(baseDir, 'host-test-runner-events.jsonl'),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
// ── Typed errors (§2.6) ───────────────────────────────────────────────────
|
|
103
|
+
/** Distinct exit code for capacity refusals (EX_TEMPFAIL — NOT a test failure). */
|
|
104
|
+
export const TEST_RUNNER_CAPACITY_EXIT_CODE = 75;
|
|
105
|
+
export class TestRunnerCapacityTimeoutError extends Error {
|
|
106
|
+
code = 'INSTAR_TEST_CAPACITY_TIMEOUT';
|
|
107
|
+
exitCode = TEST_RUNNER_CAPACITY_EXIT_CODE;
|
|
108
|
+
holders;
|
|
109
|
+
constructor(lane, budgetMs, holders) {
|
|
110
|
+
super(`[test-runner-bound] could not START within budget (${budgetMs}ms, ${lane} lane) — ` +
|
|
111
|
+
`this is NOT a test failure; ${holders.length} holder(s): ` +
|
|
112
|
+
`[${holders.map((h) => `pid ${h.pid}, age ${Math.round(h.ageMs / 1000)}s`).join('; ')}]; ` +
|
|
113
|
+
`levers: INSTAR_HOST_TEST_SEMAPHORE=off, INSTAR_HOST_TEST_MAX`);
|
|
114
|
+
this.name = 'TestRunnerCapacityTimeoutError';
|
|
115
|
+
this.holders = holders;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
export class TestRunnerStormCeilingError extends Error {
|
|
119
|
+
code = 'INSTAR_TEST_STORM_CEILING';
|
|
120
|
+
exitCode = TEST_RUNNER_CAPACITY_EXIT_CODE;
|
|
121
|
+
slots;
|
|
122
|
+
constructor(slots) {
|
|
123
|
+
super(`[test-runner-bound] wedge-storm ceiling (${WEDGE_STORM_CEILING}) reached — refusing a further ` +
|
|
124
|
+
`fail-open admit; held slots: ` +
|
|
125
|
+
`[${slots.map((s) => `slot ${s.slot}: pid ${s.pid}, age ${Math.round(s.ageMs / 1000)}s`).join('; ')}]. ` +
|
|
126
|
+
`This is NOT a test failure — the holders lock has been wedged at storm volume; ` +
|
|
127
|
+
`levers: INSTAR_HOST_TEST_SEMAPHORE=off`);
|
|
128
|
+
this.name = 'TestRunnerStormCeilingError';
|
|
129
|
+
this.slots = slots;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// ── Resolvers (env override → tuning file → code default; §2.9) ──────────
|
|
133
|
+
/**
|
|
134
|
+
* Coerce a raw ttlMs to a finite integer in [HOST_TEST_TTL_MIN_MS,
|
|
135
|
+
* HOST_TEST_TTL_MAX_MS]. Uses Number() + Number.isInteger — NEVER parseInt
|
|
136
|
+
* (§2.4: "300000abc"/"300000.9" are definitively REJECTED to the default, not
|
|
137
|
+
* truncated into range). Out-of-range/NaN/non-integer ⇒ code-default + coerced.
|
|
138
|
+
*/
|
|
139
|
+
export function coerceTtlMs(raw) {
|
|
140
|
+
const n = typeof raw === 'string' ? Number(raw) : typeof raw === 'number' ? raw : NaN;
|
|
141
|
+
if (!Number.isFinite(n) || !Number.isInteger(n)) {
|
|
142
|
+
return { ttlMs: HOST_TEST_TTL_DEFAULT_MS, coerced: true };
|
|
143
|
+
}
|
|
144
|
+
if (n < HOST_TEST_TTL_MIN_MS || n > HOST_TEST_TTL_MAX_MS) {
|
|
145
|
+
// A value BEYOND the ceiling is clamped AT the ceiling for a row read
|
|
146
|
+
// (no immortal slot); anything below the floor resolves to the default
|
|
147
|
+
// (instant-expiry abuse). Both are "coerced" (WARN-ledgered by callers).
|
|
148
|
+
if (n > HOST_TEST_TTL_MAX_MS)
|
|
149
|
+
return { ttlMs: HOST_TEST_TTL_MAX_MS, coerced: true };
|
|
150
|
+
return { ttlMs: HOST_TEST_TTL_DEFAULT_MS, coerced: true };
|
|
151
|
+
}
|
|
152
|
+
return { ttlMs: n, coerced: false };
|
|
153
|
+
}
|
|
154
|
+
/** Sanity-clamp a tuning-file cap value to an integer in [1, ceiling] (§2.9). */
|
|
155
|
+
export function sanitizeCapValue(raw, ceiling, dflt) {
|
|
156
|
+
const n = typeof raw === 'string' ? Number(raw) : typeof raw === 'number' ? raw : NaN;
|
|
157
|
+
if (!Number.isFinite(n) || !Number.isInteger(n) || n < 1 || n > ceiling) {
|
|
158
|
+
return { value: dflt, coerced: raw !== undefined };
|
|
159
|
+
}
|
|
160
|
+
return { value: n, coerced: false };
|
|
161
|
+
}
|
|
162
|
+
export function resolvePosture(env, tuning) {
|
|
163
|
+
const authority = tuning?.enforcing === true ? 'enforcing' : 'dry-run';
|
|
164
|
+
if ((env['INSTAR_HOST_TEST_SEMAPHORE'] ?? '').toLowerCase() === 'off') {
|
|
165
|
+
return { posture: 'off', authority, divergence: null };
|
|
166
|
+
}
|
|
167
|
+
const raw = env['INSTAR_HOST_TEST_ENFORCE'];
|
|
168
|
+
let override = null;
|
|
169
|
+
if (raw === '1' || raw === 'true')
|
|
170
|
+
override = 'enforcing';
|
|
171
|
+
else if (raw === '0' || raw === 'false')
|
|
172
|
+
override = 'dry-run';
|
|
173
|
+
const posture = override ?? authority;
|
|
174
|
+
let divergence = null;
|
|
175
|
+
if (override && override !== authority) {
|
|
176
|
+
divergence = override === 'enforcing' ? 'stronger' : 'weaker';
|
|
177
|
+
}
|
|
178
|
+
return { posture, authority, divergence };
|
|
179
|
+
}
|
|
180
|
+
/** Are the config-eval clamps REAL (clamp-active sub-stage or enforcing)? §2.11. */
|
|
181
|
+
export function resolveClampActive(posture, tuning) {
|
|
182
|
+
if (posture === 'off')
|
|
183
|
+
return false;
|
|
184
|
+
return posture === 'enforcing' || tuning?.clampActive === true;
|
|
185
|
+
}
|
|
186
|
+
export function resolveCap(lane, env, tuning) {
|
|
187
|
+
const dflt = lane === 'suite' ? HOST_TEST_SUITE_CAP_DEFAULT : HOST_TEST_TARGETED_CAP_DEFAULT;
|
|
188
|
+
const ceiling = lane === 'suite' ? HOST_TEST_SUITE_CAP_CEILING : HOST_TEST_TARGETED_CAP_CEILING;
|
|
189
|
+
const tuningRaw = lane === 'suite' ? tuning?.maxConcurrent : tuning?.targetedMax;
|
|
190
|
+
const sanitized = sanitizeCapValue(tuningRaw, ceiling, dflt);
|
|
191
|
+
const authorityCap = sanitized.value;
|
|
192
|
+
const envRaw = env[lane === 'suite' ? 'INSTAR_HOST_TEST_MAX' : 'INSTAR_HOST_TEST_TARGETED_MAX'];
|
|
193
|
+
if (envRaw !== undefined) {
|
|
194
|
+
const n = Number(envRaw);
|
|
195
|
+
if (Number.isFinite(n) && Number.isInteger(n) && n >= 1) {
|
|
196
|
+
return {
|
|
197
|
+
cap: n,
|
|
198
|
+
source: 'env',
|
|
199
|
+
coerced: false,
|
|
200
|
+
divergentBeyond4x: n > authorityCap * 4,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
// Malformed env value falls through to the authority (never zeroes capacity).
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
cap: authorityCap,
|
|
207
|
+
source: tuningRaw !== undefined && !sanitized.coerced ? 'tuning' : 'default',
|
|
208
|
+
coerced: sanitized.coerced,
|
|
209
|
+
divergentBeyond4x: false,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
export function resolveTtlSignal(env, tuning) {
|
|
213
|
+
const authorityArmed = tuning?.ttlSignal === true;
|
|
214
|
+
const raw = env['INSTAR_HOST_TEST_TTL_SIGNAL'];
|
|
215
|
+
if (raw === '0' || raw === 'false')
|
|
216
|
+
return { armed: false, envArmIgnored: false };
|
|
217
|
+
if ((raw === '1' || raw === 'true') && !authorityArmed) {
|
|
218
|
+
// ASYMMETRIC (§2.9): env can only DISARM — arming is tuning-file-only.
|
|
219
|
+
return { armed: false, envArmIgnored: true };
|
|
220
|
+
}
|
|
221
|
+
return { armed: authorityArmed, envArmIgnored: false };
|
|
222
|
+
}
|
|
223
|
+
/** Resolve the acquire wait budget for a lane + run class (§2.9). */
|
|
224
|
+
export function resolveAcquireBudgetMs(lane, runClass, env = process.env) {
|
|
225
|
+
const readMs = (key) => {
|
|
226
|
+
const raw = env[key];
|
|
227
|
+
if (raw === undefined)
|
|
228
|
+
return null;
|
|
229
|
+
const n = Number(raw);
|
|
230
|
+
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : null;
|
|
231
|
+
};
|
|
232
|
+
if (lane === 'targeted') {
|
|
233
|
+
return readMs('INSTAR_HOST_TEST_TARGETED_ACQUIRE_MS') ?? HOST_TEST_TARGETED_ACQUIRE_MS_DEFAULT;
|
|
234
|
+
}
|
|
235
|
+
const background = readMs('INSTAR_HOST_TEST_ACQUIRE_MS') ?? HOST_TEST_ACQUIRE_MS_DEFAULT;
|
|
236
|
+
if (runClass === 'background')
|
|
237
|
+
return background;
|
|
238
|
+
return readMs('INSTAR_HOST_TEST_ACQUIRE_MS_INTERACTIVE') ?? 5 * background;
|
|
239
|
+
}
|
|
240
|
+
/** Stamped per-row ttl at acquire (env-read, sanity-ranged). */
|
|
241
|
+
export function resolveAcquireTtlMs(env = process.env) {
|
|
242
|
+
const raw = env['INSTAR_HOST_TEST_TTL_MS'];
|
|
243
|
+
if (raw === undefined)
|
|
244
|
+
return { ttlMs: HOST_TEST_TTL_DEFAULT_MS, coerced: false };
|
|
245
|
+
return coerceTtlMs(raw);
|
|
246
|
+
}
|
|
247
|
+
// ── Tuning file (host-uniform authority; §2.9) ────────────────────────────
|
|
248
|
+
export const TUNING_HASH_ABSENT = 'absent';
|
|
249
|
+
function hashTuningRaw(raw) {
|
|
250
|
+
return crypto.createHash('sha256').update(raw).digest('hex').slice(0, 12);
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Read the tuning file. Quarantine requires CONFIRMED corruption (an immediate
|
|
254
|
+
* re-read — a transient torn write must never demote the host authority);
|
|
255
|
+
* confirmed-corrupt is quarantined aside + resolved to code defaults (§2.9).
|
|
256
|
+
*/
|
|
257
|
+
export function readTuningFile(paths) {
|
|
258
|
+
let raw;
|
|
259
|
+
try {
|
|
260
|
+
raw = fs.readFileSync(paths.tuning, 'utf-8');
|
|
261
|
+
}
|
|
262
|
+
catch {
|
|
263
|
+
// @silent-fallback-ok: an ABSENT tuning file is the normal fresh-host state
|
|
264
|
+
// — code defaults apply; the 'absent' hash sentinel makes file CREATION
|
|
265
|
+
// detectable as a change (§2.9).
|
|
266
|
+
return { file: null, hash: TUNING_HASH_ABSENT, corrupt: false };
|
|
267
|
+
}
|
|
268
|
+
const parsed = tryParseTuning(raw);
|
|
269
|
+
if (parsed)
|
|
270
|
+
return { file: parsed, hash: hashTuningRaw(raw), corrupt: false };
|
|
271
|
+
// First bad read → confirm with an immediate re-read before quarantining.
|
|
272
|
+
let raw2 = null;
|
|
273
|
+
try {
|
|
274
|
+
raw2 = fs.readFileSync(paths.tuning, 'utf-8');
|
|
275
|
+
}
|
|
276
|
+
catch {
|
|
277
|
+
// @silent-fallback-ok: the file vanished between reads — treat as absent.
|
|
278
|
+
return { file: null, hash: TUNING_HASH_ABSENT, corrupt: false };
|
|
279
|
+
}
|
|
280
|
+
const parsed2 = tryParseTuning(raw2);
|
|
281
|
+
if (parsed2)
|
|
282
|
+
return { file: parsed2, hash: hashTuningRaw(raw2), corrupt: false };
|
|
283
|
+
// CONFIRMED corrupt → quarantine aside (keep newest 5) + code defaults.
|
|
284
|
+
quarantineFileAside(paths.tuning, 'host-test-runner-tuning.corrupt', paths.baseDir);
|
|
285
|
+
return { file: null, hash: TUNING_HASH_ABSENT, corrupt: true };
|
|
286
|
+
}
|
|
287
|
+
function tryParseTuning(raw) {
|
|
288
|
+
try {
|
|
289
|
+
const obj = JSON.parse(raw);
|
|
290
|
+
if (!obj || typeof obj !== 'object' || Array.isArray(obj))
|
|
291
|
+
return null;
|
|
292
|
+
return obj;
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
// @silent-fallback-ok: parse failure is the caller's confirmed-corruption
|
|
296
|
+
// branch — never a throw into the chokepoint.
|
|
297
|
+
return null;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
/** Atomic tuning write (the recommended writer — never a raw shell redirect). */
|
|
301
|
+
export function writeTuningFile(paths, tuning) {
|
|
302
|
+
ensureBaseDir(paths);
|
|
303
|
+
atomicWriteFileSync(paths.tuning, JSON.stringify(tuning, null, 2), {
|
|
304
|
+
mode: 0o600,
|
|
305
|
+
operation: 'hostTestRunnerSemaphore.writeTuningFile',
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
/** Rename a corrupt file aside with a timestamp, retaining only the newest 5. */
|
|
309
|
+
function quarantineFileAside(filePath, quarantinePrefix, baseDir) {
|
|
310
|
+
const target = path.join(baseDir, `${quarantinePrefix}-${Date.now()}.json`);
|
|
311
|
+
try {
|
|
312
|
+
fs.renameSync(filePath, target);
|
|
313
|
+
}
|
|
314
|
+
catch {
|
|
315
|
+
// @silent-fallback-ok: a concurrent quarantine already moved it — fine.
|
|
316
|
+
return null;
|
|
317
|
+
}
|
|
318
|
+
// Keep-newest-5 retention (destructive delete → SafeFsExecutor funnel, L12).
|
|
319
|
+
try {
|
|
320
|
+
const siblings = fs
|
|
321
|
+
.readdirSync(baseDir)
|
|
322
|
+
.filter((f) => f.startsWith(`${quarantinePrefix}-`))
|
|
323
|
+
.sort()
|
|
324
|
+
.reverse();
|
|
325
|
+
for (const stale of siblings.slice(5)) {
|
|
326
|
+
SafeFsExecutor.safeUnlinkSync(path.join(baseDir, stale), {
|
|
327
|
+
operation: 'hostTestRunnerSemaphore.quarantineRetention',
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
/* @silent-fallback-ok: retention is best-effort housekeeping */
|
|
333
|
+
}
|
|
334
|
+
return target;
|
|
335
|
+
}
|
|
336
|
+
/** Best-effort ledger append — a ledger write failure never blocks a run. */
|
|
337
|
+
export function appendLedgerEvent(paths, event, opts = {}) {
|
|
338
|
+
try {
|
|
339
|
+
ensureBaseDir(paths);
|
|
340
|
+
rotateLedgerIfNeeded(paths, opts.flipRecorded === true);
|
|
341
|
+
fs.appendFileSync(paths.ledger, JSON.stringify(event) + '\n', { mode: 0o600 });
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
/* @silent-fallback-ok: best-effort append — never throws into the run (§2.8) */
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
function rotateLedgerIfNeeded(paths, flipRecorded) {
|
|
348
|
+
let size = 0;
|
|
349
|
+
try {
|
|
350
|
+
size = fs.statSync(paths.ledger).size;
|
|
351
|
+
}
|
|
352
|
+
catch {
|
|
353
|
+
// @silent-fallback-ok: no ledger yet — nothing to rotate.
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
if (size < LEDGER_ROTATE_BYTES)
|
|
357
|
+
return;
|
|
358
|
+
const segment = paths.ledger.replace(/\.jsonl$/, `.${Date.now()}.jsonl`);
|
|
359
|
+
try {
|
|
360
|
+
fs.renameSync(paths.ledger, segment);
|
|
361
|
+
}
|
|
362
|
+
catch {
|
|
363
|
+
// @silent-fallback-ok: a concurrent rotation won the rename — fine.
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
// Segments are RETAINED until the enforce-flip decision is recorded; after
|
|
367
|
+
// that a newest-LEDGER_SEGMENT_FLOOR floor applies (§2.8).
|
|
368
|
+
if (!flipRecorded)
|
|
369
|
+
return;
|
|
370
|
+
try {
|
|
371
|
+
const segs = listLedgerSegments(paths).sort().reverse();
|
|
372
|
+
for (const stale of segs.slice(LEDGER_SEGMENT_FLOOR)) {
|
|
373
|
+
SafeFsExecutor.safeUnlinkSync(stale, {
|
|
374
|
+
operation: 'hostTestRunnerSemaphore.ledgerSegmentRetention',
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
catch {
|
|
379
|
+
/* @silent-fallback-ok: retention is best-effort housekeeping */
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
export function listLedgerSegments(paths) {
|
|
383
|
+
try {
|
|
384
|
+
const base = path.basename(paths.ledger).replace(/\.jsonl$/, '');
|
|
385
|
+
return fs
|
|
386
|
+
.readdirSync(paths.baseDir)
|
|
387
|
+
.filter((f) => f.startsWith(`${base}.`) && f.endsWith('.jsonl') && f !== path.basename(paths.ledger))
|
|
388
|
+
.map((f) => path.join(paths.baseDir, f));
|
|
389
|
+
}
|
|
390
|
+
catch {
|
|
391
|
+
// @silent-fallback-ok: no base dir yet → no segments.
|
|
392
|
+
return [];
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Bounded, torn-line-tolerant read of the newest ledger events. Spans the
|
|
397
|
+
* newest rotated SEGMENT when the live file carries no events (§2.9 —
|
|
398
|
+
* rotation must not blind the baseline read).
|
|
399
|
+
*/
|
|
400
|
+
export function readLedgerTail(paths, maxLines = 200) {
|
|
401
|
+
const files = [paths.ledger];
|
|
402
|
+
const segments = listLedgerSegments(paths).sort().reverse();
|
|
403
|
+
if (segments.length > 0)
|
|
404
|
+
files.push(segments[0]);
|
|
405
|
+
const out = [];
|
|
406
|
+
for (const file of files) {
|
|
407
|
+
let raw;
|
|
408
|
+
try {
|
|
409
|
+
raw = fs.readFileSync(file, 'utf-8');
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
// @silent-fallback-ok: a missing live file / segment is an empty tail.
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
const lines = raw.split('\n').filter((l) => l.trim().length > 0);
|
|
416
|
+
for (const line of lines.slice(-maxLines)) {
|
|
417
|
+
try {
|
|
418
|
+
const obj = JSON.parse(line);
|
|
419
|
+
if (obj && typeof obj === 'object' && typeof obj.kind === 'string') {
|
|
420
|
+
out.push(obj);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
catch {
|
|
424
|
+
/* @silent-fallback-ok: torn/malformed ledger lines are tolerated (§2.6) */
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
if (out.length > 0)
|
|
428
|
+
break; // live file had events — segment not needed
|
|
429
|
+
}
|
|
430
|
+
return out.slice(-maxLines);
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Detect tuning-file mutation via the content-hash baseline marker. On
|
|
434
|
+
* no-prior-baseline the LEDGER TAIL (live + newest segment) is consulted
|
|
435
|
+
* before establishing silently — marker-deletion must not launder an edit.
|
|
436
|
+
*/
|
|
437
|
+
export function checkTuningBaseline(paths, current) {
|
|
438
|
+
let marker = null;
|
|
439
|
+
try {
|
|
440
|
+
const raw = fs.readFileSync(paths.tuningBaseline, 'utf-8');
|
|
441
|
+
const obj = JSON.parse(raw);
|
|
442
|
+
if (obj && typeof obj === 'object' && typeof obj.hash === 'string') {
|
|
443
|
+
marker = obj;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
catch {
|
|
447
|
+
// @silent-fallback-ok: absent/corrupt marker → the no-prior-baseline path
|
|
448
|
+
// below (which consults the ledger tail — deletion is not a laundering).
|
|
449
|
+
marker = null;
|
|
450
|
+
}
|
|
451
|
+
const writeMarker = () => {
|
|
452
|
+
try {
|
|
453
|
+
ensureBaseDir(paths);
|
|
454
|
+
atomicWriteFileSync(paths.tuningBaseline, JSON.stringify({ v: 1, hash: current.hash, at: new Date().toISOString(), snapshot: current.file }), { mode: 0o600, operation: 'hostTestRunnerSemaphore.tuningBaseline' });
|
|
455
|
+
}
|
|
456
|
+
catch {
|
|
457
|
+
/* @silent-fallback-ok: marker write is best-effort — the ledger stamps still detect */
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
if (marker) {
|
|
461
|
+
if (marker.hash === current.hash) {
|
|
462
|
+
return { changed: false, changedFields: [], established: false, silentEstablish: false };
|
|
463
|
+
}
|
|
464
|
+
const changedFields = diffTuningFields(marker.snapshot ?? null, current.file);
|
|
465
|
+
writeMarker();
|
|
466
|
+
return { changed: true, changedFields, established: false, silentEstablish: false };
|
|
467
|
+
}
|
|
468
|
+
// No prior baseline: consult the ledger tail for the most recent hash stamp.
|
|
469
|
+
const tail = readLedgerTail(paths);
|
|
470
|
+
const lastStamped = [...tail].reverse().find((e) => typeof e.tuningHash === 'string');
|
|
471
|
+
writeMarker();
|
|
472
|
+
if (lastStamped && lastStamped.tuningHash !== current.hash) {
|
|
473
|
+
return { changed: true, changedFields: [], established: true, silentEstablish: false };
|
|
474
|
+
}
|
|
475
|
+
return {
|
|
476
|
+
changed: false,
|
|
477
|
+
changedFields: [],
|
|
478
|
+
established: true,
|
|
479
|
+
silentEstablish: !lastStamped,
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
function diffTuningFields(oldFile, newFile) {
|
|
483
|
+
const keys = new Set([
|
|
484
|
+
...Object.keys(oldFile ?? {}),
|
|
485
|
+
...Object.keys(newFile ?? {}),
|
|
486
|
+
]);
|
|
487
|
+
const changed = [];
|
|
488
|
+
for (const k of keys) {
|
|
489
|
+
const a = oldFile?.[k];
|
|
490
|
+
const b = newFile?.[k];
|
|
491
|
+
if (JSON.stringify(a) !== JSON.stringify(b)) {
|
|
492
|
+
changed.push(`${k}: ${JSON.stringify(a)} → ${JSON.stringify(b)}`);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
return changed;
|
|
496
|
+
}
|
|
497
|
+
// ── df marker (revalidated, never trusted forever — §2.2 item 2) ──────────
|
|
498
|
+
const DF_MARKER_TTL_MS = 24 * 3600_000;
|
|
499
|
+
/**
|
|
500
|
+
* Resolve the host-local determination for the rendezvous dir, cached to the
|
|
501
|
+
* on-disk marker. THE §1.2 LESSON APPLIED: a FAILED probe ('unknown') is
|
|
502
|
+
* NEVER written to the marker — only a positive local/not-local
|
|
503
|
+
* classification is cacheable, so a df timeout under load disables reclaim
|
|
504
|
+
* for ONE pass, never for the process (or marker) lifetime.
|
|
505
|
+
*/
|
|
506
|
+
export function resolveDfLocal(paths, probe = probeDfHostLocalDetailed) {
|
|
507
|
+
let device = null;
|
|
508
|
+
try {
|
|
509
|
+
device = fs.statSync(paths.baseDir).dev;
|
|
510
|
+
}
|
|
511
|
+
catch {
|
|
512
|
+
// @silent-fallback-ok: base dir missing — treat as unknown (no reclaim).
|
|
513
|
+
return { local: false, status: 'unknown' };
|
|
514
|
+
}
|
|
515
|
+
try {
|
|
516
|
+
const raw = fs.readFileSync(paths.dfMarker, 'utf-8');
|
|
517
|
+
const obj = JSON.parse(raw);
|
|
518
|
+
if (obj &&
|
|
519
|
+
typeof obj === 'object' &&
|
|
520
|
+
typeof obj.device === 'number' &&
|
|
521
|
+
typeof obj.local === 'boolean' &&
|
|
522
|
+
typeof obj.checkedAt === 'number' &&
|
|
523
|
+
obj.device === device &&
|
|
524
|
+
Date.now() - obj.checkedAt < DF_MARKER_TTL_MS) {
|
|
525
|
+
return { local: obj.local, status: obj.local ? 'local' : 'not-local' };
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
catch {
|
|
529
|
+
/* @silent-fallback-ok: absent/corrupt marker → re-probe below */
|
|
530
|
+
}
|
|
531
|
+
const probed = probe(paths.baseDir);
|
|
532
|
+
if (probed.status === 'unknown') {
|
|
533
|
+
// NEVER cache a failed probe (§1.2 root-cause of the spawn-lane wedge).
|
|
534
|
+
return { local: false, status: 'unknown' };
|
|
535
|
+
}
|
|
536
|
+
try {
|
|
537
|
+
ensureBaseDir(paths);
|
|
538
|
+
atomicWriteFileSync(paths.dfMarker, JSON.stringify({ v: 1, device, local: probed.status === 'local', checkedAt: Date.now() }), { mode: 0o600, operation: 'hostTestRunnerSemaphore.dfMarker' });
|
|
539
|
+
}
|
|
540
|
+
catch {
|
|
541
|
+
/* @silent-fallback-ok: marker write is a cache — the determination stands */
|
|
542
|
+
}
|
|
543
|
+
return { local: probed.status === 'local', status: probed.status };
|
|
544
|
+
}
|
|
545
|
+
export function gatherPidEvidence(pids, pidAlive = isAlive) {
|
|
546
|
+
const startMs = new Map();
|
|
547
|
+
const pgid = new Map();
|
|
548
|
+
const live = [...new Set(pids)].filter((p) => Number.isInteger(p) && p >= 2 && pidAlive(p));
|
|
549
|
+
if (live.length === 0)
|
|
550
|
+
return { startMs, pgid };
|
|
551
|
+
try {
|
|
552
|
+
// lint-allow-sync-spawn: a bounded (3s) one-shot ps over a single-digit
|
|
553
|
+
// live-pid set, gathered OUTSIDE the holders lock (§2.4 round-9).
|
|
554
|
+
const out = execFileSync('ps', ['-o', 'pid=,pgid=,lstart=', '-p', live.join(',')], {
|
|
555
|
+
timeout: 3000,
|
|
556
|
+
encoding: 'utf-8',
|
|
557
|
+
});
|
|
558
|
+
for (const line of out.split('\n')) {
|
|
559
|
+
const m = line.match(/^\s*(\d+)\s+(\d+)\s+(.+)$/);
|
|
560
|
+
if (!m)
|
|
561
|
+
continue;
|
|
562
|
+
const pid = Number(m[1]);
|
|
563
|
+
const parsed = Date.parse(m[3].trim());
|
|
564
|
+
startMs.set(pid, Number.isFinite(parsed) ? parsed : null);
|
|
565
|
+
pgid.set(pid, Number(m[2]));
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
catch {
|
|
569
|
+
/* @silent-fallback-ok: ps unavailable → evidence stays null (no reclaim on doubt for the mismatch path) */
|
|
570
|
+
}
|
|
571
|
+
for (const p of live) {
|
|
572
|
+
if (!startMs.has(p))
|
|
573
|
+
startMs.set(p, null);
|
|
574
|
+
if (!pgid.has(p))
|
|
575
|
+
pgid.set(p, null);
|
|
576
|
+
}
|
|
577
|
+
return { startMs, pgid };
|
|
578
|
+
}
|
|
579
|
+
/** The holder's OWN start time + cmd (fingerprint stamped at acquire). */
|
|
580
|
+
export function selfProcessFingerprint() {
|
|
581
|
+
let startedAt = '';
|
|
582
|
+
try {
|
|
583
|
+
// lint-allow-sync-spawn: one bounded ps self-probe at acquire time.
|
|
584
|
+
startedAt = execFileSync('ps', ['-p', String(process.pid), '-o', 'lstart='], {
|
|
585
|
+
timeout: 2000,
|
|
586
|
+
encoding: 'utf-8',
|
|
587
|
+
}).trim();
|
|
588
|
+
}
|
|
589
|
+
catch {
|
|
590
|
+
/* @silent-fallback-ok: fingerprint is best-effort corroboration data */
|
|
591
|
+
}
|
|
592
|
+
const cmd = process.argv.join(' ').slice(0, 512);
|
|
593
|
+
return { startedAt, cmd };
|
|
594
|
+
}
|
|
595
|
+
export function classifyRow(row) {
|
|
596
|
+
if (!row || typeof row !== 'object' || Array.isArray(row))
|
|
597
|
+
return 'malformed';
|
|
598
|
+
const r = row;
|
|
599
|
+
if (typeof r.pid !== 'number' || !Number.isInteger(r.pid) || r.pid < 2)
|
|
600
|
+
return 'malformed';
|
|
601
|
+
if (r.state === 'held') {
|
|
602
|
+
// A held row must carry a recognizable lane to be countable.
|
|
603
|
+
if (r.lane === 'suite' || r.lane === 'targeted')
|
|
604
|
+
return 'held';
|
|
605
|
+
return 'unknown-state';
|
|
606
|
+
}
|
|
607
|
+
if (r.state === 'terminating')
|
|
608
|
+
return 'terminating';
|
|
609
|
+
return 'unknown-state';
|
|
610
|
+
}
|
|
611
|
+
/** Effective TTL window start (a sleep-wake re-arm restarts the window once). */
|
|
612
|
+
function ttlWindowStart(row) {
|
|
613
|
+
const reArmed = typeof row.reArmedAt === 'number' ? row.reArmedAt : 0;
|
|
614
|
+
return Math.max(row.acquiredAt, reArmed);
|
|
615
|
+
}
|
|
616
|
+
export class HostTestRunnerSemaphore {
|
|
617
|
+
paths;
|
|
618
|
+
env;
|
|
619
|
+
now;
|
|
620
|
+
host;
|
|
621
|
+
pidAlive;
|
|
622
|
+
dfProbe;
|
|
623
|
+
signal;
|
|
624
|
+
gatherEvidence;
|
|
625
|
+
bootTimeMs;
|
|
626
|
+
ttlMsOverride;
|
|
627
|
+
sleep;
|
|
628
|
+
pollIntervalMs;
|
|
629
|
+
genId;
|
|
630
|
+
constructor(deps = {}) {
|
|
631
|
+
this.env = deps.env ?? process.env;
|
|
632
|
+
this.paths = deps.paths ?? resolveTestRunnerPaths(this.env);
|
|
633
|
+
this.now = deps.now ?? (() => Date.now());
|
|
634
|
+
this.host = (deps.hostname ?? (() => os.hostname()))();
|
|
635
|
+
this.pidAlive = deps.pidAlive ?? isAlive;
|
|
636
|
+
this.dfProbe = deps.dfProbe ?? probeDfHostLocalDetailed;
|
|
637
|
+
this.signal = deps.signal ?? ((pid, sig) => process.kill(pid, sig));
|
|
638
|
+
this.gatherEvidence = deps.gatherEvidence ?? ((pids) => gatherPidEvidence(pids, this.pidAlive));
|
|
639
|
+
this.bootTimeMs = deps.bootTimeMs ?? readMacBootTimeMs;
|
|
640
|
+
this.ttlMsOverride = deps.ttlMsOverride;
|
|
641
|
+
this.sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
642
|
+
this.pollIntervalMs = deps.pollIntervalMs ?? resolveInternalPollMs(this.env);
|
|
643
|
+
this.genId =
|
|
644
|
+
deps.genId ??
|
|
645
|
+
(() => `test:${process.pid}:${this.now().toString(36)}:${Math.random().toString(36).slice(2, 10)}`);
|
|
646
|
+
}
|
|
647
|
+
// ── Context stamps ──────────────────────────────────────────────────
|
|
648
|
+
/** Resolve the full lever context (tuning, posture, caps, arm) once. */
|
|
649
|
+
resolveContext() {
|
|
650
|
+
const tuning = readTuningFile(this.paths);
|
|
651
|
+
const posture = resolvePosture(this.env, tuning.file);
|
|
652
|
+
return {
|
|
653
|
+
tuning,
|
|
654
|
+
posture,
|
|
655
|
+
suiteCap: resolveCap('suite', this.env, tuning.file),
|
|
656
|
+
targetedCap: resolveCap('targeted', this.env, tuning.file),
|
|
657
|
+
ttlSignal: resolveTtlSignal(this.env, tuning.file),
|
|
658
|
+
clampActive: resolveClampActive(posture.posture, tuning.file),
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
ledger(kind, fields = {}) {
|
|
662
|
+
const ctx = this.resolveContext();
|
|
663
|
+
appendLedgerEvent(this.paths, {
|
|
664
|
+
v: 1,
|
|
665
|
+
ts: new Date(this.now()).toISOString(),
|
|
666
|
+
kind,
|
|
667
|
+
pid: process.pid,
|
|
668
|
+
hostname: this.host,
|
|
669
|
+
posture: ctx.posture.posture,
|
|
670
|
+
suiteCap: ctx.suiteCap.cap,
|
|
671
|
+
targetedCap: ctx.targetedCap.cap,
|
|
672
|
+
ttlSignalArmed: ctx.ttlSignal.armed,
|
|
673
|
+
tuningHash: ctx.tuning.hash,
|
|
674
|
+
...fields,
|
|
675
|
+
}, { flipRecorded: ctx.tuning.file?.enforcing === true && ctx.tuning.file?.flippedAt !== undefined });
|
|
676
|
+
}
|
|
677
|
+
// ── Holders IO ──────────────────────────────────────────────────────
|
|
678
|
+
readHolders() {
|
|
679
|
+
let raw;
|
|
680
|
+
try {
|
|
681
|
+
raw = fs.readFileSync(this.paths.holders, 'utf-8');
|
|
682
|
+
}
|
|
683
|
+
catch {
|
|
684
|
+
// @silent-fallback-ok: a missing holders file is an empty set.
|
|
685
|
+
return { rows: [], status: 'missing', raw: null };
|
|
686
|
+
}
|
|
687
|
+
try {
|
|
688
|
+
const obj = JSON.parse(raw);
|
|
689
|
+
if (!obj || typeof obj !== 'object' || !Array.isArray(obj.holders)) {
|
|
690
|
+
return { rows: [], status: 'unparseable', raw };
|
|
691
|
+
}
|
|
692
|
+
// §2.4/§5: a holders file AT HOST_TEST_POISON_CEILING (64) rows
|
|
693
|
+
// quarantines; one below it does not (>=, not >).
|
|
694
|
+
if (obj.holders.length >= HOST_TEST_POISON_CEILING) {
|
|
695
|
+
return { rows: obj.holders, status: 'poisoned', raw };
|
|
696
|
+
}
|
|
697
|
+
return { rows: obj.holders, status: 'ok', raw };
|
|
698
|
+
}
|
|
699
|
+
catch {
|
|
700
|
+
// @silent-fallback-ok: unparseable → the caller's quarantine+admit branch (§2.4 fail-OPEN).
|
|
701
|
+
return { rows: [], status: 'unparseable', raw };
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
writeHolders(rows) {
|
|
705
|
+
ensureBaseDir(this.paths);
|
|
706
|
+
atomicWriteFileSync(this.paths.holders, JSON.stringify({ v: 1, holders: rows }), {
|
|
707
|
+
mode: 0o600,
|
|
708
|
+
operation: 'HostTestRunnerSemaphore.writeHolders',
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* Quarantine a corrupt/poisoned holders file aside and start a fresh one,
|
|
713
|
+
* RE-HOMING any pending `terminating` tombstones (parseable path) or
|
|
714
|
+
* best-effort SALVAGING them via a line scan (unparseable path, §2.4).
|
|
715
|
+
* Returns the tombstones carried into the fresh file + whether any may have
|
|
716
|
+
* been dropped.
|
|
717
|
+
*/
|
|
718
|
+
quarantineHolders(read) {
|
|
719
|
+
const rehomed = [];
|
|
720
|
+
let possibleDrop = false;
|
|
721
|
+
let salvaged = 0;
|
|
722
|
+
if (read.status === 'poisoned') {
|
|
723
|
+
for (const row of read.rows) {
|
|
724
|
+
if (classifyRow(row) === 'terminating')
|
|
725
|
+
rehomed.push(row);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
else if (read.raw) {
|
|
729
|
+
// Unparseable: best-effort SALVAGE of terminating rows (§2.4 round 8).
|
|
730
|
+
// Holder rows are FLAT JSON objects and the holders file is written as a
|
|
731
|
+
// single line, so a line split cannot isolate rows (a row's own commas
|
|
732
|
+
// would shred it) — extract flat `{…"state":"terminating"…}` objects by
|
|
733
|
+
// regex over the whole raw instead.
|
|
734
|
+
const sawTerminatingText = /"state"\s*:\s*"terminating"/.test(read.raw);
|
|
735
|
+
const candidates = read.raw.match(/\{[^{}]*"state"\s*:\s*"terminating"[^{}]*\}/g) ?? [];
|
|
736
|
+
for (const candidate of candidates) {
|
|
737
|
+
try {
|
|
738
|
+
const obj = JSON.parse(candidate);
|
|
739
|
+
if (classifyRow(obj) === 'terminating') {
|
|
740
|
+
rehomed.push(obj);
|
|
741
|
+
salvaged++;
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
catch {
|
|
746
|
+
/* @silent-fallback-ok: salvage is best-effort — the drop is ENUMERATED below */
|
|
747
|
+
}
|
|
748
|
+
possibleDrop = true;
|
|
749
|
+
}
|
|
750
|
+
// Whether or not salvage succeeds, ENUMERATE any terminating text the
|
|
751
|
+
// scan could not recover — an armed kill obligation lost to unparseable
|
|
752
|
+
// corruption is at worst LOUD, never silent (§2.4).
|
|
753
|
+
const occurrences = read.raw.match(/"state"\s*:\s*"terminating"/g)?.length ?? 0;
|
|
754
|
+
if (sawTerminatingText && salvaged < occurrences)
|
|
755
|
+
possibleDrop = true;
|
|
756
|
+
}
|
|
757
|
+
quarantineFileAside(this.paths.holders, 'host-test-runner-holders.corrupt', this.paths.baseDir);
|
|
758
|
+
this.writeHolders(rehomed);
|
|
759
|
+
return { rehomed, possibleDrop, salvaged };
|
|
760
|
+
}
|
|
761
|
+
// ── Reclaim pass (§2.4) ─────────────────────────────────────────────
|
|
762
|
+
/**
|
|
763
|
+
* Apply the test-lane ReclaimPolicy over `rows`. Evidence is gathered by the
|
|
764
|
+
* caller OUTSIDE the lock; decisions are applied here (re-validated per row).
|
|
765
|
+
* Returns kept rows + freed info. NEVER signals unless `armed && enforcing`.
|
|
766
|
+
*/
|
|
767
|
+
applyReclaimPass(rows, evidence, ctx, events) {
|
|
768
|
+
const kept = [];
|
|
769
|
+
const report = { reclaimed: [], tombstonesCompleted: 0, liveSuite: 0, liveTargeted: 0 };
|
|
770
|
+
// §2.2 item 3 (write-only-on-change): callers skip the holders rewrite on a
|
|
771
|
+
// failed acquire when the pass removed/mutated nothing. Set whenever a row
|
|
772
|
+
// is dropped, transitioned (tombstoned), or mutated (sleep-wake re-arm).
|
|
773
|
+
let changed = false;
|
|
774
|
+
const nowMs = this.now();
|
|
775
|
+
for (const rowRaw of rows) {
|
|
776
|
+
const kind = classifyRow(rowRaw);
|
|
777
|
+
if (kind === 'malformed') {
|
|
778
|
+
// Corrupt row (pid 0/1/negative/non-integer/garbage) — quarantine path:
|
|
779
|
+
// slot freed, NO signal (§2.4 gate 1).
|
|
780
|
+
events.push({ kind: 'warn', fields: { warnType: 'malformed-row-dropped' } });
|
|
781
|
+
report.reclaimed.push({ pid: NaN, reason: 'malformed-row' });
|
|
782
|
+
continue;
|
|
783
|
+
}
|
|
784
|
+
const row = rowRaw;
|
|
785
|
+
if (kind === 'unknown-state') {
|
|
786
|
+
// Tolerant rule (§2.9): excluded from the cap count (fail-open), never
|
|
787
|
+
// signaled, preserved verbatim — but preservation is BOUNDED.
|
|
788
|
+
const age = nowMs - (typeof row.acquiredAt === 'number' ? row.acquiredAt : nowMs);
|
|
789
|
+
const dead = !this.pidAlive(row.pid);
|
|
790
|
+
if (dead && age > HOST_TEST_TTL_MAX_MS && ctx.dfLocal) {
|
|
791
|
+
events.push({ kind: 'warn', fields: { warnType: 'unknown-state-row-dropped', pid: row.pid } });
|
|
792
|
+
report.reclaimed.push({ pid: row.pid, reason: 'unknown-state-expired' });
|
|
793
|
+
continue;
|
|
794
|
+
}
|
|
795
|
+
events.push({ kind: 'schema-unknown', fields: { pid: row.pid, state: row.state ?? null } });
|
|
796
|
+
kept.push(rowRaw);
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
799
|
+
if (!ctx.dfLocal) {
|
|
800
|
+
// Reclaim disabled for the pass (fail-open: runs still admit) — keep.
|
|
801
|
+
kept.push(rowRaw);
|
|
802
|
+
if (kind === 'held')
|
|
803
|
+
this.countLive(row, report);
|
|
804
|
+
continue;
|
|
805
|
+
}
|
|
806
|
+
if (row.hostname !== this.host) {
|
|
807
|
+
// A foreign-hostname holder on a df-confirmed-local disk is bogus by
|
|
808
|
+
// the host-local contract — dropped + surfaced loudly (§2.4).
|
|
809
|
+
events.push({
|
|
810
|
+
kind: 'quarantine',
|
|
811
|
+
fields: { cause: 'foreign-hostname-holder', foreignHost: String(row.hostname).slice(0, 64), pid: row.pid },
|
|
812
|
+
});
|
|
813
|
+
report.reclaimed.push({ pid: row.pid, lane: row.lane, reason: 'foreign-hostname' });
|
|
814
|
+
continue;
|
|
815
|
+
}
|
|
816
|
+
if (kind === 'terminating') {
|
|
817
|
+
const done = this.completeTombstone(row, evidence, ctx, events, nowMs);
|
|
818
|
+
if (done) {
|
|
819
|
+
report.tombstonesCompleted++;
|
|
820
|
+
}
|
|
821
|
+
else {
|
|
822
|
+
kept.push(rowRaw); // obligation persists (non-cap-counting)
|
|
823
|
+
}
|
|
824
|
+
continue;
|
|
825
|
+
}
|
|
826
|
+
// kind === 'held'
|
|
827
|
+
const pidDead = !this.pidAlive(row.pid);
|
|
828
|
+
if (pidDead) {
|
|
829
|
+
// IMMEDIATE reclaim of a provably-dead pid — no heartbeat gate (§2.4).
|
|
830
|
+
events.push({ kind: 'reclaim-dead', fields: { pid: row.pid, lane: row.lane } });
|
|
831
|
+
report.reclaimed.push({ pid: row.pid, lane: row.lane, reason: 'pid-dead' });
|
|
832
|
+
continue;
|
|
833
|
+
}
|
|
834
|
+
// Start-time corroboration on the DEFAULT capacity path (§2.4): a live
|
|
835
|
+
// pid whose start time postdates the row's acquiredAt (+skew) is
|
|
836
|
+
// provably NOT the recorded holder → treated as dead → capacity reclaim.
|
|
837
|
+
const startMs = evidence.startMs.get(row.pid);
|
|
838
|
+
if (typeof startMs === 'number' && startMs > row.acquiredAt + START_TIME_SKEW_MS) {
|
|
839
|
+
events.push({
|
|
840
|
+
kind: 'reclaim-mismatch',
|
|
841
|
+
fields: { pid: row.pid, lane: row.lane, cause: 'start-time-postdates-acquire' },
|
|
842
|
+
});
|
|
843
|
+
report.reclaimed.push({ pid: row.pid, lane: row.lane, reason: 'pid-reuse' });
|
|
844
|
+
continue;
|
|
845
|
+
}
|
|
846
|
+
// Max-hold TTL — sanity-RANGED on read (§2.4).
|
|
847
|
+
const ttl = coerceTtlMs(row.ttlMs);
|
|
848
|
+
if (ttl.coerced) {
|
|
849
|
+
events.push({ kind: 'warn', fields: { warnType: 'ttl-coerced-on-read', pid: row.pid, raw: String(row.ttlMs).slice(0, 32) } });
|
|
850
|
+
}
|
|
851
|
+
const age = nowMs - ttlWindowStart(row);
|
|
852
|
+
if (age >= ttl.ttlMs) {
|
|
853
|
+
if (ctx.armed && ctx.posture === 'enforcing') {
|
|
854
|
+
const outcome = this.signalHungHolder(row, evidence, events, nowMs);
|
|
855
|
+
if (outcome === 'tombstoned') {
|
|
856
|
+
changed = true; // row mutated in place (state → terminating)
|
|
857
|
+
kept.push(rowRaw); // now terminating (non-counting)
|
|
858
|
+
continue;
|
|
859
|
+
}
|
|
860
|
+
if (outcome === 'rearmed') {
|
|
861
|
+
changed = true; // row mutated in place (reArmedAt stamped)
|
|
862
|
+
kept.push(rowRaw);
|
|
863
|
+
this.countLive(row, report);
|
|
864
|
+
continue;
|
|
865
|
+
}
|
|
866
|
+
// 'freed' — corroboration failed → slot freed, no signal.
|
|
867
|
+
report.reclaimed.push({ pid: row.pid, lane: row.lane, reason: 'ttl-mismatch' });
|
|
868
|
+
continue;
|
|
869
|
+
}
|
|
870
|
+
// DEFAULT: capacity-reclaim-ONLY — slot freed, loud event, NO signal.
|
|
871
|
+
events.push({
|
|
872
|
+
kind: 'stale-holder-reclaimed',
|
|
873
|
+
fields: { pid: row.pid, lane: row.lane, ageMs: age, ttlMs: ttl.ttlMs, pidAlive: true },
|
|
874
|
+
});
|
|
875
|
+
report.reclaimed.push({ pid: row.pid, lane: row.lane, reason: 'ttl-capacity-reclaim' });
|
|
876
|
+
continue;
|
|
877
|
+
}
|
|
878
|
+
if (age >= TTL_WARN_FRACTION * ttl.ttlMs) {
|
|
879
|
+
events.push({ kind: 'approaching-ttl', fields: { pid: row.pid, lane: row.lane, ageMs: age, ttlMs: ttl.ttlMs } });
|
|
880
|
+
}
|
|
881
|
+
kept.push(rowRaw);
|
|
882
|
+
this.countLive(row, report);
|
|
883
|
+
}
|
|
884
|
+
// Every drop path records into the report; only in-place mutations set the
|
|
885
|
+
// flag directly above.
|
|
886
|
+
changed = changed || report.reclaimed.length > 0 || report.tombstonesCompleted > 0;
|
|
887
|
+
return { kept, report, changed };
|
|
888
|
+
}
|
|
889
|
+
countLive(row, report) {
|
|
890
|
+
if (row.lane === 'suite')
|
|
891
|
+
report.liveSuite++;
|
|
892
|
+
else if (row.lane === 'targeted')
|
|
893
|
+
report.liveTargeted++;
|
|
894
|
+
}
|
|
895
|
+
/**
|
|
896
|
+
* The opt-in TTL signal path (§2.4) — four mandatory gates; a holder row is
|
|
897
|
+
* data, not a warrant. Returns 'tombstoned' | 'freed' | 'rearmed'.
|
|
898
|
+
*/
|
|
899
|
+
signalHungHolder(row, evidence, events, nowMs) {
|
|
900
|
+
// Gate 1 — pid sanity clamp (classifyRow already excluded <2/non-integer;
|
|
901
|
+
// re-assert + never self).
|
|
902
|
+
if (!Number.isInteger(row.pid) || row.pid < 2 || row.pid === process.pid) {
|
|
903
|
+
events.push({ kind: 'warn', fields: { warnType: 'signal-pid-sanity-refused', pid: row.pid } });
|
|
904
|
+
return 'freed';
|
|
905
|
+
}
|
|
906
|
+
// Gate 2 — identity corroboration: start time ≤ acquiredAt (+skew) AND a
|
|
907
|
+
// test-runner-shaped command line.
|
|
908
|
+
const startMs = evidence.startMs.get(row.pid);
|
|
909
|
+
const cmdOk = typeof row.cmd === 'string' && /vitest|node/i.test(row.cmd);
|
|
910
|
+
if (typeof startMs !== 'number' || startMs > row.acquiredAt + START_TIME_SKEW_MS || !cmdOk) {
|
|
911
|
+
events.push({ kind: 'reclaim-mismatch', fields: { pid: row.pid, lane: row.lane, cause: 'signal-corroboration-failed' } });
|
|
912
|
+
return 'freed';
|
|
913
|
+
}
|
|
914
|
+
// Gate 5 — sleep-wake honesty: on evidence of a boot/wake gap overlapping
|
|
915
|
+
// the hold, RE-ARM the TTL window once rather than signaling.
|
|
916
|
+
const boot = this.bootTimeMs();
|
|
917
|
+
if (typeof boot === 'number' && boot > ttlWindowStart(row) && row.reArmedAt === undefined) {
|
|
918
|
+
row.reArmedAt = nowMs;
|
|
919
|
+
events.push({ kind: 'warn', fields: { warnType: 'ttl-rearmed-sleep-wake', pid: row.pid } });
|
|
920
|
+
return 'rearmed';
|
|
921
|
+
}
|
|
922
|
+
// Gate 3 — group-leadership: group-signal ONLY when the root is its own
|
|
923
|
+
// group leader; otherwise the corroborated single PID only (+ ledger the
|
|
924
|
+
// downgrade).
|
|
925
|
+
const pgid = evidence.pgid.get(row.pid);
|
|
926
|
+
const isLeader = typeof pgid === 'number' && pgid === row.pid;
|
|
927
|
+
try {
|
|
928
|
+
this.signal(isLeader ? -row.pid : row.pid, 'SIGTERM');
|
|
929
|
+
}
|
|
930
|
+
catch {
|
|
931
|
+
/* @silent-fallback-ok: the target may have exited between corroboration and signal */
|
|
932
|
+
}
|
|
933
|
+
events.push({
|
|
934
|
+
kind: 'signal-term',
|
|
935
|
+
fields: {
|
|
936
|
+
pid: row.pid,
|
|
937
|
+
lane: row.lane,
|
|
938
|
+
groupSignal: isLeader,
|
|
939
|
+
leadershipDowngrade: !isLeader,
|
|
940
|
+
corroboration: 'matched',
|
|
941
|
+
},
|
|
942
|
+
});
|
|
943
|
+
// Gate 4 — durable escalation: transition to a non-cap-counting
|
|
944
|
+
// `terminating` tombstone (capacity freed, obligation persists).
|
|
945
|
+
row.state = 'terminating';
|
|
946
|
+
row.signaledAt = nowMs;
|
|
947
|
+
// Completer (a): an unref'd grace timer in THIS process.
|
|
948
|
+
const timer = setTimeout(() => {
|
|
949
|
+
try {
|
|
950
|
+
this.prune({ source: 'grace-timer' });
|
|
951
|
+
}
|
|
952
|
+
catch {
|
|
953
|
+
/* @silent-fallback-ok: completers (b)/(c) — later passes — still fire */
|
|
954
|
+
}
|
|
955
|
+
}, TOMBSTONE_GRACE_MS + 500);
|
|
956
|
+
timer.unref();
|
|
957
|
+
return 'tombstoned';
|
|
958
|
+
}
|
|
959
|
+
/** Complete (or keep) a terminating tombstone. Returns true when dropped. */
|
|
960
|
+
completeTombstone(row, evidence, ctx, events, nowMs) {
|
|
961
|
+
const pidDead = !this.pidAlive(row.pid);
|
|
962
|
+
if (pidDead) {
|
|
963
|
+
events.push({ kind: 'warn', fields: { warnType: 'tombstone-completed-dead', pid: row.pid } });
|
|
964
|
+
return true;
|
|
965
|
+
}
|
|
966
|
+
const signaledAt = typeof row.signaledAt === 'number' ? row.signaledAt : row.acquiredAt;
|
|
967
|
+
if (nowMs - signaledAt < TOMBSTONE_GRACE_MS)
|
|
968
|
+
return false; // grace not elapsed
|
|
969
|
+
// Re-corroborate before the SIGKILL — the fingerprint must STILL match.
|
|
970
|
+
const startMs = evidence.startMs.get(row.pid);
|
|
971
|
+
if (typeof startMs !== 'number' || startMs > row.acquiredAt + START_TIME_SKEW_MS) {
|
|
972
|
+
events.push({ kind: 'reclaim-mismatch', fields: { pid: row.pid, cause: 'tombstone-fingerprint-mismatch' } });
|
|
973
|
+
return true; // drop, no signal
|
|
974
|
+
}
|
|
975
|
+
if (!(ctx.armed && ctx.posture === 'enforcing')) {
|
|
976
|
+
// Arm has been turned off since the SIGTERM — the kill obligation is
|
|
977
|
+
// void; drop the tombstone without signaling (capacity already free).
|
|
978
|
+
events.push({ kind: 'warn', fields: { warnType: 'tombstone-dropped-arm-off', pid: row.pid } });
|
|
979
|
+
return true;
|
|
980
|
+
}
|
|
981
|
+
const pgid = evidence.pgid.get(row.pid);
|
|
982
|
+
const isLeader = typeof pgid === 'number' && pgid === row.pid;
|
|
983
|
+
try {
|
|
984
|
+
this.signal(isLeader ? -row.pid : row.pid, 'SIGKILL');
|
|
985
|
+
}
|
|
986
|
+
catch {
|
|
987
|
+
/* @silent-fallback-ok: target exited between corroboration and kill */
|
|
988
|
+
}
|
|
989
|
+
events.push({ kind: 'signal-kill', fields: { pid: row.pid, groupSignal: isLeader } });
|
|
990
|
+
return true;
|
|
991
|
+
}
|
|
992
|
+
// ── Witness records + storm slots (§2.4) ────────────────────────────
|
|
993
|
+
writeWitness() {
|
|
994
|
+
try {
|
|
995
|
+
fs.mkdirSync(this.paths.witnessDir, { recursive: true, mode: 0o700 });
|
|
996
|
+
const file = path.join(this.paths.witnessDir, `w-${process.pid}-${Math.random().toString(36).slice(2, 8)}.json`);
|
|
997
|
+
fs.writeFileSync(file, JSON.stringify({ v: 1, pid: process.pid, hostname: this.host, acquiredAt: this.now() }), { mode: 0o600 });
|
|
998
|
+
return file;
|
|
999
|
+
}
|
|
1000
|
+
catch {
|
|
1001
|
+
// @silent-fallback-ok: the witness is best-effort observability.
|
|
1002
|
+
return null;
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
/**
|
|
1006
|
+
* Claim a numbered O_EXCL storm slot (§2.4). Throws TestRunnerStormCeilingError
|
|
1007
|
+
* when all WEDGE_STORM_CEILING slots are held by pid-ALIVE claimants.
|
|
1008
|
+
*/
|
|
1009
|
+
claimStormSlot() {
|
|
1010
|
+
fs.mkdirSync(this.paths.witnessDir, { recursive: true, mode: 0o700 });
|
|
1011
|
+
const held = [];
|
|
1012
|
+
for (let n = 1; n <= WEDGE_STORM_CEILING; n++) {
|
|
1013
|
+
const slotPath = path.join(this.paths.witnessDir, `slot-${n}`);
|
|
1014
|
+
const record = JSON.stringify({ v: 1, pid: process.pid, hostname: this.host, at: this.now() });
|
|
1015
|
+
const take = tryTakeLockOnce(slotPath, record);
|
|
1016
|
+
if (take.ok) {
|
|
1017
|
+
try {
|
|
1018
|
+
fs.closeSync(take.fd);
|
|
1019
|
+
}
|
|
1020
|
+
catch {
|
|
1021
|
+
/* @silent-fallback-ok: slot fd close is benign */
|
|
1022
|
+
}
|
|
1023
|
+
return n;
|
|
1024
|
+
}
|
|
1025
|
+
// Occupied: sweepable when the claimant is dead or over the TTL ceiling.
|
|
1026
|
+
const info = this.readSlot(slotPath);
|
|
1027
|
+
if (info && (!this.pidAlive(info.pid) || this.now() - info.at > HOST_TEST_TTL_MAX_MS)) {
|
|
1028
|
+
if (this.renameAsideVerified(slotPath)) {
|
|
1029
|
+
const retake = tryTakeLockOnce(slotPath, record);
|
|
1030
|
+
if (retake.ok) {
|
|
1031
|
+
try {
|
|
1032
|
+
fs.closeSync(retake.fd);
|
|
1033
|
+
}
|
|
1034
|
+
catch {
|
|
1035
|
+
/* @silent-fallback-ok: slot fd close is benign */
|
|
1036
|
+
}
|
|
1037
|
+
return n;
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
// Lost the reclaim race — treat as held by the winner.
|
|
1041
|
+
}
|
|
1042
|
+
if (info)
|
|
1043
|
+
held.push({ slot: n, pid: info.pid, ageMs: this.now() - info.at });
|
|
1044
|
+
else
|
|
1045
|
+
held.push({ slot: n, pid: -1, ageMs: 0 });
|
|
1046
|
+
}
|
|
1047
|
+
throw new TestRunnerStormCeilingError(held);
|
|
1048
|
+
}
|
|
1049
|
+
readSlot(slotPath) {
|
|
1050
|
+
try {
|
|
1051
|
+
const obj = JSON.parse(fs.readFileSync(slotPath, 'utf-8'));
|
|
1052
|
+
if (obj && typeof obj.pid === 'number' && typeof obj.at === 'number')
|
|
1053
|
+
return obj;
|
|
1054
|
+
return { pid: -1, at: 0 }; // unparseable-but-present: age-sweepable
|
|
1055
|
+
}
|
|
1056
|
+
catch {
|
|
1057
|
+
// @silent-fallback-ok: a torn/mid-write slot reads as fresh-unknown (kept).
|
|
1058
|
+
return null;
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
/**
|
|
1062
|
+
* Rename a file aside with dev+ino verification (§2.4 round 9/10): capture
|
|
1063
|
+
* identity BEFORE the rename, verify the moved object AFTER — on mismatch
|
|
1064
|
+
* (a peer's FRESH file created in the race) drop the claim and leave the
|
|
1065
|
+
* mis-grabbed file for the age sweep. Returns true when we moved the exact
|
|
1066
|
+
* observed object.
|
|
1067
|
+
*/
|
|
1068
|
+
renameAsideVerified(target) {
|
|
1069
|
+
let before;
|
|
1070
|
+
try {
|
|
1071
|
+
before = fs.lstatSync(target);
|
|
1072
|
+
}
|
|
1073
|
+
catch {
|
|
1074
|
+
// @silent-fallback-ok: it vanished — nothing to reclaim.
|
|
1075
|
+
return false;
|
|
1076
|
+
}
|
|
1077
|
+
const aside = `${target}.reclaim-${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
|
|
1078
|
+
try {
|
|
1079
|
+
fs.renameSync(target, aside);
|
|
1080
|
+
}
|
|
1081
|
+
catch {
|
|
1082
|
+
// @silent-fallback-ok: lost the rename race — the winner proceeds.
|
|
1083
|
+
return false;
|
|
1084
|
+
}
|
|
1085
|
+
try {
|
|
1086
|
+
const after = fs.lstatSync(aside);
|
|
1087
|
+
// Identity = dev + ino + mtime. dev+ino alone is NOT sufficient: a freed
|
|
1088
|
+
// inode can be REUSED for the peer's FRESH file (observed on CI's
|
|
1089
|
+
// filesystem), giving a false identity match. rename(2) preserves mtime,
|
|
1090
|
+
// so the object we actually moved matches `before.mtimeMs` on the happy
|
|
1091
|
+
// path — while a swapped-in fresh file (written "now") never will (the
|
|
1092
|
+
// reclaimed object was provably wedged/dead, so its mtime is old).
|
|
1093
|
+
if (after.dev === before.dev &&
|
|
1094
|
+
after.ino === before.ino &&
|
|
1095
|
+
after.mtimeMs === before.mtimeMs)
|
|
1096
|
+
return true;
|
|
1097
|
+
}
|
|
1098
|
+
catch {
|
|
1099
|
+
/* @silent-fallback-ok: the aside vanished — treat as lost */
|
|
1100
|
+
}
|
|
1101
|
+
// Mismatch: we grabbed a FRESH file created in the race. Do NOT rename
|
|
1102
|
+
// back (itself racy — round 10): leave it under the private aside name
|
|
1103
|
+
// (age-swept later) and report failure.
|
|
1104
|
+
return false;
|
|
1105
|
+
}
|
|
1106
|
+
/** Liveness-gated witness + storm-slot + reclaim-temp sweep (§2.4). */
|
|
1107
|
+
sweepWitnesses() {
|
|
1108
|
+
// §2.4 round 10: a mis-grabbed/aged LOCK aside (lock.reclaim-<pid>-<nonce>,
|
|
1109
|
+
// living beside the lock in the base dir) is swept by age like any stale
|
|
1110
|
+
// reclaim-temp — the abort path deliberately never renames back.
|
|
1111
|
+
try {
|
|
1112
|
+
const lockBase = path.basename(this.paths.lock);
|
|
1113
|
+
for (const name of fs.readdirSync(this.paths.baseDir)) {
|
|
1114
|
+
if (!name.startsWith(`${lockBase}.reclaim-`))
|
|
1115
|
+
continue;
|
|
1116
|
+
const p = path.join(this.paths.baseDir, name);
|
|
1117
|
+
try {
|
|
1118
|
+
const st = fs.statSync(p);
|
|
1119
|
+
if (this.now() - st.mtimeMs > LOCK_WEDGE_AGE_MS * 6) {
|
|
1120
|
+
SafeFsExecutor.safeUnlinkSync(p, {
|
|
1121
|
+
operation: 'HostTestRunnerSemaphore.sweepLockReclaimTemp',
|
|
1122
|
+
});
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
catch {
|
|
1126
|
+
/* @silent-fallback-ok: sweep is best-effort housekeeping */
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
catch {
|
|
1131
|
+
/* @silent-fallback-ok: no base dir yet */
|
|
1132
|
+
}
|
|
1133
|
+
let entries;
|
|
1134
|
+
try {
|
|
1135
|
+
entries = fs.readdirSync(this.paths.witnessDir);
|
|
1136
|
+
}
|
|
1137
|
+
catch {
|
|
1138
|
+
// @silent-fallback-ok: no witness dir yet.
|
|
1139
|
+
return;
|
|
1140
|
+
}
|
|
1141
|
+
for (const name of entries) {
|
|
1142
|
+
const p = path.join(this.paths.witnessDir, name);
|
|
1143
|
+
try {
|
|
1144
|
+
if (name.includes('.reclaim-')) {
|
|
1145
|
+
// Aged reclaim-temp litter.
|
|
1146
|
+
const st = fs.statSync(p);
|
|
1147
|
+
if (this.now() - st.mtimeMs > LOCK_WEDGE_AGE_MS * 6) {
|
|
1148
|
+
SafeFsExecutor.safeUnlinkSync(p, { operation: 'HostTestRunnerSemaphore.sweepReclaimTemp' });
|
|
1149
|
+
}
|
|
1150
|
+
continue;
|
|
1151
|
+
}
|
|
1152
|
+
const obj = JSON.parse(fs.readFileSync(p, 'utf-8'));
|
|
1153
|
+
const pid = typeof obj?.pid === 'number' ? obj.pid : null;
|
|
1154
|
+
const at = typeof obj?.acquiredAt === 'number' ? obj.acquiredAt : typeof obj?.at === 'number' ? obj.at : null;
|
|
1155
|
+
const dead = pid === null || !this.pidAlive(pid);
|
|
1156
|
+
const overTtl = at !== null && this.now() - at > HOST_TEST_TTL_MAX_MS;
|
|
1157
|
+
// Liveness-gated: swept ONLY when the pid is DEAD or over the max TTL —
|
|
1158
|
+
// a still-running admitted-open suite's witness is never erased mid-run.
|
|
1159
|
+
if (dead || overTtl) {
|
|
1160
|
+
SafeFsExecutor.safeUnlinkSync(p, { operation: 'HostTestRunnerSemaphore.sweepWitness' });
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
catch {
|
|
1164
|
+
/* @silent-fallback-ok: sweep is best-effort housekeeping */
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
/** Live (pid-alive) witness records — the route's `admittedOpen` field. */
|
|
1169
|
+
readAdmittedOpen() {
|
|
1170
|
+
let entries;
|
|
1171
|
+
try {
|
|
1172
|
+
entries = fs.readdirSync(this.paths.witnessDir);
|
|
1173
|
+
}
|
|
1174
|
+
catch {
|
|
1175
|
+
// @silent-fallback-ok: no witness dir → none admitted-open.
|
|
1176
|
+
return [];
|
|
1177
|
+
}
|
|
1178
|
+
const out = [];
|
|
1179
|
+
for (const name of entries) {
|
|
1180
|
+
if (!name.startsWith('w-'))
|
|
1181
|
+
continue;
|
|
1182
|
+
try {
|
|
1183
|
+
const obj = JSON.parse(fs.readFileSync(path.join(this.paths.witnessDir, name), 'utf-8'));
|
|
1184
|
+
if (typeof obj?.pid === 'number' && this.pidAlive(obj.pid)) {
|
|
1185
|
+
out.push({ pid: obj.pid, acquiredAt: typeof obj.acquiredAt === 'number' ? obj.acquiredAt : 0 });
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
catch {
|
|
1189
|
+
/* @silent-fallback-ok: torn witness rows are ignored for display */
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
return out;
|
|
1193
|
+
}
|
|
1194
|
+
// ── Lock helpers ────────────────────────────────────────────────────
|
|
1195
|
+
/**
|
|
1196
|
+
* Try to take the holders lock within LOCK_ACQUIRE_DEADLINE_MS (only the
|
|
1197
|
+
* sub-millisecond critical section may spin). Returns the fd or null.
|
|
1198
|
+
*/
|
|
1199
|
+
takeLockBounded() {
|
|
1200
|
+
const deadline = this.now() + LOCK_ACQUIRE_DEADLINE_MS;
|
|
1201
|
+
for (;;) {
|
|
1202
|
+
const res = tryTakeLockOnce(this.paths.lock, JSON.stringify({ pid: process.pid, hostname: this.host, at: this.now() }));
|
|
1203
|
+
if (res.ok)
|
|
1204
|
+
return res.fd;
|
|
1205
|
+
if (res.reason === 'error') {
|
|
1206
|
+
// Could be a missing base dir — create and retry once.
|
|
1207
|
+
try {
|
|
1208
|
+
ensureBaseDir(this.paths);
|
|
1209
|
+
}
|
|
1210
|
+
catch {
|
|
1211
|
+
/* @silent-fallback-ok: the outer loop treats persistent errors as lock-unavailable */
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
if (this.now() >= deadline)
|
|
1215
|
+
return null;
|
|
1216
|
+
busyWaitSubMs();
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
/**
|
|
1220
|
+
* Race-safe AGE-reclaim of a provably-wedged lock (§2.4): atomic rename of
|
|
1221
|
+
* the observed lock + dev+ino verify. The test lane's ONLY lock reclaim —
|
|
1222
|
+
* it never pid-probes the lock (the spawn lane's legacy reclaim is not
|
|
1223
|
+
* inherited).
|
|
1224
|
+
*/
|
|
1225
|
+
ageReclaimWedgedLock() {
|
|
1226
|
+
let st;
|
|
1227
|
+
try {
|
|
1228
|
+
st = fs.lstatSync(this.paths.lock);
|
|
1229
|
+
}
|
|
1230
|
+
catch {
|
|
1231
|
+
// @silent-fallback-ok: lock vanished — nothing to reclaim (retry take).
|
|
1232
|
+
return true;
|
|
1233
|
+
}
|
|
1234
|
+
if (this.now() - st.mtimeMs <= LOCK_WEDGE_AGE_MS)
|
|
1235
|
+
return false; // fresh — live
|
|
1236
|
+
return this.renameAsideVerified(this.paths.lock);
|
|
1237
|
+
}
|
|
1238
|
+
lockFileAgeMs() {
|
|
1239
|
+
try {
|
|
1240
|
+
return this.now() - fs.lstatSync(this.paths.lock).mtimeMs;
|
|
1241
|
+
}
|
|
1242
|
+
catch {
|
|
1243
|
+
// @silent-fallback-ok: no lock file → no age.
|
|
1244
|
+
return null;
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
// ── Acquire / release (§2.2–§2.4) ───────────────────────────────────
|
|
1248
|
+
async acquire(req) {
|
|
1249
|
+
const ctx = this.resolveContext();
|
|
1250
|
+
const budgetMs = req.budgetMs ?? resolveAcquireBudgetMs(req.lane, req.runClass, this.env);
|
|
1251
|
+
const startMs = this.now();
|
|
1252
|
+
const deadline = startMs + budgetMs;
|
|
1253
|
+
let lockEverAcquired = false;
|
|
1254
|
+
let lastHolders = [];
|
|
1255
|
+
// df determination — OUTSIDE the lock, marker-cached (§2.2 item 2).
|
|
1256
|
+
const df = resolveDfLocal(this.paths, this.dfProbe);
|
|
1257
|
+
if (!df.local) {
|
|
1258
|
+
// §6: df can't confirm host-local → admit the run; reclaim disabled.
|
|
1259
|
+
const witnessFile = this.writeWitness();
|
|
1260
|
+
this.ledger('fail-open-admit', { cause: `df-${df.status}`, lane: req.lane });
|
|
1261
|
+
return { kind: 'fail-open-admit', cause: `df-${df.status}`, witnessFile };
|
|
1262
|
+
}
|
|
1263
|
+
// Self-identity fingerprint + group leadership for the row we may write —
|
|
1264
|
+
// gathered OUTSIDE the lock, once per acquire (§2.4 round 9, mirroring the
|
|
1265
|
+
// df-outside-lock discipline of §2.2 item 2: the sub-ms critical section
|
|
1266
|
+
// must never spawn `ps`).
|
|
1267
|
+
const selfFp = selfProcessFingerprint();
|
|
1268
|
+
const selfPgid = this.gatherEvidence([process.pid]).pgid.get(process.pid) ?? null;
|
|
1269
|
+
for (;;) {
|
|
1270
|
+
// Evidence gathered OUTSIDE the lock (§2.4): snapshot pids, run ps on
|
|
1271
|
+
// the live ones; decisions re-validated under the lock.
|
|
1272
|
+
const preRead = this.readHolders();
|
|
1273
|
+
const prePids = preRead.rows
|
|
1274
|
+
.map((r) => (r && typeof r === 'object' ? r.pid : NaN))
|
|
1275
|
+
.filter((p) => Number.isInteger(p));
|
|
1276
|
+
const evidence = this.gatherEvidence(prePids);
|
|
1277
|
+
const fd = this.takeLockBounded();
|
|
1278
|
+
if (fd !== null) {
|
|
1279
|
+
lockEverAcquired = true;
|
|
1280
|
+
try {
|
|
1281
|
+
const read = this.readHolders();
|
|
1282
|
+
const events = [];
|
|
1283
|
+
let rows = read.rows;
|
|
1284
|
+
if (read.status === 'unparseable' || read.status === 'poisoned') {
|
|
1285
|
+
// Fail-OPEN: admit AND quarantine (§2.4).
|
|
1286
|
+
const q = this.quarantineHolders(read);
|
|
1287
|
+
this.ledger('quarantine', {
|
|
1288
|
+
cause: read.status === 'poisoned' ? 'poison-ceiling' : 'unparseable',
|
|
1289
|
+
rehomedTombstones: q.rehomed.length,
|
|
1290
|
+
salvaged: q.salvaged,
|
|
1291
|
+
possibleTombstoneDrop: q.possibleDrop,
|
|
1292
|
+
lane: req.lane,
|
|
1293
|
+
});
|
|
1294
|
+
rows = q.rehomed;
|
|
1295
|
+
const row = this.buildRow(req, ctx, selfFp, selfPgid);
|
|
1296
|
+
rows.push(row);
|
|
1297
|
+
this.writeHolders(rows);
|
|
1298
|
+
this.ledger('acquire', this.acquireFields(req, row, false));
|
|
1299
|
+
this.armSelfTtlWarn(row);
|
|
1300
|
+
return { kind: 'acquired', id: row.id, wouldBlock: false, ttlMs: row.ttlMs };
|
|
1301
|
+
}
|
|
1302
|
+
const pass = this.applyReclaimPass(rows, evidence, {
|
|
1303
|
+
dfLocal: true,
|
|
1304
|
+
armed: ctx.ttlSignal.armed,
|
|
1305
|
+
posture: ctx.posture.posture,
|
|
1306
|
+
}, events);
|
|
1307
|
+
this.flushEvents(events);
|
|
1308
|
+
const live = req.lane === 'suite' ? pass.report.liveSuite : pass.report.liveTargeted;
|
|
1309
|
+
const cap = req.lane === 'suite' ? ctx.suiteCap.cap : ctx.targetedCap.cap;
|
|
1310
|
+
if (live < cap) {
|
|
1311
|
+
const row = this.buildRow(req, ctx, selfFp, selfPgid);
|
|
1312
|
+
pass.kept.push(row);
|
|
1313
|
+
this.writeHolders(pass.kept);
|
|
1314
|
+
this.ledger('acquire', this.acquireFields(req, row, false));
|
|
1315
|
+
this.armSelfTtlWarn(row);
|
|
1316
|
+
this.sweepWitnesses();
|
|
1317
|
+
return { kind: 'acquired', id: row.id, wouldBlock: false, ttlMs: row.ttlMs };
|
|
1318
|
+
}
|
|
1319
|
+
// Lane full.
|
|
1320
|
+
lastHolders = this.holderAges(pass.kept, req.lane);
|
|
1321
|
+
if (ctx.posture.posture !== 'enforcing') {
|
|
1322
|
+
// Dry-run (§2.11): full bookkeeping — the run that WOULD block
|
|
1323
|
+
// logs `would-block` (with the live holder set) and ADMITS.
|
|
1324
|
+
const row = this.buildRow(req, ctx, selfFp, selfPgid);
|
|
1325
|
+
pass.kept.push(row);
|
|
1326
|
+
this.writeHolders(pass.kept);
|
|
1327
|
+
this.ledger('would-block', { lane: req.lane, holders: lastHolders, cap });
|
|
1328
|
+
this.ledger('acquire', this.acquireFields(req, row, true));
|
|
1329
|
+
this.armSelfTtlWarn(row);
|
|
1330
|
+
return { kind: 'acquired', id: row.id, wouldBlock: true, ttlMs: row.ttlMs };
|
|
1331
|
+
}
|
|
1332
|
+
// Enforcing: persist reclaim work — but ONLY when the pass actually
|
|
1333
|
+
// changed something (§2.2 item 3: skip the holders rewrite on a
|
|
1334
|
+
// failed acquire when prune removed nothing — no O(waiters) churn).
|
|
1335
|
+
if (pass.changed)
|
|
1336
|
+
this.writeHolders(pass.kept);
|
|
1337
|
+
}
|
|
1338
|
+
finally {
|
|
1339
|
+
releaseLock(this.paths.lock, fd, 'HostTestRunnerSemaphore.acquire:release');
|
|
1340
|
+
}
|
|
1341
|
+
if (this.now() >= deadline) {
|
|
1342
|
+
this.ledger('block', { lane: req.lane, holders: lastHolders, budgetMs });
|
|
1343
|
+
throw new TestRunnerCapacityTimeoutError(req.lane, budgetMs, lastHolders);
|
|
1344
|
+
}
|
|
1345
|
+
req.onWaitTick?.(this.now() - startMs, lastHolders);
|
|
1346
|
+
await this.sleep(this.jitteredPoll());
|
|
1347
|
+
continue;
|
|
1348
|
+
}
|
|
1349
|
+
// Lock unavailable within the attempt deadline.
|
|
1350
|
+
const age = this.lockFileAgeMs();
|
|
1351
|
+
if (age !== null && age > LOCK_WEDGE_AGE_MS) {
|
|
1352
|
+
// Provably wedged (critical section is sub-ms) — race-safe age-reclaim.
|
|
1353
|
+
this.ageReclaimWedgedLock();
|
|
1354
|
+
continue; // retake immediately — a missed attempt is NEVER an admit
|
|
1355
|
+
}
|
|
1356
|
+
if (this.now() >= deadline) {
|
|
1357
|
+
if (!lockEverAcquired) {
|
|
1358
|
+
// Continuously unavailable for the entire budget → fail-open admit,
|
|
1359
|
+
// gated by the atomic storm ceiling (§2.4).
|
|
1360
|
+
const slot = this.claimStormSlot(); // throws TestRunnerStormCeilingError at the ceiling
|
|
1361
|
+
const witnessFile = this.writeWitness();
|
|
1362
|
+
this.ledger('fail-open-admit', { cause: 'lock-unavailable-full-budget', lane: req.lane, stormSlot: slot });
|
|
1363
|
+
return { kind: 'fail-open-admit', cause: 'lock-unavailable-full-budget', witnessFile };
|
|
1364
|
+
}
|
|
1365
|
+
this.ledger('block', { lane: req.lane, holders: lastHolders, budgetMs });
|
|
1366
|
+
throw new TestRunnerCapacityTimeoutError(req.lane, budgetMs, lastHolders);
|
|
1367
|
+
}
|
|
1368
|
+
req.onWaitTick?.(this.now() - startMs, lastHolders);
|
|
1369
|
+
await this.sleep(this.jitteredPoll());
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
jitteredPoll() {
|
|
1373
|
+
// ±POLL_JITTER_MS uniform jitter desynchronizes the thundering herd (§2.2
|
|
1374
|
+
// item 3). Floored at 0 so an internal test seam that sets a sub-jitter
|
|
1375
|
+
// poll interval can never produce a negative sleep (defaults, 5s ≫ 1s
|
|
1376
|
+
// jitter, are unaffected).
|
|
1377
|
+
return Math.max(0, this.pollIntervalMs + Math.floor(Math.random() * POLL_JITTER_MS * 2) - POLL_JITTER_MS);
|
|
1378
|
+
}
|
|
1379
|
+
/**
|
|
1380
|
+
* Build the holder row for THIS process. The self fingerprint + pgid are
|
|
1381
|
+
* passed in by acquire() — gathered OUTSIDE the lock (§2.4 round 9), never
|
|
1382
|
+
* probed from inside the critical section.
|
|
1383
|
+
*/
|
|
1384
|
+
buildRow(req, ctx, selfFp, selfPgid) {
|
|
1385
|
+
void ctx;
|
|
1386
|
+
const ttl = this.ttlMsOverride !== undefined
|
|
1387
|
+
? { ttlMs: this.ttlMsOverride, coerced: false } // INTERNAL policy parameter (tests)
|
|
1388
|
+
: resolveAcquireTtlMs(this.env);
|
|
1389
|
+
if (ttl.coerced) {
|
|
1390
|
+
this.ledger('warn', { warnType: 'ttl-coerced-at-acquire', raw: String(this.env['INSTAR_HOST_TEST_TTL_MS']).slice(0, 32) });
|
|
1391
|
+
}
|
|
1392
|
+
return {
|
|
1393
|
+
v: 1,
|
|
1394
|
+
id: this.genId(),
|
|
1395
|
+
lane: req.lane,
|
|
1396
|
+
pid: process.pid,
|
|
1397
|
+
hostname: this.host,
|
|
1398
|
+
acquiredAt: this.now(),
|
|
1399
|
+
startedAt: selfFp.startedAt,
|
|
1400
|
+
cmd: selfFp.cmd,
|
|
1401
|
+
ttlMs: ttl.ttlMs,
|
|
1402
|
+
state: 'held',
|
|
1403
|
+
runClass: req.runClass,
|
|
1404
|
+
...(req.fileCount !== undefined ? { fileCount: req.fileCount } : {}),
|
|
1405
|
+
pgidLeader: typeof selfPgid === 'number' ? selfPgid === process.pid : false,
|
|
1406
|
+
};
|
|
1407
|
+
}
|
|
1408
|
+
acquireFields(req, row, wouldBlock) {
|
|
1409
|
+
return {
|
|
1410
|
+
lane: req.lane,
|
|
1411
|
+
runClass: req.runClass,
|
|
1412
|
+
holderId: row.id,
|
|
1413
|
+
ttlMs: row.ttlMs,
|
|
1414
|
+
wouldBlock,
|
|
1415
|
+
...(req.fileCount !== undefined ? { fileCount: req.fileCount } : {}),
|
|
1416
|
+
};
|
|
1417
|
+
}
|
|
1418
|
+
flushEvents(events) {
|
|
1419
|
+
for (const e of events)
|
|
1420
|
+
this.ledger(e.kind, e.fields);
|
|
1421
|
+
}
|
|
1422
|
+
holderAges(rows, lane) {
|
|
1423
|
+
const out = [];
|
|
1424
|
+
for (const r of rows) {
|
|
1425
|
+
if (classifyRow(r) !== 'held')
|
|
1426
|
+
continue;
|
|
1427
|
+
const row = r;
|
|
1428
|
+
if (row.lane !== lane)
|
|
1429
|
+
continue;
|
|
1430
|
+
out.push({ pid: row.pid, ageMs: this.now() - row.acquiredAt });
|
|
1431
|
+
}
|
|
1432
|
+
return out;
|
|
1433
|
+
}
|
|
1434
|
+
/** The 80%-of-TTL warning timer, armed in the holding root (§2.4). */
|
|
1435
|
+
armSelfTtlWarn(row) {
|
|
1436
|
+
const delay = Math.max(1000, Math.floor(row.ttlMs * TTL_WARN_FRACTION));
|
|
1437
|
+
const timer = setTimeout(() => {
|
|
1438
|
+
try {
|
|
1439
|
+
process.stderr.write(`[test-runner-bound] WARN: this run has held its ${row.lane}-lane slot for ${Math.round((row.ttlMs * TTL_WARN_FRACTION) / 60000)}m (80% of its ${Math.round(row.ttlMs / 60000)}m TTL) — the slot will be capacity-reclaimed at TTL.\n`);
|
|
1440
|
+
this.ledger('approaching-ttl', { pid: process.pid, lane: row.lane, self: true });
|
|
1441
|
+
}
|
|
1442
|
+
catch {
|
|
1443
|
+
/* @silent-fallback-ok: the prune-pass approaching-ttl event is the durable backstop */
|
|
1444
|
+
}
|
|
1445
|
+
}, delay);
|
|
1446
|
+
timer.unref();
|
|
1447
|
+
}
|
|
1448
|
+
/** Release the slot held under `id`. Never throws; bounded lock wait. */
|
|
1449
|
+
release(id) {
|
|
1450
|
+
for (let attempt = 0; attempt < 8; attempt++) {
|
|
1451
|
+
const fd = this.takeLockBounded();
|
|
1452
|
+
if (fd === null) {
|
|
1453
|
+
const age = this.lockFileAgeMs();
|
|
1454
|
+
if (age !== null && age > LOCK_WEDGE_AGE_MS) {
|
|
1455
|
+
this.ageReclaimWedgedLock();
|
|
1456
|
+
continue;
|
|
1457
|
+
}
|
|
1458
|
+
busyWaitSubMs();
|
|
1459
|
+
continue;
|
|
1460
|
+
}
|
|
1461
|
+
try {
|
|
1462
|
+
const read = this.readHolders();
|
|
1463
|
+
if (read.status === 'ok' || read.status === 'poisoned') {
|
|
1464
|
+
const remaining = read.rows.filter((r) => !(r && typeof r === 'object' && r.id === id));
|
|
1465
|
+
this.writeHolders(remaining);
|
|
1466
|
+
}
|
|
1467
|
+
this.ledger('release', { holderId: id });
|
|
1468
|
+
return;
|
|
1469
|
+
}
|
|
1470
|
+
finally {
|
|
1471
|
+
releaseLock(this.paths.lock, fd, 'HostTestRunnerSemaphore.release:release');
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
// Couldn't take the lock — pid-death reclaim will free the slot; loud trace.
|
|
1475
|
+
this.ledger('warn', { warnType: 'release-lock-unavailable', holderId: id });
|
|
1476
|
+
}
|
|
1477
|
+
// ── Prune (POST /prune + acquire path are the ONLY persistent reclaimers) ──
|
|
1478
|
+
static _pruneInFlight = false;
|
|
1479
|
+
static _lastPruneAt = 0;
|
|
1480
|
+
/**
|
|
1481
|
+
* Force a full reclaim pass (the §2.6 recovery lever). Single-flight +
|
|
1482
|
+
* rate-limited (one forced pass per 5s). Returns what was reclaimed.
|
|
1483
|
+
*/
|
|
1484
|
+
prune(opts = { source: 'manual' }) {
|
|
1485
|
+
const nowMs = this.now();
|
|
1486
|
+
if (HostTestRunnerSemaphore._pruneInFlight) {
|
|
1487
|
+
return { reclaimed: [], tombstonesCompleted: 0, liveSuite: 0, liveTargeted: 0, rateLimited: true };
|
|
1488
|
+
}
|
|
1489
|
+
if (!opts.force && nowMs - HostTestRunnerSemaphore._lastPruneAt < 5000) {
|
|
1490
|
+
return { reclaimed: [], tombstonesCompleted: 0, liveSuite: 0, liveTargeted: 0, rateLimited: true };
|
|
1491
|
+
}
|
|
1492
|
+
HostTestRunnerSemaphore._pruneInFlight = true;
|
|
1493
|
+
HostTestRunnerSemaphore._lastPruneAt = nowMs;
|
|
1494
|
+
try {
|
|
1495
|
+
const ctx = this.resolveContext();
|
|
1496
|
+
const df = resolveDfLocal(this.paths, this.dfProbe);
|
|
1497
|
+
const preRead = this.readHolders();
|
|
1498
|
+
const prePids = preRead.rows
|
|
1499
|
+
.map((r) => (r && typeof r === 'object' ? r.pid : NaN))
|
|
1500
|
+
.filter((p) => Number.isInteger(p));
|
|
1501
|
+
const evidence = this.gatherEvidence(prePids);
|
|
1502
|
+
const fd = this.takeLockBounded();
|
|
1503
|
+
if (fd === null) {
|
|
1504
|
+
return { reclaimed: [], tombstonesCompleted: 0, liveSuite: 0, liveTargeted: 0 };
|
|
1505
|
+
}
|
|
1506
|
+
try {
|
|
1507
|
+
const read = this.readHolders();
|
|
1508
|
+
if (read.status === 'unparseable' || read.status === 'poisoned') {
|
|
1509
|
+
const q = this.quarantineHolders(read);
|
|
1510
|
+
this.ledger('quarantine', {
|
|
1511
|
+
cause: read.status === 'poisoned' ? 'poison-ceiling' : 'unparseable',
|
|
1512
|
+
rehomedTombstones: q.rehomed.length,
|
|
1513
|
+
salvaged: q.salvaged,
|
|
1514
|
+
possibleTombstoneDrop: q.possibleDrop,
|
|
1515
|
+
source: opts.source,
|
|
1516
|
+
});
|
|
1517
|
+
return { reclaimed: [], tombstonesCompleted: 0, liveSuite: 0, liveTargeted: 0 };
|
|
1518
|
+
}
|
|
1519
|
+
const events = [];
|
|
1520
|
+
const pass = this.applyReclaimPass(read.rows, evidence, {
|
|
1521
|
+
dfLocal: df.local,
|
|
1522
|
+
armed: ctx.ttlSignal.armed,
|
|
1523
|
+
posture: ctx.posture.posture,
|
|
1524
|
+
}, events);
|
|
1525
|
+
this.flushEvents(events);
|
|
1526
|
+
// Write-only-on-change (§2.2 item 3) — a no-op prune pass never churns
|
|
1527
|
+
// the holders file.
|
|
1528
|
+
if (pass.changed)
|
|
1529
|
+
this.writeHolders(pass.kept);
|
|
1530
|
+
this.sweepWitnesses();
|
|
1531
|
+
return pass.report;
|
|
1532
|
+
}
|
|
1533
|
+
finally {
|
|
1534
|
+
releaseLock(this.paths.lock, fd, 'HostTestRunnerSemaphore.prune:release');
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
finally {
|
|
1538
|
+
HostTestRunnerSemaphore._pruneInFlight = false;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
// ── Status (§2.7 — PURE read: lock-free, write-free, signal-free) ────
|
|
1542
|
+
status() {
|
|
1543
|
+
const ctx = this.resolveContext();
|
|
1544
|
+
const read = this.readHolders();
|
|
1545
|
+
const nowMs = this.now();
|
|
1546
|
+
const clampField = (s) => String(s ?? '')
|
|
1547
|
+
.replace(/[^a-zA-Z0-9._:\-\/ ]/g, '')
|
|
1548
|
+
.slice(0, 128);
|
|
1549
|
+
const liveByLane = {
|
|
1550
|
+
suite: [],
|
|
1551
|
+
targeted: [],
|
|
1552
|
+
};
|
|
1553
|
+
for (const r of read.rows) {
|
|
1554
|
+
if (classifyRow(r) !== 'held')
|
|
1555
|
+
continue; // tombstones/unknown excluded from live counts
|
|
1556
|
+
const row = r;
|
|
1557
|
+
// VIRTUAL prune for display only — the file is never written and nothing
|
|
1558
|
+
// is ever signaled from a GET (§2.7).
|
|
1559
|
+
if (!this.pidAlive(row.pid))
|
|
1560
|
+
continue;
|
|
1561
|
+
const ttl = coerceTtlMs(row.ttlMs);
|
|
1562
|
+
if (nowMs - ttlWindowStart(row) >= ttl.ttlMs)
|
|
1563
|
+
continue;
|
|
1564
|
+
liveByLane[row.lane].push({
|
|
1565
|
+
pid: row.pid,
|
|
1566
|
+
hostname: clampField(row.hostname),
|
|
1567
|
+
acquiredAt: row.acquiredAt,
|
|
1568
|
+
ttlMs: ttl.ttlMs,
|
|
1569
|
+
state: 'held',
|
|
1570
|
+
});
|
|
1571
|
+
}
|
|
1572
|
+
const recentEvents = readLedgerTail(this.paths, 50);
|
|
1573
|
+
const skipHistogram = {};
|
|
1574
|
+
for (const e of recentEvents) {
|
|
1575
|
+
if (e.kind === 'skip' && typeof e.reason === 'string') {
|
|
1576
|
+
skipHistogram[e.reason] = (skipHistogram[e.reason] ?? 0) + 1;
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
return {
|
|
1580
|
+
cap: ctx.suiteCap.cap,
|
|
1581
|
+
targetedCap: ctx.targetedCap.cap,
|
|
1582
|
+
posture: ctx.posture.posture,
|
|
1583
|
+
clampActive: ctx.clampActive,
|
|
1584
|
+
ttlSignalArmed: ctx.ttlSignal.armed,
|
|
1585
|
+
liveHolders: liveByLane.suite,
|
|
1586
|
+
targetedHolders: liveByLane.targeted,
|
|
1587
|
+
admittedOpen: this.readAdmittedOpen(),
|
|
1588
|
+
suite: {
|
|
1589
|
+
available: Math.max(0, ctx.suiteCap.cap - liveByLane.suite.length),
|
|
1590
|
+
saturated: liveByLane.suite.length >= ctx.suiteCap.cap,
|
|
1591
|
+
},
|
|
1592
|
+
targeted: {
|
|
1593
|
+
available: Math.max(0, ctx.targetedCap.cap - liveByLane.targeted.length),
|
|
1594
|
+
saturated: liveByLane.targeted.length >= ctx.targetedCap.cap,
|
|
1595
|
+
},
|
|
1596
|
+
recentEvents,
|
|
1597
|
+
skipHistogram,
|
|
1598
|
+
};
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
// ── Small shared helpers ──────────────────────────────────────────────────
|
|
1602
|
+
function ensureBaseDir(paths) {
|
|
1603
|
+
if (!fs.existsSync(paths.baseDir))
|
|
1604
|
+
fs.mkdirSync(paths.baseDir, { recursive: true, mode: 0o700 });
|
|
1605
|
+
}
|
|
1606
|
+
/** Sub-millisecond spin — permitted ONLY inside the lock critical section (§2.2). */
|
|
1607
|
+
function busyWaitSubMs() {
|
|
1608
|
+
const end = Date.now() + 1;
|
|
1609
|
+
while (Date.now() < end) {
|
|
1610
|
+
/* spin ~1ms */
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
/** INTERNAL poll override for meta-tests (undocumented — not a public lever). */
|
|
1614
|
+
function resolveInternalPollMs(env) {
|
|
1615
|
+
const raw = env['INSTAR_HOST_TEST_POLL_MS'];
|
|
1616
|
+
if (raw !== undefined) {
|
|
1617
|
+
const n = Number(raw);
|
|
1618
|
+
if (Number.isFinite(n) && n >= 10)
|
|
1619
|
+
return Math.floor(n);
|
|
1620
|
+
}
|
|
1621
|
+
return POLL_INTERVAL_MS;
|
|
1622
|
+
}
|
|
1623
|
+
/** macOS boot time via sysctl (best-effort; null when unobtainable). */
|
|
1624
|
+
export function readMacBootTimeMs() {
|
|
1625
|
+
try {
|
|
1626
|
+
// lint-allow-sync-spawn: bounded one-shot sysctl for the sleep-wake gate.
|
|
1627
|
+
const out = execFileSync('sysctl', ['-n', 'kern.boottime'], { timeout: 2000, encoding: 'utf-8' });
|
|
1628
|
+
const m = out.match(/sec\s*=\s*(\d+)/);
|
|
1629
|
+
if (m)
|
|
1630
|
+
return Number(m[1]) * 1000;
|
|
1631
|
+
return null;
|
|
1632
|
+
}
|
|
1633
|
+
catch {
|
|
1634
|
+
// @silent-fallback-ok: unobtainable boot signal → the NAMED residual (§2.4 gate 5).
|
|
1635
|
+
return null;
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
// ── Process-wide singleton (thin, per-module — deliberately NOT in the core) ──
|
|
1639
|
+
let _singleton = null;
|
|
1640
|
+
export function getHostTestRunnerSemaphore() {
|
|
1641
|
+
if (!_singleton)
|
|
1642
|
+
_singleton = new HostTestRunnerSemaphore();
|
|
1643
|
+
return _singleton;
|
|
1644
|
+
}
|
|
1645
|
+
export function _resetHostTestRunnerSemaphoreForTest() {
|
|
1646
|
+
_singleton = null;
|
|
1647
|
+
}
|
|
1648
|
+
//# sourceMappingURL=hostTestRunnerSemaphore.js.map
|