@ultimat3/cli 23.0.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +15 -7
- package/README.md +3 -3
- package/package.json +28 -28
- package/src/affected.ts +27 -35
- package/src/api-registration.ts +89 -0
- package/src/app-artifacts.ts +1 -1
- package/src/app-auth.ts +4 -27
- package/src/app-config-load.ts +51 -0
- package/src/app-env.ts +5 -11
- package/src/app-manifest.ts +2 -8
- package/src/app-mcp.ts +10 -23
- package/src/app-permissions-borrowed.ts +129 -0
- package/src/app-permissions-site.ts +24 -0
- package/src/app-permissions.ts +7 -17
- package/src/budgets.ts +4 -35
- package/src/cdp-shot-driver.ts +3 -1
- package/src/cmd-auth-spec.ts +16 -0
- package/src/cmd-auth.ts +65 -0
- package/src/cmd-build.ts +64 -15
- package/src/cmd-db-backfill.ts +25 -12
- package/src/cmd-db.ts +6 -5
- package/src/cmd-deploy-helm.ts +7 -6
- package/src/cmd-deploy.ts +4 -4
- package/src/cmd-dev.ts +9 -8
- package/src/cmd-doctor-spec.ts +2 -9
- package/src/cmd-doctor.ts +41 -14
- package/src/cmd-env.ts +26 -2
- package/src/cmd-generate.ts +79 -32
- package/src/cmd-help.ts +19 -6
- package/src/cmd-i18n.ts +10 -9
- package/src/cmd-jobs.ts +18 -6
- package/src/cmd-manifest-spec.ts +1 -1
- package/src/cmd-manifest.ts +6 -25
- package/src/cmd-mcp.ts +46 -15
- package/src/cmd-planned.ts +15 -8
- package/src/cmd-routes.ts +12 -2
- package/src/cmd-secrets.ts +47 -6
- package/src/cmd-shot-matrix.ts +1 -3
- package/src/cmd-shot.ts +12 -3
- package/src/cmd-tasks.ts +2 -2
- package/src/cmd-test-spec.ts +1 -7
- package/src/cmd-test.ts +13 -29
- package/src/cmd-verify.ts +4 -4
- package/src/db-subscribes.ts +12 -4
- package/src/dev-dashboard-guard.ts +76 -0
- package/src/dev-dashboard.ts +51 -29
- package/src/dev-lock.ts +4 -7
- package/src/dispatch.ts +3 -6
- package/src/doctor-app-url.ts +39 -0
- package/src/doctor-auth.ts +95 -0
- package/src/drift-replica-identity.ts +42 -0
- package/src/error-catalog.ts +6 -9
- package/src/error-codes.ts +23 -3
- package/src/error-unthrown.ts +29 -11
- package/src/errors.ts +54 -83
- package/src/framework-schema-apply.ts +78 -0
- package/src/framework-schema-stamp.ts +96 -0
- package/src/framework-schema.ts +13 -2
- package/src/generate-files.ts +4 -3
- package/src/generate-kinds.ts +33 -1
- package/src/generate-shadow.ts +46 -0
- package/src/generate-write.ts +47 -13
- package/src/gh-target.ts +12 -7
- package/src/i18n-audit.ts +4 -1
- package/src/index.ts +1 -3
- package/src/island-bundle.ts +3 -15
- package/src/jobs-drain.ts +82 -15
- package/src/jobs-report.ts +4 -4
- package/src/loopback-host.ts +13 -0
- package/src/manifest-staleness.ts +62 -0
- package/src/mcp-errors.ts +12 -3
- package/src/mcp-host.ts +16 -9
- package/src/mcp-test-run.ts +38 -0
- package/src/messages.ts +2 -0
- package/src/metrics-endpoint.ts +25 -1
- package/src/output.ts +6 -0
- package/src/page-navigation.ts +7 -23
- package/src/page-speculation.ts +5 -20
- package/src/prerender-out.ts +34 -7
- package/src/prerender.ts +8 -5
- package/src/pwa-artifacts.ts +26 -94
- package/src/registry.ts +2 -0
- package/src/revalidated-response.ts +4 -3
- package/src/role-replicator.ts +16 -0
- package/src/role-start.ts +9 -4
- package/src/role-sync.ts +55 -20
- package/src/runtime-assets.ts +11 -7
- package/src/runtime-bindings.ts +4 -4
- package/src/runtime-cache.ts +38 -47
- package/src/runtime-idempotency-scope.ts +23 -0
- package/src/runtime-isr.ts +36 -0
- package/src/runtime-jobs.ts +8 -33
- package/src/runtime-mfa-warning.ts +29 -0
- package/src/runtime-notify-retention.ts +10 -46
- package/src/runtime-overrides.ts +1 -1
- package/src/runtime-purge.ts +38 -16
- package/src/runtime-queue.ts +31 -10
- package/src/runtime-realtime.ts +9 -45
- package/src/runtime-render.ts +62 -35
- package/src/runtime-services.ts +20 -17
- package/src/runtime-storage.ts +10 -5
- package/src/schema-diff.ts +16 -0
- package/src/schema-drift.ts +45 -15
- package/src/seo-routes.ts +80 -9
- package/src/serve-boot.ts +34 -6
- package/src/serve-drain.ts +9 -33
- package/src/serve-env.ts +9 -6
- package/src/serve-web.ts +12 -7
- package/src/shot-locale.ts +5 -23
- package/src/site-assets.ts +3 -5
- package/src/site-config.ts +9 -48
- package/src/site-seo.ts +1 -1
- package/src/sitemap-lastmod.ts +5 -0
- package/src/solid-loader.ts +12 -3
- package/src/source-files.ts +6 -0
- package/src/stored-object-headers.ts +43 -0
- package/src/sync-url.ts +2 -2
- package/src/tasks-facts.ts +3 -29
- package/src/templates/action.ts +3 -0
- package/src/templates/naming.ts +28 -0
- package/src/templates/route.ts +3 -6
- package/src/templates/scaffold-app.ts +6 -2
- package/src/templates/scaffold-container-compose.ts +10 -1
- package/src/templates/scaffold-container.ts +4 -2
- package/src/templates/scaffold-db-client.ts +4 -2
- package/src/templates/scaffold-helm-fences.ts +76 -0
- package/src/templates/scaffold-helm-templates.ts +32 -7
- package/src/templates/scaffold-helm.ts +31 -1
- package/src/templates/scaffold-repo.ts +9 -1
- package/src/templates/scaffold-shell.ts +39 -36
- package/src/test-counts.ts +2 -6
- package/src/test-ignores.ts +25 -0
- package/src/test-passes.ts +0 -15
- package/src/test-select.ts +5 -14
- package/src/test-shards.ts +41 -93
- package/src/test-workers.ts +26 -4
- package/src/theme-boot.ts +6 -31
- package/src/trusted-hops-error.ts +17 -0
- package/src/unthrown-codes.ts +26 -0
- package/src/verify-checks.ts +17 -46
- package/src/verify-deadline.ts +130 -33
- package/src/verify-merge.ts +60 -27
- package/src/verify-part-step.ts +90 -0
- package/src/verify-run.ts +29 -8
- package/src/verify-shard.ts +13 -2
- package/src/verify-stalled.ts +152 -0
- package/src/verify-tests.ts +5 -2
- package/src/build-errors.ts +0 -54
- package/src/verify-errors.ts +0 -58
package/src/verify-deadline.ts
CHANGED
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
// tag every process a step starts carries, and the kill that leaves none of them behind. A hung
|
|
3
3
|
// step used to eat the whole CI job timeout and report nothing (#589); now it fails by name.
|
|
4
4
|
|
|
5
|
-
import { ERROR_DOCS_URL } from '@ultimat3/core';
|
|
6
5
|
import { TEST_TYPES } from '@ultimat3/testing';
|
|
7
6
|
import type { ExecResult, Runner } from './exec';
|
|
8
|
-
import
|
|
7
|
+
import { execOutput } from './exec';
|
|
9
8
|
import type { VerifyStepName } from './verify-step';
|
|
10
9
|
|
|
11
10
|
/** The `x.verify.json` key: `{ "stepTimeoutMs": { "unit": 900000 } }`. */
|
|
@@ -76,11 +75,47 @@ export const STEP_TAG_ENV = 'ULTIMATE_VERIFY_STEP';
|
|
|
76
75
|
/** `bun test`-style "the command was stopped" exit, as `timeout(1)` reports it. */
|
|
77
76
|
const EXPIRED_EXIT = 124;
|
|
78
77
|
|
|
78
|
+
/** A process the kill found, as it was the moment before it died. */
|
|
79
|
+
export interface KilledProcess {
|
|
80
|
+
readonly pid: number;
|
|
81
|
+
/** Its argv, clipped — a `bun test` coordinator's names every file of its batch. */
|
|
82
|
+
readonly command: string;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** A child the step was still waiting on when it expired. */
|
|
86
|
+
export interface InFlightRun {
|
|
87
|
+
readonly command: readonly string[];
|
|
88
|
+
/** Everything it had printed before the kill; absent when it did not settle after it. */
|
|
89
|
+
readonly output?: string;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** What a step was doing when its deadline passed — `verify-stalled.ts` turns it into the finding. */
|
|
93
|
+
export interface StepExpiry {
|
|
94
|
+
readonly killed: readonly KilledProcess[];
|
|
95
|
+
readonly inFlight: readonly InFlightRun[];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** A killed child's pipes close with it; this is only the bound on a runner that is not `exec`. */
|
|
99
|
+
const SETTLE_MS = 2_000;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* How a `bun test --parallel` worker is told from its coordinator: the flag bun starts each with
|
|
103
|
+
* (probed on 1.4.0). Killing the WORKERS first is what names the stuck file — the coordinator
|
|
104
|
+
* outlives them just long enough to print `✗ <file> (worker crashed: SIGKILL)` for whatever each
|
|
105
|
+
* was holding, in every reporter mode: plain, GitHub Actions' `::group::`, and the failures-only
|
|
106
|
+
* one an agent's shell switches on. A file's own header line cannot do it — bun prints that as
|
|
107
|
+
* results stream, so a file stuck in its third test has one.
|
|
108
|
+
*/
|
|
109
|
+
const isTestWorker = (command: string): boolean => command.includes(' --test-worker');
|
|
110
|
+
|
|
111
|
+
/** How long a coordinator gets to say what its workers were holding before it is killed too. */
|
|
112
|
+
const CRASH_REPORT_MS = 1_500;
|
|
113
|
+
|
|
79
114
|
export interface StepGuard {
|
|
80
115
|
/** `runner`, with every child tagged — and refusing to start one once the step has expired. */
|
|
81
116
|
readonly runner: Runner;
|
|
82
|
-
/** Stop the step: no further child starts,
|
|
83
|
-
expire(): Promise<
|
|
117
|
+
/** Stop the step: no further child starts, every tagged process is killed, and what was in flight is told. */
|
|
118
|
+
expire(): Promise<StepExpiry>;
|
|
84
119
|
}
|
|
85
120
|
|
|
86
121
|
/**
|
|
@@ -98,6 +133,8 @@ export function guardStep(
|
|
|
98
133
|
env: Readonly<Record<string, string | undefined>> = Bun.env,
|
|
99
134
|
): StepGuard {
|
|
100
135
|
let expired = false;
|
|
136
|
+
// Keyed by the promise itself: two batches of one step may run the very same argv.
|
|
137
|
+
const inFlight = new Map<Promise<ExecResult>, readonly string[]>();
|
|
101
138
|
const inherited = env[STEP_TAG_ENV];
|
|
102
139
|
const value = inherited === undefined || inherited === '' ? tag : `${inherited} ${tag}`;
|
|
103
140
|
return {
|
|
@@ -114,11 +151,35 @@ export function guardStep(
|
|
|
114
151
|
durationMs: 0,
|
|
115
152
|
};
|
|
116
153
|
}
|
|
117
|
-
|
|
154
|
+
const run = runner(command, { ...options, env: { ...options.env, [STEP_TAG_ENV]: value } });
|
|
155
|
+
inFlight.set(run, command);
|
|
156
|
+
try {
|
|
157
|
+
return await run;
|
|
158
|
+
} finally {
|
|
159
|
+
inFlight.delete(run);
|
|
160
|
+
}
|
|
118
161
|
},
|
|
119
|
-
async expire(): Promise<
|
|
162
|
+
async expire(): Promise<StepExpiry> {
|
|
120
163
|
expired = true;
|
|
121
|
-
|
|
164
|
+
// Before the kill: a killed child settles its runner call, which takes it off the map.
|
|
165
|
+
const waiting = [...inFlight];
|
|
166
|
+
const workers = await killTagged(tag, undefined, isTestWorker);
|
|
167
|
+
if (workers.length > 0) {
|
|
168
|
+
await raceDeadline(Promise.allSettled(waiting.map(([run]) => run)), CRASH_REPORT_MS);
|
|
169
|
+
}
|
|
170
|
+
const rest = await killTagged(tag);
|
|
171
|
+
const killed = [...workers, ...rest.filter((one) => workers.every((w) => w.pid !== one.pid))];
|
|
172
|
+
// After it: the kill closed the child's pipes, so its call now resolves with everything it
|
|
173
|
+
// had printed — for `bun test`, the crash line of each file a worker was holding.
|
|
174
|
+
const settled = await Promise.all(
|
|
175
|
+
waiting.map(async ([run, command]): Promise<InFlightRun> => {
|
|
176
|
+
const raced = await raceDeadline(run.then(execOutput), SETTLE_MS).catch(() => undefined);
|
|
177
|
+
return raced === undefined || raced.timedOut
|
|
178
|
+
? { command }
|
|
179
|
+
: { command, output: raced.value };
|
|
180
|
+
}),
|
|
181
|
+
);
|
|
182
|
+
return { killed, inFlight: settled };
|
|
122
183
|
},
|
|
123
184
|
};
|
|
124
185
|
}
|
|
@@ -162,19 +223,33 @@ async function procfsProcesses(): Promise<readonly ProcessEnviron[] | undefined>
|
|
|
162
223
|
return out;
|
|
163
224
|
}
|
|
164
225
|
|
|
226
|
+
/** One `ps` listing; past this the sweep answers what it has rather than outliving the step. */
|
|
227
|
+
const PS_TIMEOUT_MS = 10_000;
|
|
228
|
+
|
|
229
|
+
const PS_LISTING: readonly string[] = ['ps', 'axeww', '-o', 'pid=,command='];
|
|
230
|
+
|
|
165
231
|
/**
|
|
166
232
|
* Elsewhere (macOS, BSD): `ps` in its BSD spelling — `e` prints each process's environment after
|
|
167
233
|
* its command, `ww` never truncates it. The same spelling answers on Linux, which is where it is
|
|
168
234
|
* tested; procfs is preferred there because it needs no process and no parsing.
|
|
169
235
|
*/
|
|
170
|
-
export async function psProcesses(
|
|
236
|
+
export async function psProcesses(
|
|
237
|
+
argv: readonly string[] = PS_LISTING,
|
|
238
|
+
timeoutMs: number = PS_TIMEOUT_MS,
|
|
239
|
+
): Promise<readonly ProcessEnviron[]> {
|
|
171
240
|
try {
|
|
172
|
-
const ps = Bun.spawn([
|
|
241
|
+
const ps = Bun.spawn([...argv], {
|
|
173
242
|
stdout: 'pipe',
|
|
174
243
|
stderr: 'ignore',
|
|
244
|
+
// The deadline's own sweep: a `ps` that hangs would hang the step it exists to end. SIGKILL,
|
|
245
|
+
// never the default SIGTERM, which a wedged child may ignore and so outlive the bound.
|
|
246
|
+
timeout: timeoutMs,
|
|
247
|
+
killSignal: 'SIGKILL',
|
|
175
248
|
});
|
|
176
|
-
const text = await new Response(ps.stdout).text();
|
|
177
|
-
|
|
249
|
+
const [text, code] = await Promise.all([new Response(ps.stdout).text(), ps.exited]);
|
|
250
|
+
// A listing cut short by the timeout is NOT a smaller table: read as one, a tagged worker past
|
|
251
|
+
// the cut is "nothing left to kill". Unreadable, exactly like a `ps` that never ran.
|
|
252
|
+
if (code !== 0) return [];
|
|
178
253
|
return text.split('\n').flatMap((line) => {
|
|
179
254
|
const match = /^\s*(\d+)\s+(.*)$/.exec(line);
|
|
180
255
|
return match === null ? [] : [{ pid: Number(match[1]), environ: match[2] ?? '' }];
|
|
@@ -186,26 +261,65 @@ export async function psProcesses(): Promise<readonly ProcessEnviron[]> {
|
|
|
186
261
|
|
|
187
262
|
const SWEEPS = 5;
|
|
188
263
|
|
|
264
|
+
const COMMAND_CHARS = 240;
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* A process's argv, read BEFORE it is killed — afterwards there is nothing to read. procfs where
|
|
268
|
+
* there is one, `ps` otherwise; a process that is already gone is named by its pid alone.
|
|
269
|
+
*/
|
|
270
|
+
async function commandOf(pid: number): Promise<string> {
|
|
271
|
+
let text = await Bun.file(`/proc/${String(pid)}/cmdline`)
|
|
272
|
+
.text()
|
|
273
|
+
.catch(() => '');
|
|
274
|
+
if (text === '') {
|
|
275
|
+
try {
|
|
276
|
+
const ps = Bun.spawn(['ps', '-ww', '-o', 'command=', '-p', String(pid)], {
|
|
277
|
+
stdout: 'pipe',
|
|
278
|
+
stderr: 'ignore',
|
|
279
|
+
timeout: PS_TIMEOUT_MS,
|
|
280
|
+
killSignal: 'SIGKILL',
|
|
281
|
+
});
|
|
282
|
+
text = await new Response(ps.stdout).text();
|
|
283
|
+
await ps.exited;
|
|
284
|
+
} catch {
|
|
285
|
+
text = '';
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
const line = text.split('\0').join(' ').trim();
|
|
289
|
+
return line.length > COMMAND_CHARS ? `${line.slice(0, COMMAND_CHARS)}…` : line;
|
|
290
|
+
}
|
|
291
|
+
|
|
189
292
|
/**
|
|
190
293
|
* SIGKILL every process carrying `tag`, and sweep again until a pass finds none: a worker can fork
|
|
191
294
|
* between the listing and its own death. Never this process — a nested gate carries the outer tag.
|
|
295
|
+
* Returns the processes it killed, each with the command line it was running.
|
|
192
296
|
*/
|
|
193
297
|
export async function killTagged(
|
|
194
298
|
tag: string,
|
|
195
299
|
/** The process table. Absent is this platform's: procfs where there is one, `ps` otherwise. */
|
|
196
300
|
list: () => Promise<readonly ProcessEnviron[]> = async () =>
|
|
197
301
|
(await procfsProcesses()) ?? (await psProcesses()),
|
|
198
|
-
|
|
199
|
-
|
|
302
|
+
/** Kill only the processes whose command line this accepts. Absent kills every tagged one. */
|
|
303
|
+
only?: (command: string) => boolean,
|
|
304
|
+
): Promise<readonly KilledProcess[]> {
|
|
305
|
+
const killed = new Map<number, KilledProcess>();
|
|
306
|
+
// Read once per pid and BEFORE the first signal: a worker exits on its own the moment its
|
|
307
|
+
// coordinator dies, and one read between two kills would find it already gone.
|
|
308
|
+
const commands = new Map<number, string>();
|
|
200
309
|
for (let sweep = 0; sweep < SWEEPS; sweep += 1) {
|
|
201
310
|
const processes = await list();
|
|
202
|
-
const
|
|
311
|
+
const tagged = taggedPids(processes, tag).filter((pid) => pid !== process.pid);
|
|
312
|
+
for (const pid of tagged) {
|
|
313
|
+
if (!commands.has(pid)) commands.set(pid, await commandOf(pid));
|
|
314
|
+
}
|
|
315
|
+
const found =
|
|
316
|
+
only === undefined ? tagged : tagged.filter((pid) => only(commands.get(pid) ?? ''));
|
|
203
317
|
const fresh = found.filter((pid) => !killed.has(pid));
|
|
204
318
|
if (found.length === 0) break;
|
|
205
319
|
for (const pid of found) {
|
|
206
320
|
try {
|
|
207
321
|
process.kill(pid, 'SIGKILL');
|
|
208
|
-
killed.
|
|
322
|
+
if (!killed.has(pid)) killed.set(pid, { pid, command: commands.get(pid) ?? '' });
|
|
209
323
|
} catch {
|
|
210
324
|
// Already gone between the listing and the signal — which is the outcome wanted.
|
|
211
325
|
}
|
|
@@ -214,26 +328,9 @@ export async function killTagged(
|
|
|
214
328
|
if (fresh.length === 0) break;
|
|
215
329
|
await Bun.sleep(20);
|
|
216
330
|
}
|
|
217
|
-
return killed.
|
|
331
|
+
return [...killed.values()];
|
|
218
332
|
}
|
|
219
333
|
|
|
220
|
-
/**
|
|
221
|
-
* A step that ran past its deadline, on that step's own line. `command` is the gate as it was
|
|
222
|
-
* invoked here (`x verify`, or `bun run verify` at the framework root), so the fix runs where the
|
|
223
|
-
* finding was raised.
|
|
224
|
-
*/
|
|
225
|
-
export const stepTimeoutFinding = (
|
|
226
|
-
step: VerifyStepName,
|
|
227
|
-
ms: number,
|
|
228
|
-
killed: number,
|
|
229
|
-
command: string,
|
|
230
|
-
): Finding => ({
|
|
231
|
-
code: 'X_VERIFY_STEP_TIMEOUT',
|
|
232
|
-
cause: `step "${step}" did not finish within its ${String(ms)} ms deadline, so it was stopped and ${String(killed)} process(es) it had started were killed`,
|
|
233
|
-
fix: `${command} --only ${step} --json # reproduce it alone; if it legitimately needs longer, edit x.verify.json — add "${STEP_TIMEOUT_FIELD}": { "${step}": <milliseconds> }`,
|
|
234
|
-
docs: ERROR_DOCS_URL,
|
|
235
|
-
});
|
|
236
|
-
|
|
237
334
|
/** `work`, or `{ timedOut: true }` once `ms` passes. The abandoned promise is never unhandled. */
|
|
238
335
|
export async function raceDeadline<T>(
|
|
239
336
|
work: Promise<T>,
|
package/src/verify-merge.ts
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
import { existsSync } from 'node:fs';
|
|
17
17
|
// why: Bun ships no path-join primitive.
|
|
18
18
|
import { join } from 'node:path';
|
|
19
|
-
import { ERROR_DOCS_URL, renderThrowable } from '@ultimat3/core';
|
|
19
|
+
import { ERROR_DOCS_URL, renderThrowable, UltimateError } from '@ultimat3/core';
|
|
20
20
|
import { APP_CONFIG_FILE } from './app-root';
|
|
21
21
|
import { judgeCoverage } from './coverage-floor';
|
|
22
22
|
import type { CoverageMap } from './coverage-lcov';
|
|
@@ -24,27 +24,29 @@ import { decodeCoverage, mergeCoverage } from './coverage-lcov';
|
|
|
24
24
|
import { msg } from './messages';
|
|
25
25
|
import type { CommandResult, Finding, StepResult } from './output';
|
|
26
26
|
import type { TestCounts } from './test-counts';
|
|
27
|
-
import { VerifyMergeInputError } from './verify-errors';
|
|
28
27
|
import type { VerifyFloor } from './verify-floor';
|
|
29
28
|
import { floorRequires, skippedSuiteFinding } from './verify-floor';
|
|
29
|
+
import type { PartStep } from './verify-part-step';
|
|
30
|
+
import { readPartStep } from './verify-part-step';
|
|
30
31
|
import { verifySummary } from './verify-run';
|
|
32
|
+
import { corpusHash } from './verify-shard';
|
|
31
33
|
import { GATE_COMMAND, VERIFY_STEP_NAMES } from './verify-step';
|
|
32
34
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
35
|
+
/**
|
|
36
|
+
* A part handed to `x verify merge` that is not an `x verify --json` document. `command` is the
|
|
37
|
+
* entry that raised it — `bun run verify` at the framework root, where `x verify` answers
|
|
38
|
+
* `X_NOT_IN_APP` — so the `fix:` runs where it was raised.
|
|
39
|
+
*/
|
|
40
|
+
export class VerifyMergeInputError extends UltimateError {
|
|
41
|
+
constructor(input: { file: string; reason: string; sourceError?: unknown; command?: string }) {
|
|
42
|
+
const command = input.command ?? GATE_COMMAND;
|
|
43
|
+
super({
|
|
44
|
+
code: 'X_VERIFY_MERGE_INPUT',
|
|
45
|
+
cause: `${command} merge: ${input.file} ${input.reason}`,
|
|
46
|
+
fix: `${command} --only unit --json > part.json # one document per part, the last line of stdout`,
|
|
47
|
+
...(input.sourceError === undefined ? {} : { sourceError: input.sourceError }),
|
|
48
|
+
});
|
|
49
|
+
}
|
|
48
50
|
}
|
|
49
51
|
|
|
50
52
|
export interface VerifyPart {
|
|
@@ -70,8 +72,9 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
|
70
72
|
typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
71
73
|
|
|
72
74
|
/**
|
|
73
|
-
* The LAST non-empty line
|
|
74
|
-
* the gate's
|
|
75
|
+
* The LAST non-empty line, and only it: `bin/check --json` prints the build's document and then
|
|
76
|
+
* the gate's. Never "the last line that parses" — a part whose tail is not the gate's document is
|
|
77
|
+
* a truncated or appended upload, and falling back to an earlier line would believe it.
|
|
75
78
|
*/
|
|
76
79
|
export function parsePart(file: string, text: string, command: string = GATE_COMMAND): VerifyPart {
|
|
77
80
|
const lines = text
|
|
@@ -100,14 +103,9 @@ export function parsePart(file: string, text: string, command: string = GATE_COM
|
|
|
100
103
|
const data = isRecord(doc['data']) ? doc['data'] : {};
|
|
101
104
|
const steps: PartStep[] = [];
|
|
102
105
|
for (const raw of doc['steps'] as unknown[]) {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
reason: 'holds a step without a name and ok',
|
|
107
|
-
command,
|
|
108
|
-
});
|
|
109
|
-
}
|
|
110
|
-
steps.push(raw as unknown as PartStep);
|
|
106
|
+
const step = readPartStep(raw);
|
|
107
|
+
if (typeof step === 'string') throw new VerifyMergeInputError({ file, reason: step, command });
|
|
108
|
+
steps.push(step);
|
|
111
109
|
}
|
|
112
110
|
const coverage = new Map<string, CoverageMap>();
|
|
113
111
|
for (const [step, entry] of Object.entries(isRecord(data['coverage']) ? data['coverage'] : {})) {
|
|
@@ -184,6 +182,39 @@ const sumCounts = (steps: readonly PartStep[]): TestCounts | undefined => {
|
|
|
184
182
|
};
|
|
185
183
|
};
|
|
186
184
|
|
|
185
|
+
/**
|
|
186
|
+
* The shards' `files` must PARTITION the corpus they all hashed: each file in exactly one shard,
|
|
187
|
+
* and the union hashing to `corpusHash`. Agreeing on the hash is not agreeing on the split — shard
|
|
188
|
+
* 1/2 round-robin beside 2/2 with `--timings` ran one file twice and another nowhere, green.
|
|
189
|
+
*/
|
|
190
|
+
function partitionGaps(
|
|
191
|
+
name: string,
|
|
192
|
+
sharded: readonly { readonly step: PartStep }[],
|
|
193
|
+
gap: Gap,
|
|
194
|
+
): readonly Finding[] {
|
|
195
|
+
const counts = new Map<string, number>();
|
|
196
|
+
for (const entry of sharded) {
|
|
197
|
+
for (const file of entry.step.shard?.files ?? []) counts.set(file, (counts.get(file) ?? 0) + 1);
|
|
198
|
+
}
|
|
199
|
+
const twice = [...counts].filter(([, count]) => count > 1).map(([file]) => file);
|
|
200
|
+
const found: Finding[] = [];
|
|
201
|
+
if (twice.length > 0) {
|
|
202
|
+
found.push(gap(`step "${name}" ran ${twice.join(', ')} in more than one shard`, name));
|
|
203
|
+
}
|
|
204
|
+
// A list's fingerprint, not a credential: what the shards ran against what they all counted.
|
|
205
|
+
const ranCorpus = corpusHash([...counts.keys()]);
|
|
206
|
+
const countedCorpus = sharded[0]?.step.shard?.corpusHash;
|
|
207
|
+
if (ranCorpus !== countedCorpus) {
|
|
208
|
+
found.push(
|
|
209
|
+
gap(
|
|
210
|
+
`step "${name}"'s shards ran ${String(counts.size)} distinct file(s), which is not the corpus they hashed — some file ran in no shard; every shard must split one list the same way (all with --timings, or none)`,
|
|
211
|
+
name,
|
|
212
|
+
),
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
return found;
|
|
216
|
+
}
|
|
217
|
+
|
|
187
218
|
/** One step's entries from every part, folded into one step result and its gaps. */
|
|
188
219
|
function mergeStep(
|
|
189
220
|
name: string,
|
|
@@ -246,6 +277,8 @@ function mergeStep(
|
|
|
246
277
|
if (doubled.length > 0) {
|
|
247
278
|
gaps.push(gap(`step "${name}" has shard(s) ${doubled.join(', ')} more than once`, name));
|
|
248
279
|
}
|
|
280
|
+
// Only over a complete, single-corpus set — any gap above already explains a partial union.
|
|
281
|
+
if (gaps.length === 0) gaps.push(...partitionGaps(name, sharded, gap));
|
|
249
282
|
}
|
|
250
283
|
const steps = entries.map((entry) => entry.step);
|
|
251
284
|
const tests = sumCounts(steps);
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// One step of an `x verify --json` part, read against the shape `mergeStep` relies on. It was a
|
|
2
|
+
// cast past `name` and `ok`: `findings: [null]` merged green, and a step with no `durationMs`
|
|
3
|
+
// reached `Math.max` as `undefined` — a part a CI job hand-edited, or truncated, was believed.
|
|
4
|
+
|
|
5
|
+
import type { Finding } from './output';
|
|
6
|
+
import type { TestCounts } from './test-counts';
|
|
7
|
+
|
|
8
|
+
export interface PartStep {
|
|
9
|
+
readonly name: string;
|
|
10
|
+
readonly ok: boolean;
|
|
11
|
+
readonly durationMs: number;
|
|
12
|
+
readonly skipped: boolean;
|
|
13
|
+
readonly findings: readonly Finding[];
|
|
14
|
+
readonly workers?: number;
|
|
15
|
+
readonly tests?: TestCounts;
|
|
16
|
+
readonly output?: string;
|
|
17
|
+
readonly shard?: {
|
|
18
|
+
readonly index: number;
|
|
19
|
+
readonly total: number;
|
|
20
|
+
readonly corpusHash: string;
|
|
21
|
+
readonly files: readonly string[];
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
26
|
+
typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
27
|
+
|
|
28
|
+
const isCount = (value: unknown): value is number =>
|
|
29
|
+
typeof value === 'number' && Number.isInteger(value) && value >= 0;
|
|
30
|
+
|
|
31
|
+
/** What every finding is: code, cause and fix as strings. Optional fields ride along as written. */
|
|
32
|
+
const isFinding = (value: unknown): value is Finding =>
|
|
33
|
+
isRecord(value) &&
|
|
34
|
+
typeof value['code'] === 'string' &&
|
|
35
|
+
typeof value['cause'] === 'string' &&
|
|
36
|
+
typeof value['fix'] === 'string';
|
|
37
|
+
|
|
38
|
+
const isTests = (value: unknown): value is TestCounts =>
|
|
39
|
+
isRecord(value) &&
|
|
40
|
+
isCount(value['ran']) &&
|
|
41
|
+
isCount(value['skipped']) &&
|
|
42
|
+
(value['errors'] === undefined || isCount(value['errors']));
|
|
43
|
+
|
|
44
|
+
const isShard = (value: unknown): value is NonNullable<PartStep['shard']> =>
|
|
45
|
+
isRecord(value) &&
|
|
46
|
+
isCount(value['index']) &&
|
|
47
|
+
isCount(value['total']) &&
|
|
48
|
+
typeof value['corpusHash'] === 'string' &&
|
|
49
|
+
Array.isArray(value['files']) &&
|
|
50
|
+
value['files'].every((file) => typeof file === 'string');
|
|
51
|
+
|
|
52
|
+
/** The step, or the reason it is not one — phrased to follow "holds a step …". */
|
|
53
|
+
export function readPartStep(raw: unknown): PartStep | string {
|
|
54
|
+
if (!isRecord(raw) || typeof raw['name'] !== 'string' || typeof raw['ok'] !== 'boolean') {
|
|
55
|
+
return 'holds a step without a name and ok';
|
|
56
|
+
}
|
|
57
|
+
const name = raw['name'];
|
|
58
|
+
const durationMs = raw['durationMs'];
|
|
59
|
+
if (typeof durationMs !== 'number' || !Number.isFinite(durationMs) || durationMs < 0) {
|
|
60
|
+
return `holds step "${name}" without a durationMs in milliseconds`;
|
|
61
|
+
}
|
|
62
|
+
const findings = raw['findings'];
|
|
63
|
+
if (!Array.isArray(findings) || !findings.every(isFinding)) {
|
|
64
|
+
return `holds step "${name}" whose findings are not each a code, a cause and a fix`;
|
|
65
|
+
}
|
|
66
|
+
const { tests, shard, output, workers, skipped } = raw;
|
|
67
|
+
if (tests !== undefined && !isTests(tests)) {
|
|
68
|
+
return `holds step "${name}" whose tests are not counts`;
|
|
69
|
+
}
|
|
70
|
+
if (shard !== undefined && !isShard(shard)) {
|
|
71
|
+
return `holds step "${name}" whose shard is not an index, a total, a corpusHash and its files`;
|
|
72
|
+
}
|
|
73
|
+
if (output !== undefined && typeof output !== 'string') {
|
|
74
|
+
return `holds step "${name}" whose output is not text`;
|
|
75
|
+
}
|
|
76
|
+
if (workers !== undefined && !isCount(workers)) {
|
|
77
|
+
return `holds step "${name}" whose workers is not a count`;
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
name,
|
|
81
|
+
ok: raw['ok'],
|
|
82
|
+
durationMs,
|
|
83
|
+
skipped: skipped === true,
|
|
84
|
+
findings,
|
|
85
|
+
...(workers === undefined ? {} : { workers }),
|
|
86
|
+
...(tests === undefined ? {} : { tests }),
|
|
87
|
+
...(output === undefined ? {} : { output }),
|
|
88
|
+
...(shard === undefined ? {} : { shard }),
|
|
89
|
+
};
|
|
90
|
+
}
|
package/src/verify-run.ts
CHANGED
|
@@ -7,14 +7,22 @@ import type { CoverageMap } from './coverage-lcov';
|
|
|
7
7
|
import { encodeCoverage } from './coverage-lcov';
|
|
8
8
|
import { msg } from './messages';
|
|
9
9
|
import type { CommandResult, Finding, JsonValue, StepResult } from './output';
|
|
10
|
-
import {
|
|
10
|
+
import type { StepGuard } from './verify-deadline';
|
|
11
|
+
import { guardStep, raceDeadline, stepTimeoutMs } from './verify-deadline';
|
|
11
12
|
import {
|
|
12
13
|
floorRequires,
|
|
13
14
|
readVerifyFloor,
|
|
14
15
|
skippedSuiteFinding,
|
|
15
16
|
vanishedSuiteFinding,
|
|
16
17
|
} from './verify-floor';
|
|
17
|
-
import
|
|
18
|
+
import { stalledOutput, stepTimeoutFinding } from './verify-stalled';
|
|
19
|
+
import type {
|
|
20
|
+
CoverageNumbers,
|
|
21
|
+
StepOutcome,
|
|
22
|
+
VerifyContext,
|
|
23
|
+
VerifyStep,
|
|
24
|
+
VerifyStepName,
|
|
25
|
+
} from './verify-step';
|
|
18
26
|
import { GATE_COMMAND } from './verify-step';
|
|
19
27
|
|
|
20
28
|
/**
|
|
@@ -172,7 +180,7 @@ export function verifySummary(input: {
|
|
|
172
180
|
* `manifest` joined them 2026-09-23: it compares committed files against the code and writes
|
|
173
181
|
* nothing, and at the repo root its host checks are ~20 whole-tree reads that sat alone at the end.
|
|
174
182
|
*/
|
|
175
|
-
export const BESIDE_SERIAL_SUITES: ReadonlySet<
|
|
183
|
+
export const BESIDE_SERIAL_SUITES: ReadonlySet<VerifyStepName> = new Set<VerifyStepName>([
|
|
176
184
|
'lint',
|
|
177
185
|
'boundaries',
|
|
178
186
|
'filesize',
|
|
@@ -182,7 +190,7 @@ export const BESIDE_SERIAL_SUITES: ReadonlySet<string> = new Set([
|
|
|
182
190
|
]);
|
|
183
191
|
|
|
184
192
|
/** The consecutive run of steps the static group overlaps, in the order `VERIFY_STEP_NAMES` holds. */
|
|
185
|
-
export const SERIAL_SUITES: readonly
|
|
193
|
+
export const SERIAL_SUITES: readonly VerifyStepName[] = ['live', 'job', 'e2e', 'eval'];
|
|
186
194
|
|
|
187
195
|
async function runStep(
|
|
188
196
|
step: VerifyStep,
|
|
@@ -243,10 +251,7 @@ async function runStep(
|
|
|
243
251
|
limit,
|
|
244
252
|
);
|
|
245
253
|
const outcome: StepOutcome = raced.timedOut
|
|
246
|
-
?
|
|
247
|
-
ok: false,
|
|
248
|
-
findings: [stepTimeoutFinding(step.name, limit, await guard.expire(), command)],
|
|
249
|
-
}
|
|
254
|
+
? await expired(step.name, limit, guard, command)
|
|
250
255
|
: raced.value;
|
|
251
256
|
// A suite that executed nothing did not run, whatever its exit code says: `bun test` exits 0
|
|
252
257
|
// over an all-skipped file, so the counts are the only channel that can tell the two apart.
|
|
@@ -282,6 +287,22 @@ async function runStep(
|
|
|
282
287
|
};
|
|
283
288
|
}
|
|
284
289
|
|
|
290
|
+
/** The timed-out step's outcome: the finding naming what was in flight, over what it last printed. */
|
|
291
|
+
async function expired(
|
|
292
|
+
step: VerifyStepName,
|
|
293
|
+
limit: number,
|
|
294
|
+
guard: StepGuard,
|
|
295
|
+
command: string,
|
|
296
|
+
): Promise<StepOutcome> {
|
|
297
|
+
const expiry = await guard.expire();
|
|
298
|
+
const output = stalledOutput(expiry);
|
|
299
|
+
return {
|
|
300
|
+
ok: false,
|
|
301
|
+
findings: [stepTimeoutFinding(step, limit, expiry, command)],
|
|
302
|
+
...(output === undefined ? {} : { output }),
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
285
306
|
function findingOf(error: unknown, step: string, command: string): Finding {
|
|
286
307
|
// A step may throw anything, including an Error that fights being read: `instanceof` runs a
|
|
287
308
|
// Proxy's `getPrototypeOf` trap and `.message` runs a getter, so a hostile throw would take the
|
package/src/verify-shard.ts
CHANGED
|
@@ -5,13 +5,24 @@
|
|
|
5
5
|
// of the machine, the clock or the order a glob yielded — so job 3 of 8 is the same files on every
|
|
6
6
|
// runner, and a failing shard reproduces locally with the same flags.
|
|
7
7
|
|
|
8
|
-
import { renderFixShellArg } from '@ultimat3/core';
|
|
8
|
+
import { renderFixShellArg, UltimateError } from '@ultimat3/core';
|
|
9
9
|
import { CryptoHasher } from 'bun';
|
|
10
10
|
import type { ShardFacts } from './output';
|
|
11
|
-
import { VerifyShardInvalidError } from './verify-errors';
|
|
12
11
|
import type { VerifyStepName } from './verify-step';
|
|
13
12
|
import { GATE_COMMAND } from './verify-step';
|
|
14
13
|
|
|
14
|
+
/** `x verify --shard` asked for a split the gate cannot make. */
|
|
15
|
+
export class VerifyShardInvalidError extends UltimateError {
|
|
16
|
+
constructor(input: { reason: string; fix?: string; command?: string }) {
|
|
17
|
+
const command = input.command ?? GATE_COMMAND;
|
|
18
|
+
super({
|
|
19
|
+
code: 'X_VERIFY_SHARD_INVALID',
|
|
20
|
+
cause: `${command} --shard: ${input.reason}`,
|
|
21
|
+
fix: input.fix ?? `${command} --only unit --shard 1/4 --json`,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
15
26
|
/** The steps a shard may split: the parallel suites. `live` and `e2e` are serial by design. */
|
|
16
27
|
export const SHARDABLE_STEPS: readonly VerifyStepName[] = ['unit', 'contract', 'job'];
|
|
17
28
|
|