@molecule/api-ai-tools 1.5.20 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +326 -38
- package/dist/command-position.d.ts +62 -0
- package/dist/command-position.d.ts.map +1 -0
- package/dist/command-position.js +352 -0
- package/dist/index.d.ts +8 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -3
- package/dist/no-test-run.d.ts +13 -0
- package/dist/no-test-run.d.ts.map +1 -1
- package/dist/no-test-run.js +31 -22
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +10 -4
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +147 -54
- package/dist/utilities.d.ts +208 -38
- package/dist/utilities.d.ts.map +1 -1
- package/dist/utilities.js +616 -105
- package/package.json +1 -1
package/dist/utilities.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* @module
|
|
6
6
|
*/
|
|
7
7
|
import { posix } from 'path';
|
|
8
|
+
import { commandSegments } from './command-position.js';
|
|
8
9
|
import { commandRunsNoTests } from './no-test-run.js';
|
|
9
10
|
/**
|
|
10
11
|
* Shell-safe quoting using single quotes. Unlike JSON.stringify (double quotes),
|
|
@@ -288,8 +289,43 @@ export function outputRedirectTarget(command) {
|
|
|
288
289
|
}
|
|
289
290
|
/** A trailing `| tail -N` / `| tail -n N` (stderr redirects before it stay). */
|
|
290
291
|
const TRAILING_TAIL = /\s*\|\s*tail(?:\s+-n)?\s+[-+]?\d+\s*$/;
|
|
291
|
-
/** `--reporter
|
|
292
|
-
const
|
|
292
|
+
/** `--reporter line` / `--reporter=line` / `--reporter="list,line"`, with its value. */
|
|
293
|
+
const REPORTER_FLAG = /--reporter(=|\s+)(?:"([^"]*)"|'([^']*)'|(\S+))/g;
|
|
294
|
+
/**
|
|
295
|
+
* The reporters a step's LAST `--reporter` names (`--reporter=list,line` → `['list', 'line']`).
|
|
296
|
+
*
|
|
297
|
+
* Playwright takes the last `--reporter`, so an earlier one is not what runs.
|
|
298
|
+
*
|
|
299
|
+
* @param step - The command's last step.
|
|
300
|
+
* @returns The flag's span and its reporters, or null when the step names none.
|
|
301
|
+
*/
|
|
302
|
+
function lastReporterFlag(step) {
|
|
303
|
+
REPORTER_FLAG.lastIndex = 0;
|
|
304
|
+
let last = null;
|
|
305
|
+
for (let m = REPORTER_FLAG.exec(step); m !== null; m = REPORTER_FLAG.exec(step))
|
|
306
|
+
last = m;
|
|
307
|
+
if (!last)
|
|
308
|
+
return null;
|
|
309
|
+
const value = last[2] ?? last[3] ?? last[4] ?? '';
|
|
310
|
+
return {
|
|
311
|
+
start: last.index,
|
|
312
|
+
end: last.index + last[0].length,
|
|
313
|
+
separator: last[1] ?? '=',
|
|
314
|
+
values: value
|
|
315
|
+
.split(',')
|
|
316
|
+
.map((v) => v.trim())
|
|
317
|
+
.filter(Boolean),
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* The reporters that print one line (or one mark) per test into the log, `list` among them.
|
|
322
|
+
*
|
|
323
|
+
* `list` is the only one of these the platform can read back — `suite-fail-fast.ts`'s fail-fast
|
|
324
|
+
* note and the Tests bar both parse its ` ✓/✘ <n> … (…)` lines — so it is the one a background
|
|
325
|
+
* run is put on. The rest are here because they print each failure TOO: a log with two of them
|
|
326
|
+
* carries every result twice.
|
|
327
|
+
*/
|
|
328
|
+
const PRINTING_REPORTERS = new Set(['list', 'line', 'dot', 'basic', 'spec']);
|
|
293
329
|
/**
|
|
294
330
|
* Rewrite a command about to run in the background so its log grows while it
|
|
295
331
|
* runs. Two shapes left the log empty until exit, and an empty log is what
|
|
@@ -302,19 +338,18 @@ const LINE_REPORTER_FLAG = /--reporter([= ])line\b/;
|
|
|
302
338
|
* often `html`, which prints nothing per test. `--reporter=list` is added
|
|
303
339
|
* when the playwright run is the command's last step and nothing is piped or
|
|
304
340
|
* redirected from it: it writes each result as it finishes, and it is the
|
|
305
|
-
* shape the platform reads a background suite back in
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
*
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
* `TimeoutError: page.waitForURL` from one missing `navigate('/')`.
|
|
341
|
+
* shape the platform reads a background suite back in.
|
|
342
|
+
*
|
|
343
|
+
* The run ends up with exactly ONE reporter that prints each failure — `list`. A run that names
|
|
344
|
+
* any other printing reporter has it REPLACED, never joined: two of them write every test result
|
|
345
|
+
* into the log twice, which on X1 flash-m83 (2026-10-16) made a `--reporter=list,line` run (its
|
|
346
|
+
* `line` added by this function one run earlier) report two distinct failures as "4 of this run's
|
|
347
|
+
* tests have already failed the same way", and the executor stopped a 139-test walk at test 21
|
|
348
|
+
* with a single cause in hand. `line` alone is no use either: it writes `[n/N] <title>` with no
|
|
349
|
+
* ✓/✘ mark and no duration, so on X1 flash-m62 an 8.3-minute walk of one suite could never trip
|
|
350
|
+
* the fail-fast note. A reporter set that writes a machine format to a file (`json`, `html`,
|
|
351
|
+
* `junit`) is the
|
|
352
|
+
* executor's own and is left exactly as it is — so is any reporter this build does not know.
|
|
318
353
|
*
|
|
319
354
|
* Only the command's last step is touched, and only its reporter, and only when that step runs
|
|
320
355
|
* something: `npx playwright test --list` takes two seconds and writes nothing per test, so
|
|
@@ -353,40 +388,54 @@ export function streamingBackgroundCommand(command) {
|
|
|
353
388
|
// which had dropped the `--retries=0` it used one command earlier. The uncapped proof run
|
|
354
389
|
// keeps the suite's retries.
|
|
355
390
|
const iterating = /--max-failures\b|--last-failed\b|(?:^|\s)(?:-g|--grep)\b|\S+\.(?:spec|test)\.[cm]?[jt]sx?(?::\d+)?\b/.test(lastStep);
|
|
356
|
-
//
|
|
357
|
-
// the
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
391
|
+
// The last step's reporters: one printer, and it is `list`. Every branch below only ever moves
|
|
392
|
+
// the command TO that shape — a reporter set is never left with two printers in it.
|
|
393
|
+
const reporters = finalPlaywrightRun ? lastReporterFlag(lastStep) : null;
|
|
394
|
+
if (reporters) {
|
|
395
|
+
const printing = reporters.values.filter((v) => PRINTING_REPORTERS.has(v));
|
|
396
|
+
// Already exactly `list`: the shape, and nothing printing beside it. Left verbatim — the
|
|
397
|
+
// executor's own spelling of the command is not rewritten for nothing.
|
|
398
|
+
if (printing.length > 0 && !(printing.length === 1 && printing[0] === 'list')) {
|
|
399
|
+
// The printing reporters become ONE `list`, in the place of the first of them; every other
|
|
400
|
+
// reporter (a `json`/`html`/`junit` file the executor asked for, or one this build does not
|
|
401
|
+
// know) is left where it was.
|
|
402
|
+
let placed = false;
|
|
403
|
+
const kept = reporters.values
|
|
404
|
+
.map((v) => {
|
|
405
|
+
if (!PRINTING_REPORTERS.has(v))
|
|
406
|
+
return v;
|
|
407
|
+
if (placed)
|
|
408
|
+
return null;
|
|
409
|
+
placed = true;
|
|
410
|
+
return 'list';
|
|
411
|
+
})
|
|
412
|
+
.filter((v) => v !== null);
|
|
413
|
+
// Only the last step is rewritten, and `lastStep` is a suffix of `out`.
|
|
414
|
+
out =
|
|
415
|
+
out.slice(0, out.length - lastStep.length) +
|
|
416
|
+
lastStep.slice(0, reporters.start) +
|
|
417
|
+
`--reporter${reporters.separator}${kept.join(',')}` +
|
|
418
|
+
lastStep.slice(reporters.end);
|
|
419
|
+
changes.push(
|
|
420
|
+
// `list` was already there and a second printer sat beside it: that one is dropped. When
|
|
421
|
+
// no `list` was named at all, the printer is SWITCHED to it (flash-m62's `line`).
|
|
422
|
+
printing.includes('list')
|
|
423
|
+
? `Dropped \`${printing
|
|
424
|
+
.filter((v) => v !== 'list')
|
|
425
|
+
.join(',')}\` from your \`--reporter=${reporters.values.join(',')}\`: ` +
|
|
426
|
+
'two reporters that both print each failure write every test result into the log ' +
|
|
427
|
+
'twice — the whole log doubled for nothing (X1 flash-m83) — so this run keeps ' +
|
|
428
|
+
'`list` alone.'
|
|
429
|
+
: `Switched the test reporters to \`--reporter${reporters.separator}${kept.join(',')}\`: ` +
|
|
430
|
+
'`list` writes each test result as it finishes and is the shape the platform reads ' +
|
|
431
|
+
'back from a background run, and it is the only printing reporter — two of them ' +
|
|
432
|
+
'write every result into the log twice (X1 flash-m83).');
|
|
433
|
+
}
|
|
382
434
|
}
|
|
383
|
-
else if (finalPlaywrightRun
|
|
384
|
-
out = `${out.trimEnd()} --reporter
|
|
385
|
-
changes.push(
|
|
386
|
-
'(the config reporter may print nothing until the end).'
|
|
387
|
-
(iterating
|
|
388
|
-
? ' `line` writes each failure’s error as it happens, which is what a mid-run look reads.'
|
|
389
|
-
: ''));
|
|
435
|
+
else if (finalPlaywrightRun) {
|
|
436
|
+
out = `${out.trimEnd()} --reporter=list`;
|
|
437
|
+
changes.push('Added `--reporter=list` so each test result is written to the log as it finishes ' +
|
|
438
|
+
'(the config reporter may print nothing until the end).');
|
|
390
439
|
}
|
|
391
440
|
if (finalPlaywrightRun && iterating && !/--retries\b/.test(lastStep)) {
|
|
392
441
|
out = `${out.trimEnd()} --retries=0`;
|
|
@@ -400,14 +449,18 @@ export function streamingBackgroundCommand(command) {
|
|
|
400
449
|
* of from a sandbox; `suiteRunKind` (molecule-dev `api/src/ai/suite-proof.ts`)
|
|
401
450
|
* is the platform's fuller list of URL-testing runners, and the shapes here are
|
|
402
451
|
* its shapes.
|
|
452
|
+
*
|
|
453
|
+
* Each is matched at the START of a pipeline segment's command-position text
|
|
454
|
+
* (`commandSegments`), so a runner named as an argument is not an invocation:
|
|
455
|
+
* `grep -n playwright e2e/README.md` reads a file and starts nothing.
|
|
403
456
|
*/
|
|
404
457
|
const SUITE_RUN_COMMANDS = [
|
|
405
458
|
// `playwright test`, not a bare `playwright` (`--version` is not a run).
|
|
406
|
-
|
|
459
|
+
/^playwright\s+test\b/,
|
|
407
460
|
// Vitest, however it is invoked (`vitest`, `npx vitest run`).
|
|
408
|
-
|
|
461
|
+
/^vitest\b/,
|
|
409
462
|
// `hurl --test …` or `hurl … x.hurl`.
|
|
410
|
-
|
|
463
|
+
/^hurl\s+[^|\n]*(?:--test|\.hurl)/,
|
|
411
464
|
];
|
|
412
465
|
/**
|
|
413
466
|
* Whether a command runs a whole test suite — a Playwright, Vitest or Hurl run
|
|
@@ -421,21 +474,29 @@ const SUITE_RUN_COMMANDS = [
|
|
|
421
474
|
* @returns True when the command starts one of those runners.
|
|
422
475
|
*/
|
|
423
476
|
export function isSuiteRunCommand(command) {
|
|
424
|
-
|
|
477
|
+
if (commandRunsNoTests(command))
|
|
478
|
+
return false;
|
|
479
|
+
return commandSegments(command).some((segment) => SUITE_RUN_COMMANDS.some((re) => re.test(segment.text)));
|
|
425
480
|
}
|
|
426
481
|
/**
|
|
427
|
-
* Whether a command starts a Playwright run — `playwright test` in
|
|
482
|
+
* Whether a command starts a Playwright run — a `playwright test` in command position, in a form
|
|
483
|
+
* that runs something.
|
|
428
484
|
*
|
|
429
485
|
* `npx playwright test --list` matched every `\bplaywright\s+test\b` test in the platform and was
|
|
430
486
|
* then treated as a run: held (flash-m79's config check), backgrounded for a run that outlasts
|
|
431
487
|
* the tool's ceiling, and rewritten with `--reporter`. A list takes two seconds, prints its lines
|
|
432
488
|
* and exits, so it is none of those; {@link commandRunsNoTests} is what decides.
|
|
433
489
|
*
|
|
490
|
+
* A `playwright test` named as an argument is not a run either (`head -30 run-tests.sh`,
|
|
491
|
+
* `grep -n "playwright test" e2e/README.md`): the command that runs is the reader.
|
|
492
|
+
*
|
|
434
493
|
* @param command - The shell command.
|
|
435
494
|
* @returns True when the command starts a Playwright run.
|
|
436
495
|
*/
|
|
437
496
|
export function isPlaywrightRunCommand(command) {
|
|
438
|
-
|
|
497
|
+
if (commandRunsNoTests(command))
|
|
498
|
+
return false;
|
|
499
|
+
return commandSegments(command).some((segment) => /^playwright\s+test\b/.test(segment.text));
|
|
439
500
|
}
|
|
440
501
|
/**
|
|
441
502
|
* Start a background suite run at lower CPU and IO priority, so the sandbox it
|
|
@@ -669,6 +730,30 @@ export function pathArgError(path, tool) {
|
|
|
669
730
|
return `${tool} requires a non-empty "path" argument (a file path relative to the project root, e.g. "api/src/handlers/index.ts").`;
|
|
670
731
|
return null;
|
|
671
732
|
}
|
|
733
|
+
/**
|
|
734
|
+
* The error for a file tool called with NO arguments at all — `edit_file {}`, the
|
|
735
|
+
* shape a truncated provider stream leaves behind.
|
|
736
|
+
*
|
|
737
|
+
* The path message is the wrong reading of it: "requires a non-empty path
|
|
738
|
+
* argument" says the call was understood and the path was left out, so the model
|
|
739
|
+
* retries the call it just made instead of sending the whole call again (x0
|
|
740
|
+
* flash-m83 +29:32: `edit_file {}`, one round lost). An empty object and a
|
|
741
|
+
* missing `path` are different failures and deserve different sentences.
|
|
742
|
+
*
|
|
743
|
+
* @param input - The raw tool input.
|
|
744
|
+
* @param tool - The tool name, for the message.
|
|
745
|
+
* @returns The truncation message, or null when the call carried any argument.
|
|
746
|
+
*/
|
|
747
|
+
export function emptyCallError(input, tool) {
|
|
748
|
+
const empty = input === undefined ||
|
|
749
|
+
input === null ||
|
|
750
|
+
(typeof input === 'object' &&
|
|
751
|
+
!Array.isArray(input) &&
|
|
752
|
+
Object.keys(input).length === 0);
|
|
753
|
+
return empty
|
|
754
|
+
? `${tool} arrived with no arguments at all — the call was cut short, usually a truncated response. Send the whole call again.`
|
|
755
|
+
: null;
|
|
756
|
+
}
|
|
672
757
|
/**
|
|
673
758
|
* Detect a "read/edit targeted a directory, not a file" failure from a backend
|
|
674
759
|
* error message (local fs `EISDIR` or the sandbox's `cat: X: Is a directory`),
|
|
@@ -740,31 +825,96 @@ export function sliceLines(content, window) {
|
|
|
740
825
|
truncated: offset > 1 || offset - 1 + picked.length < totalLines,
|
|
741
826
|
};
|
|
742
827
|
}
|
|
743
|
-
/**
|
|
828
|
+
/**
|
|
829
|
+
* A file's text as the request counts it: the length of its JSON string.
|
|
830
|
+
*
|
|
831
|
+
* JSON escaping (`\n` → `\\n`, `"` → `\"`) costs characters, and the request
|
|
832
|
+
* side counts the SERIALIZED tool result. A window of 59,000 characters of
|
|
833
|
+
* source — 1,200 lines — arrives as ~60,200 characters of JSON, over the
|
|
834
|
+
* ceiling it was sized for, so a window is measured with this rather than by
|
|
835
|
+
* its raw text length.
|
|
836
|
+
*
|
|
837
|
+
* @param text - The text.
|
|
838
|
+
* @returns Its serialized length, without the surrounding quotes.
|
|
839
|
+
*/
|
|
840
|
+
export function serializedChars(text) {
|
|
841
|
+
return JSON.stringify(text).length - 2;
|
|
842
|
+
}
|
|
843
|
+
/**
|
|
844
|
+
* How many leading lines of a file fit inside a budget of SERIALIZED chars,
|
|
845
|
+
* the way `read_file` sizes a window (see {@link serializedChars}).
|
|
846
|
+
*
|
|
847
|
+
* @param content - The file's full text.
|
|
848
|
+
* @param chars - The budget in serialized chars.
|
|
849
|
+
* @returns The line count, at least one.
|
|
850
|
+
*/
|
|
851
|
+
export function linesWithinSerialized(content, chars) {
|
|
852
|
+
let used = 0;
|
|
853
|
+
let lines = 0;
|
|
854
|
+
for (const line of content.split('\n')) {
|
|
855
|
+
// +2: the newline between two lines escapes to `\n`. The last line of a
|
|
856
|
+
// window has no trailing newline, so this counts 2 chars it does not use —
|
|
857
|
+
// a window that errs small is the safe direction here.
|
|
858
|
+
used += serializedChars(line) + 2;
|
|
859
|
+
if (used > chars && lines > 0)
|
|
860
|
+
break;
|
|
861
|
+
lines++;
|
|
862
|
+
}
|
|
863
|
+
return Math.max(1, lines);
|
|
864
|
+
}
|
|
744
865
|
/**
|
|
745
866
|
* Most files one batched `read_file` call returns. The cap exists so a survey
|
|
746
|
-
* of a large tree cannot swallow the turn's context in one result;
|
|
747
|
-
*
|
|
867
|
+
* of a large tree cannot swallow the turn's context in one result; every path
|
|
868
|
+
* past it is named in the result's `note` and listed with its own `error`, so
|
|
869
|
+
* the model continues from there.
|
|
748
870
|
*/
|
|
749
871
|
export const MAX_BATCH_READ_FILES = 25;
|
|
750
872
|
/**
|
|
751
|
-
*
|
|
752
|
-
*
|
|
873
|
+
* The most one `read_file` RESULT may carry, in chars of its JSON — every file
|
|
874
|
+
* the call returns, the envelope and the note included. That is how the request
|
|
875
|
+
* side counts a tool result, so it is how this tool counts one too
|
|
876
|
+
* ({@link serializedChars}).
|
|
877
|
+
*
|
|
878
|
+
* One tool result is ONE block in the request, and the request side caps a
|
|
879
|
+
* block: molecule-dev cuts any block over `MAX_TAIL_BLOCK_CHARS` (60,000) in
|
|
880
|
+
* place, keeping ~30 KB of each end. A batched read that crossed it lost its
|
|
881
|
+
* MIDDLE — the cut lands between two `"content"` values, so whole files fell
|
|
882
|
+
* out and the note in their place said only "N characters cut here" — while
|
|
883
|
+
* this tool's own note had promised each of those files that it "fits in one
|
|
884
|
+
* call, so it is not paged": X0 flash-m81 (2026-10-16) read ten e2e specs in
|
|
885
|
+
* one call (121,467 chars), lost the middle six, and bought them again in six
|
|
886
|
+
* further calls (52,908 chars).
|
|
887
|
+
*
|
|
888
|
+
* So the CALL gets the ceiling the request gives it. Whole files are returned
|
|
889
|
+
* up to it; the paths that did not fit are named in `note` and listed in
|
|
890
|
+
* `files` with their own `error`, the shape every other entry has. No file is
|
|
891
|
+
* ever cut, and no read result reaches the request-side cut. A test in
|
|
892
|
+
* molecule-dev's `context-fit` asserts the two ceilings still agree.
|
|
893
|
+
*/
|
|
894
|
+
export const MAX_READ_CALL_CHARS = 60_000;
|
|
895
|
+
/**
|
|
896
|
+
* Room inside one read result for what is not file content: the `path`, the
|
|
897
|
+
* offset/line fields, and the note.
|
|
898
|
+
*
|
|
899
|
+
* A batch sizes each file against what the call has left
|
|
900
|
+
* ({@link MAX_READ_CALL_CHARS} less this), so the entry that does not fit is
|
|
901
|
+
* caught here rather than by the request-side cut.
|
|
753
902
|
*/
|
|
754
|
-
export const
|
|
903
|
+
export const READ_RESULT_ENVELOPE_CHARS = 1_000;
|
|
755
904
|
/**
|
|
756
905
|
* The most one read_file call returns for ONE file when no window was asked
|
|
757
|
-
* for, in chars
|
|
758
|
-
* saying how to read the rest.
|
|
906
|
+
* for, in serialized chars: larger files come back as their first window, with
|
|
907
|
+
* a note saying how to read the rest.
|
|
759
908
|
*
|
|
760
909
|
* MAX_READ_SIZE (5 MB) is a bound on what the tool will open, not on what it
|
|
761
910
|
* should hand a model: a single batched read that returned a 1.2 MB README
|
|
762
911
|
* put the next three provider calls at 366k tokens (~330k of them fresh)
|
|
763
|
-
* against a 120k-token context cap (X0 run x5, call 26).
|
|
764
|
-
*
|
|
765
|
-
*
|
|
912
|
+
* against a 120k-token context cap (X0 run x5, call 26). This is the call's
|
|
913
|
+
* own ceiling ({@link MAX_READ_CALL_CHARS}) less the room its envelope takes,
|
|
914
|
+
* so one read result — one file or twenty-five — fits inside what the request
|
|
915
|
+
* side carries without being cut.
|
|
766
916
|
*/
|
|
767
|
-
export const MAX_READ_RETURN_CHARS =
|
|
917
|
+
export const MAX_READ_RETURN_CHARS = MAX_READ_CALL_CHARS - READ_RESULT_ENVELOPE_CHARS;
|
|
768
918
|
/**
|
|
769
919
|
* A file this small is returned whole on a second window of it (paging it costs a call each).
|
|
770
920
|
* The same bound as {@link MAX_READ_RETURN_CHARS}: the windowed-read note promises that a
|
|
@@ -1065,7 +1215,10 @@ export function runningCauseCommand(log, own) {
|
|
|
1065
1215
|
const read = `cat ${log}${own ? ` ${shellQuote(own)}` : ''} 2>/dev/null`;
|
|
1066
1216
|
return `${read} | tail -c ${REPEATING_CAUSE_TAIL_BYTES}`;
|
|
1067
1217
|
}
|
|
1068
|
-
/**
|
|
1218
|
+
/**
|
|
1219
|
+
* DISTINCT tests that must share one signature before a running suite's repeating cause is named.
|
|
1220
|
+
* A test the log prints twice, or a test that failed twice, is still one test.
|
|
1221
|
+
*/
|
|
1069
1222
|
export const REPEATING_FAILURE_MIN = 3;
|
|
1070
1223
|
/** ANSI SGR sequences — the `line` reporter writes cursor moves even into a file. */
|
|
1071
1224
|
// eslint-disable-next-line no-control-regex -- an escape sequence IS a control character
|
|
@@ -1130,23 +1283,106 @@ function failedTests(log) {
|
|
|
1130
1283
|
* while running "beforeEach" hook.`), which changes with the hook; the thrown error below it is
|
|
1131
1284
|
* what repeats verbatim, so it is preferred.
|
|
1132
1285
|
*
|
|
1133
|
-
* @param
|
|
1286
|
+
* @param body - The block's lines, from {@link failureBody}.
|
|
1134
1287
|
* @returns The error line, or null when the block has no body.
|
|
1135
1288
|
*/
|
|
1136
|
-
function
|
|
1137
|
-
const
|
|
1289
|
+
function failureErrorLine(body) {
|
|
1290
|
+
const thrown = body.find((l) => /^[A-Za-z_$][\w.$]*(?:Error|Exception)\b/.test(l));
|
|
1291
|
+
return thrown ?? body[0] ?? null;
|
|
1292
|
+
}
|
|
1293
|
+
/**
|
|
1294
|
+
* A failure block's own words, one per line: its body without the runner's progress lines (the
|
|
1295
|
+
* `list` reporter interleaves them with the `line` reporter's blocks), without a code frame or an
|
|
1296
|
+
* `at …` line (where the error happened, not what it said), and without the report's file list.
|
|
1297
|
+
*
|
|
1298
|
+
* @param block - One block from {@link failureBlocks}.
|
|
1299
|
+
* @returns The body's lines, ANSI stripped and trimmed.
|
|
1300
|
+
*/
|
|
1301
|
+
function failureBody(block) {
|
|
1302
|
+
return block
|
|
1138
1303
|
.split('\n')
|
|
1139
1304
|
.slice(1)
|
|
1140
1305
|
.map((l) => l.replace(ANSI_SGR, '').trim())
|
|
1141
|
-
// Progress lines the `list` reporter interleaves with the `line` reporter's blocks are not
|
|
1142
|
-
// the block's own words, and a code frame or an `at …` line is where the error happened, not
|
|
1143
|
-
// what it said.
|
|
1144
1306
|
.filter((l) => l &&
|
|
1145
1307
|
!LIST_RESULT_LINE.test(l) &&
|
|
1146
1308
|
!/^(?:at\s|\d+ \||>\s*\d+ \||\|)/.test(l) &&
|
|
1147
1309
|
!/^Error Context:/.test(l));
|
|
1148
|
-
|
|
1149
|
-
|
|
1310
|
+
}
|
|
1311
|
+
/**
|
|
1312
|
+
* What a failure block says the failure is ABOUT, when its error line does not already say it:
|
|
1313
|
+
* the locator it waited for, the navigation it waited on, or the locator and the expected/received
|
|
1314
|
+
* pair of a failed assertion.
|
|
1315
|
+
*
|
|
1316
|
+
* Without this, every `TimeoutError: page.press: Timeout 5000ms exceeded.` in a log is one
|
|
1317
|
+
* message, because that line names the ACTION (`page.press`) and never its subject. X1 flash-m84
|
|
1318
|
+
* (2026-10-16) had four such failures in one run — one waiting on
|
|
1319
|
+
* `button:has-text("Publish Article")`, one on `.article-preview h1`, one on
|
|
1320
|
+
* `.article-preview').first().locator('.author')`, one on a generated `h1:has-text("Test
|
|
1321
|
+
* Article …")` — and the run's summary called them two causes while its mid-run note called them
|
|
1322
|
+
* one. The subject is what separates causes; it is in the block, and nothing else in the block
|
|
1323
|
+
* carries it.
|
|
1324
|
+
*
|
|
1325
|
+
* @param body - The block's lines, from {@link failureBody}.
|
|
1326
|
+
* @returns The detail line, or null when the block names none.
|
|
1327
|
+
*/
|
|
1328
|
+
function failureDetail(body) {
|
|
1329
|
+
const waited = body.find((l) => /^-?\s*waiting for (?:locator\(|navigation|element|selector)/.test(l));
|
|
1330
|
+
if (waited !== undefined)
|
|
1331
|
+
return waited.replace(/^-\s*/, '');
|
|
1332
|
+
// A failed assertion names its locator and the two values that differ.
|
|
1333
|
+
const locator = body.find((l) => /^Locator:\s/.test(l));
|
|
1334
|
+
if (locator === undefined)
|
|
1335
|
+
return null;
|
|
1336
|
+
const parts = [
|
|
1337
|
+
locator,
|
|
1338
|
+
body.find((l) => /^Expected\b/.test(l)),
|
|
1339
|
+
body.find((l) => /^Received\b/.test(l)),
|
|
1340
|
+
];
|
|
1341
|
+
return parts.filter((p) => p !== undefined).join(' · ');
|
|
1342
|
+
}
|
|
1343
|
+
/**
|
|
1344
|
+
* A cause signature with the values that differ between two tests of ONE cause replaced by `<n>`:
|
|
1345
|
+
* a duration, a generated id or timestamp, a host's port.
|
|
1346
|
+
*
|
|
1347
|
+
* The duration is the one that matters. A suite that sets `actionTimeout: 5_000` — the X1
|
|
1348
|
+
* realworld suite does — spends `5 s` plus teardown on every "the locator never appeared"
|
|
1349
|
+
* failure, so `~7 s` is a property of the CONFIG, not of the cause, and four different missing
|
|
1350
|
+
* elements print the same number (X1 flash-m84, 2026-10-16). Two tests waiting on the same
|
|
1351
|
+
* locator for 5 s and for 10 s wait for the same thing; two waiting on different locators do not
|
|
1352
|
+
* become one cause by timing out together.
|
|
1353
|
+
*
|
|
1354
|
+
* @param s - An error line, with its detail.
|
|
1355
|
+
* @returns The signature.
|
|
1356
|
+
*/
|
|
1357
|
+
function normalizedSignature(s) {
|
|
1358
|
+
return (s
|
|
1359
|
+
// A duration: the suite's own timeout, in whatever unit the runner printed.
|
|
1360
|
+
.replace(/\b\d+(?:\.\d+)?\s*(?:ms|s|m)\b/g, '<n>')
|
|
1361
|
+
// A host's port. The shape is what makes it a port: a bare `:32` is a line number.
|
|
1362
|
+
.replace(/\b(\d{1,3}(?:\.\d{1,3}){3}|localhost|0\.0\.0\.0):\d+/g, '$1:<n>')
|
|
1363
|
+
// A generated id, a timestamp, a serial: five digits or more is no part of a selector.
|
|
1364
|
+
.replace(/\b\d{5,}\b/g, '<n>')
|
|
1365
|
+
.replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, '<id>'));
|
|
1366
|
+
}
|
|
1367
|
+
/**
|
|
1368
|
+
* What separates one cause from another in a `playwright test` log: the error CLASS plus the
|
|
1369
|
+
* normalized message — the error line, the thing it waited for or asserted on, and the values
|
|
1370
|
+
* with the test's own data normalized out.
|
|
1371
|
+
*
|
|
1372
|
+
* One test printed twice, or two tests failing the same way, share a signature; two tests whose
|
|
1373
|
+
* errors differ in their locator do not, however alike the rest of the line is.
|
|
1374
|
+
*
|
|
1375
|
+
* @param block - One block from {@link failureBlocks}.
|
|
1376
|
+
* @returns The signature, or null when the block has no body.
|
|
1377
|
+
*/
|
|
1378
|
+
function failureMessage(block) {
|
|
1379
|
+
const body = failureBody(block);
|
|
1380
|
+
const line = failureErrorLine(body);
|
|
1381
|
+
if (line === null)
|
|
1382
|
+
return null;
|
|
1383
|
+
const detail = failureDetail(body);
|
|
1384
|
+
const signature = detail !== null && !line.includes(detail) ? `${line} — ${detail}` : line;
|
|
1385
|
+
return normalizedSignature(signature);
|
|
1150
1386
|
}
|
|
1151
1387
|
/**
|
|
1152
1388
|
* The largest group of items sharing a key.
|
|
@@ -1162,42 +1398,84 @@ function largestGroup(groups) {
|
|
|
1162
1398
|
}
|
|
1163
1399
|
return best;
|
|
1164
1400
|
}
|
|
1401
|
+
/**
|
|
1402
|
+
* One test's identity in a log: where it sits and what it is called.
|
|
1403
|
+
*
|
|
1404
|
+
* A test printed twice by two reporters, or failed twice by a retry, is the same test — the
|
|
1405
|
+
* identity is what it is called and where it lives, never which line of the log it came from.
|
|
1406
|
+
*
|
|
1407
|
+
* @param t - A failed test.
|
|
1408
|
+
* @returns The key two prints of one test share.
|
|
1409
|
+
*/
|
|
1410
|
+
function testKey(t) {
|
|
1411
|
+
return `${t.where}|${t.title}`;
|
|
1412
|
+
}
|
|
1413
|
+
/**
|
|
1414
|
+
* The same tests with each one kept once, in first-seen order.
|
|
1415
|
+
*
|
|
1416
|
+
* @param tests - Failed tests, as the log printed them (possibly more than once each).
|
|
1417
|
+
* @returns The distinct tests.
|
|
1418
|
+
*/
|
|
1419
|
+
function distinctTests(tests) {
|
|
1420
|
+
const seen = new Set();
|
|
1421
|
+
return tests.filter((t) => {
|
|
1422
|
+
const key = testKey(t);
|
|
1423
|
+
if (seen.has(key))
|
|
1424
|
+
return false;
|
|
1425
|
+
seen.add(key);
|
|
1426
|
+
return true;
|
|
1427
|
+
});
|
|
1428
|
+
}
|
|
1165
1429
|
/**
|
|
1166
1430
|
* The cause repeating among a running suite's failures, or null.
|
|
1167
1431
|
*
|
|
1168
1432
|
* Playwright's `list` reporter prints one line per test as it finishes — its mark, its title
|
|
1169
1433
|
* path and how long it ran — and defers each failure's error to the summary at the end, so
|
|
1170
|
-
* mid-run the only signature those lines carry is the duration.
|
|
1171
|
-
* failure bodies
|
|
1172
|
-
*
|
|
1173
|
-
*
|
|
1434
|
+
* mid-run the only signature those lines carry is the duration. Where the log carries the
|
|
1435
|
+
* failure bodies (a `line` reporter's output, or a finished run's summary) the signature is the
|
|
1436
|
+
* error itself — {@link failureMessage}: the error class plus the locator or the values that
|
|
1437
|
+
* differ — which is stronger (it needs no assumption about how long a failure should take), so
|
|
1438
|
+
* the message is looked for first and the duration is then left with the tests whose bodies the
|
|
1439
|
+
* log does NOT carry.
|
|
1174
1440
|
*
|
|
1175
1441
|
* Either way the point is the same: a suite whose failures repeat one cause spends its whole
|
|
1176
1442
|
* runtime proving that one cause, and the executor watching it can be told so at the third
|
|
1177
1443
|
* failure instead of at the last. X1 flash-m76 walked 15 tests in 4.0 minutes where 15.6 s
|
|
1178
1444
|
* each was the test timeout and one missing `navigate('/')` was the cause of all of them.
|
|
1179
1445
|
*
|
|
1446
|
+
* The quorum is of DISTINCT tests — {@link REPEATING_FAILURE_MIN} of them — because a log that
|
|
1447
|
+
* prints each failure twice (a second reporter, a retry) had X1 flash-m83 told that four of its
|
|
1448
|
+
* tests had failed the same way when two had, so the executor stopped a 139-test walk at test 21
|
|
1449
|
+
* with one cause in hand (2026-10-16).
|
|
1450
|
+
*
|
|
1180
1451
|
* @param log - The log so far.
|
|
1181
1452
|
* @returns The repeating cause, or null when no signature has {@link REPEATING_FAILURE_MIN}
|
|
1182
|
-
*
|
|
1453
|
+
* distinct tests behind it.
|
|
1183
1454
|
*/
|
|
1184
1455
|
export function repeatingFailure(log) {
|
|
1185
1456
|
const text = log.replace(ANSI_SGR, '');
|
|
1186
|
-
// The failure BODIES first: a message repeating verbatim names the cause itself.
|
|
1457
|
+
// The failure BODIES first: a message repeating verbatim names the cause itself. Grouped by
|
|
1458
|
+
// DISTINCT test, so the largest group is the cause the most tests share — not the one whose
|
|
1459
|
+
// failures happen to have been printed the most times.
|
|
1187
1460
|
const byMessage = new Map();
|
|
1461
|
+
const withBody = new Set();
|
|
1188
1462
|
for (const block of failureBlocks(text)) {
|
|
1463
|
+
const m = FAILURE_BLOCK_HEADER.exec(block.split('\n')[0] ?? '');
|
|
1464
|
+
const { where, title } = splitTitlePath(m?.[1] ?? '');
|
|
1189
1465
|
const message = failureMessage(block);
|
|
1190
1466
|
if (message === null)
|
|
1191
1467
|
continue;
|
|
1192
|
-
|
|
1193
|
-
const { where, title } = splitTitlePath(m?.[1] ?? '');
|
|
1468
|
+
withBody.add(testKey({ where, title, seconds: null }));
|
|
1194
1469
|
const group = byMessage.get(message);
|
|
1195
1470
|
if (group)
|
|
1196
1471
|
group.push({ where, title, seconds: null });
|
|
1197
1472
|
else
|
|
1198
1473
|
byMessage.set(message, [{ where, title, seconds: null }]);
|
|
1199
1474
|
}
|
|
1200
|
-
const
|
|
1475
|
+
const distinctByMessage = new Map();
|
|
1476
|
+
for (const [message, members] of byMessage)
|
|
1477
|
+
distinctByMessage.set(message, distinctTests(members));
|
|
1478
|
+
const messageGroup = largestGroup(distinctByMessage);
|
|
1201
1479
|
if (messageGroup && messageGroup.members.length >= REPEATING_FAILURE_MIN)
|
|
1202
1480
|
return {
|
|
1203
1481
|
count: messageGroup.members.length,
|
|
@@ -1205,10 +1483,16 @@ export function repeatingFailure(log) {
|
|
|
1205
1483
|
seconds: null,
|
|
1206
1484
|
tests: messageGroup.members,
|
|
1207
1485
|
};
|
|
1208
|
-
// Otherwise the durations
|
|
1209
|
-
//
|
|
1210
|
-
//
|
|
1211
|
-
|
|
1486
|
+
// Otherwise the durations — of the tests the log says nothing ELSE about. A test whose failure
|
|
1487
|
+
// body is in the log has a signature, and how long it took must not overrule it: a suite whose
|
|
1488
|
+
// config sets a timeout makes every failure that waits for anything land on the same number, so
|
|
1489
|
+
// the duration would merge causes the bodies just separated (X1 flash-m84, 2026-10-16 — four
|
|
1490
|
+
// failures at 6.8-7.4 s, four locators, `actionTimeout: 5_000`). What is left is the shape this
|
|
1491
|
+
// was written for: a `list`-only log, or a run whose bodies have not been printed yet, where
|
|
1492
|
+
// the duration is the ONLY signature the log carries. A test that runs to its timeout prints
|
|
1493
|
+
// slightly different durations on each attempt (15.6 s, 15.1 s, 15.0 s), so the band is 0.5 s
|
|
1494
|
+
// or 5% of the duration, whichever is larger.
|
|
1495
|
+
const tests = distinctTests(failedTests(text)).filter((t) => !withBody.has(testKey(t)));
|
|
1212
1496
|
const withSeconds = tests.filter((t) => t.seconds !== null);
|
|
1213
1497
|
let best = [];
|
|
1214
1498
|
for (const t of withSeconds) {
|
|
@@ -1224,37 +1508,241 @@ export function repeatingFailure(log) {
|
|
|
1224
1508
|
}
|
|
1225
1509
|
/** How many of a repeating cause's failures the note lists before it says "and N more". */
|
|
1226
1510
|
export const REPEATING_FAILURE_LISTED = 5;
|
|
1511
|
+
/** How many causes a finished run's summary lists before it says "and N more". */
|
|
1512
|
+
export const FAILURE_CAUSES_LISTED = 5;
|
|
1513
|
+
/** How many of one cause's tests that summary names before it says "and N more". */
|
|
1514
|
+
export const FAILURE_CAUSE_TESTS_LISTED = 6;
|
|
1227
1515
|
/**
|
|
1228
|
-
* The shortest unambiguous tail of a logged path (`…/specs/e2e/articles.spec.ts
|
|
1229
|
-
* `e2e/articles.spec.ts
|
|
1516
|
+
* The shortest unambiguous tail of a logged path's FILE (`…/specs/e2e/articles.spec.ts` →
|
|
1517
|
+
* `e2e/articles.spec.ts`).
|
|
1230
1518
|
*
|
|
1231
|
-
* @param
|
|
1519
|
+
* @param file - A path as {@link specFile} returns it.
|
|
1232
1520
|
* @returns The last two path segments.
|
|
1233
1521
|
*/
|
|
1234
|
-
function
|
|
1235
|
-
|
|
1236
|
-
|
|
1522
|
+
function shortFile(file) {
|
|
1523
|
+
return file.split('/').slice(-2).join('/');
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* The spec file a logged location names (`…/e2e/articles.spec.ts:32` → `…/e2e/articles.spec.ts`).
|
|
1527
|
+
*
|
|
1528
|
+
* @param where - A location from {@link FailedTest.where}.
|
|
1529
|
+
* @returns The path without its line number.
|
|
1530
|
+
*/
|
|
1531
|
+
function specFile(where) {
|
|
1532
|
+
return where.replace(/:\d+$/, '');
|
|
1533
|
+
}
|
|
1534
|
+
/**
|
|
1535
|
+
* The mark of every test result a log names, in the order it printed them (`✘` failed, `✓`
|
|
1536
|
+
* passed).
|
|
1537
|
+
*
|
|
1538
|
+
* A `line`-only log writes none: `[n/N] <title>` carries no mark. An empty list therefore means
|
|
1539
|
+
* the log cannot say whether anything passed, not that nothing did.
|
|
1540
|
+
*
|
|
1541
|
+
* @param text - The log (ANSI already stripped).
|
|
1542
|
+
* @returns The marks.
|
|
1543
|
+
*/
|
|
1544
|
+
function resultMarks(text) {
|
|
1545
|
+
const out = [];
|
|
1546
|
+
for (const line of text.split('\n')) {
|
|
1547
|
+
const m = LIST_RESULT_LINE.exec(line);
|
|
1548
|
+
if (m)
|
|
1549
|
+
out.push(m[1]);
|
|
1550
|
+
}
|
|
1551
|
+
return out;
|
|
1552
|
+
}
|
|
1553
|
+
/**
|
|
1554
|
+
* Every test the log's numbered failure blocks name, in the order it names them.
|
|
1555
|
+
*
|
|
1556
|
+
* @param text - The log (ANSI already stripped).
|
|
1557
|
+
* @returns The failed tests, with the location and title their header carries.
|
|
1558
|
+
*/
|
|
1559
|
+
function failureBlockTests(text) {
|
|
1560
|
+
return failureBlocks(text).map((block) => {
|
|
1561
|
+
const m = FAILURE_BLOCK_HEADER.exec(block.split('\n')[0] ?? '');
|
|
1562
|
+
const { where, title } = splitTitlePath(m?.[1] ?? '');
|
|
1563
|
+
return { where, title, seconds: null };
|
|
1564
|
+
});
|
|
1565
|
+
}
|
|
1566
|
+
/**
|
|
1567
|
+
* Distinct spec FILES a repeating cause must span — with every failure sharing it and nothing
|
|
1568
|
+
* passing since — before a run is called broken for the whole suite.
|
|
1569
|
+
*
|
|
1570
|
+
* One file's failures can be that file's own setup, so a cause confined to one file is never
|
|
1571
|
+
* this. Across {@link UNIVERSAL_CAUSE_FILES} files, on the other hand, whatever it is is a
|
|
1572
|
+
* property of the run rather than of a page.
|
|
1573
|
+
*/
|
|
1574
|
+
export const UNIVERSAL_CAUSE_FILES = 3;
|
|
1575
|
+
/**
|
|
1576
|
+
* Whether a repeating cause is failing the whole run — the one case where stopping it is the
|
|
1577
|
+
* right call.
|
|
1578
|
+
*
|
|
1579
|
+
* The test is: the cause accounts for every failure the log names, it has hit
|
|
1580
|
+
* {@link UNIVERSAL_CAUSE_FILES} spec files or more, and nothing has passed since its first
|
|
1581
|
+
* failure. An app that is not up, a 5xx on every request, or a selector that is missing on every
|
|
1582
|
+
* page all look like that, and every remaining test will fail the same way — so the run's
|
|
1583
|
+
* remaining minutes measure nothing. Anything less is NOT this: a cause that stops at one file,
|
|
1584
|
+
* or that leaves other tests passing, or that shares the run with failures of another shape,
|
|
1585
|
+
* leaves the run with something to say, and the run is the only thing that will say it. X1
|
|
1586
|
+
* flash-m83 (2026-10-16) stopped eight local runs at their first repeating cause over 14.5
|
|
1587
|
+
* minutes and never finished one; three of its deploys each fixed a single cause that a finished
|
|
1588
|
+
* run would have listed alongside the others.
|
|
1589
|
+
*
|
|
1590
|
+
* @param log - The log so far.
|
|
1591
|
+
* @param found - The repeating cause {@link repeatingFailure} found in it.
|
|
1592
|
+
* @returns The cause's count and files, or null when the run is not failing whole.
|
|
1593
|
+
*/
|
|
1594
|
+
export function universalCause(log, found) {
|
|
1595
|
+
const text = log.replace(ANSI_SGR, '');
|
|
1596
|
+
const files = [...new Set(found.tests.map((t) => specFile(t.where)).filter(Boolean))];
|
|
1597
|
+
if (files.length < UNIVERSAL_CAUSE_FILES)
|
|
1598
|
+
return null;
|
|
1599
|
+
// Every failure the log names has to be this cause: another cause anywhere in the run means
|
|
1600
|
+
// the run still has something to say, and stopping it throws that away.
|
|
1601
|
+
const named = Math.max(distinctTests(failedTests(text)).length, distinctTests(failureBlockTests(text)).length);
|
|
1602
|
+
if (found.count < named)
|
|
1603
|
+
return null;
|
|
1604
|
+
// And nothing may have passed since its first failure. A pass after it means the app answered
|
|
1605
|
+
// a request, so a test that still fails the same way fails for a reason this run has not seen.
|
|
1606
|
+
const marks = resultMarks(text);
|
|
1607
|
+
const firstFailure = marks.indexOf('✘');
|
|
1608
|
+
if (firstFailure === -1 || marks.slice(firstFailure).includes('✓'))
|
|
1609
|
+
return null;
|
|
1610
|
+
return { count: found.count, files };
|
|
1611
|
+
}
|
|
1612
|
+
/**
|
|
1613
|
+
* Every failure a run names, grouped by the cause behind it — one group per cause, with the
|
|
1614
|
+
* tests it hit.
|
|
1615
|
+
*
|
|
1616
|
+
* The point is that ONE run yields EVERY cause. A Playwright report prints its failures one
|
|
1617
|
+
* after another with no statement of which of them share a cause, so a run of 19 failures is 19
|
|
1618
|
+
* bodies to read; X1 flash-m83 (2026-10-16) spent three deploys (6.7 minutes) each fixing a
|
|
1619
|
+
* single cause, two of them found by runs stopped before they walked the rest.
|
|
1620
|
+
*
|
|
1621
|
+
* The signature is the cause itself where the log carries the failures' bodies (a finished run,
|
|
1622
|
+
* or a `line` reporter) — {@link failureMessage}: the error class plus the locator or the values
|
|
1623
|
+
* that differ — and how long they took where it does not (a still-running `list` log, which is
|
|
1624
|
+
* why a duration group is labelled as a duration and not as a proven cause). A test the log
|
|
1625
|
+
* printed twice, or failed twice, is one test.
|
|
1626
|
+
*
|
|
1627
|
+
* @param log - The log of a finished or running run.
|
|
1628
|
+
* @returns The causes, the one with the most tests first.
|
|
1629
|
+
*/
|
|
1630
|
+
export function failuresByCause(log) {
|
|
1631
|
+
const text = log.replace(ANSI_SGR, '');
|
|
1632
|
+
const byMessage = new Map();
|
|
1633
|
+
for (const block of failureBlocks(text)) {
|
|
1634
|
+
const message = failureMessage(block);
|
|
1635
|
+
if (message === null)
|
|
1636
|
+
continue;
|
|
1637
|
+
const m = FAILURE_BLOCK_HEADER.exec(block.split('\n')[0] ?? '');
|
|
1638
|
+
const { where, title } = splitTitlePath(m?.[1] ?? '');
|
|
1639
|
+
const group = byMessage.get(message);
|
|
1640
|
+
if (group)
|
|
1641
|
+
group.push({ where, title, seconds: null });
|
|
1642
|
+
else
|
|
1643
|
+
byMessage.set(message, [{ where, title, seconds: null }]);
|
|
1644
|
+
}
|
|
1645
|
+
const causes = [];
|
|
1646
|
+
for (const [message, members] of byMessage) {
|
|
1647
|
+
const tests = distinctTests(members);
|
|
1648
|
+
if (tests.length > 0)
|
|
1649
|
+
causes.push({ message, seconds: null, tests });
|
|
1650
|
+
}
|
|
1651
|
+
// The failures with no body of their own — every failure of a still-running `list` log — are
|
|
1652
|
+
// grouped by how long they took: a band is opened by its first test and takes the ones that
|
|
1653
|
+
// then ran the same length, the same band {@link repeatingFailure} uses.
|
|
1654
|
+
const claimed = new Set(causes.flatMap((c) => c.tests.map(testKey)));
|
|
1655
|
+
const unclaimed = distinctTests(failedTests(text)).filter((t) => t.seconds !== null && !claimed.has(testKey(t)));
|
|
1656
|
+
const byDuration = [];
|
|
1657
|
+
for (const t of unclaimed) {
|
|
1658
|
+
const band = Math.max(0.5, t.seconds * 0.05);
|
|
1659
|
+
const open = byDuration.find((g) => Math.abs(g.seconds - t.seconds) <= Math.max(band, g.seconds * 0.05));
|
|
1660
|
+
if (open)
|
|
1661
|
+
open.tests.push(t);
|
|
1662
|
+
else
|
|
1663
|
+
byDuration.push({ seconds: t.seconds, tests: [t] });
|
|
1664
|
+
}
|
|
1665
|
+
for (const g of byDuration)
|
|
1666
|
+
causes.push({ message: null, seconds: g.seconds, tests: g.tests });
|
|
1667
|
+
return causes.sort((a, b) => b.tests.length - a.tests.length);
|
|
1668
|
+
}
|
|
1669
|
+
/**
|
|
1670
|
+
* The one-line-per-cause summary of a finished run's failures, or null when the run named fewer
|
|
1671
|
+
* than two of them.
|
|
1672
|
+
*
|
|
1673
|
+
* This is what makes a run worth finishing: every cause the run found, in the result the
|
|
1674
|
+
* executor reads, so one round of fixes covers all of them rather than the one cause a run
|
|
1675
|
+
* stopped early happened to reach.
|
|
1676
|
+
*
|
|
1677
|
+
* @param log - The finished run's output.
|
|
1678
|
+
* @returns The note, or null.
|
|
1679
|
+
*/
|
|
1680
|
+
export function failureCausesNote(log) {
|
|
1681
|
+
const causes = failuresByCause(log);
|
|
1682
|
+
const total = causes.reduce((n, c) => n + c.tests.length, 0);
|
|
1683
|
+
if (total < 2)
|
|
1684
|
+
return null;
|
|
1685
|
+
const listed = causes.slice(0, FAILURE_CAUSES_LISTED);
|
|
1686
|
+
const lines = listed.map((c, i) => {
|
|
1687
|
+
// The path the log itself wrote, never a shortened one: it is relative to the command's own
|
|
1688
|
+
// cwd, so it is the form the executor can paste back (X1 flash-m82 read a shortened
|
|
1689
|
+
// `e2e/auth.spec.ts:19` as the suite's own stale starter spec, rewrote its Playwright config
|
|
1690
|
+
// over it and deleted the app's tests — 4 minutes of a 60-minute run on a working config).
|
|
1691
|
+
const tests = c.tests.slice(0, FAILURE_CAUSE_TESTS_LISTED).map((t) => t.where || t.title);
|
|
1692
|
+
const more = c.tests.length > tests.length ? `, … and ${c.tests.length - tests.length} more` : '';
|
|
1693
|
+
const signature = c.message !== null
|
|
1694
|
+
? `\`${c.message}\``
|
|
1695
|
+
: `each after ~${c.seconds}s (a shared duration, not a proven cause)`;
|
|
1696
|
+
const files = new Set(c.tests.map((t) => specFile(t.where)).filter(Boolean)).size;
|
|
1697
|
+
return (` ${i + 1}. ${signature} — ${c.tests.length} test${c.tests.length === 1 ? '' : 's'} ` +
|
|
1698
|
+
`in ${files} file${files === 1 ? '' : 's'}: ${tests.join(', ')}${more}`);
|
|
1699
|
+
});
|
|
1700
|
+
const moreCauses = causes.length > listed.length ? `\n … and ${causes.length - listed.length} more causes` : '';
|
|
1701
|
+
return (`${total} failure${total === 1 ? '' : 's'} in this run, grouped by cause — fix every cause ` +
|
|
1702
|
+
`below before the next deploy:\n${lines.join('\n')}${moreCauses}`);
|
|
1237
1703
|
}
|
|
1238
1704
|
/**
|
|
1239
|
-
* The sentence that names a cause repeating among a running suite's failures,
|
|
1240
|
-
* that stops the run so the cause can be fixed first.
|
|
1705
|
+
* The sentence that names a cause repeating among a running suite's failures, mid-run.
|
|
1241
1706
|
*
|
|
1242
1707
|
* Kept here, beside {@link repeatingFailure}, so the two stay in step: every word below is about
|
|
1243
|
-
* what that function could actually see in the log
|
|
1708
|
+
* what that function could actually see in the log — including the count, which is of DISTINCT
|
|
1709
|
+
* tests, so the note never names one or two tests as a cause.
|
|
1710
|
+
*
|
|
1711
|
+
* The note names the cause early so the executor can START FIXING while the run keeps walking,
|
|
1712
|
+
* and it asks for the run to be stopped only when {@link universalCause} says the cause is
|
|
1713
|
+
* failing the whole run. X1 flash-m76 (2026-10-16) spent 4.0 minutes walking 15 tests whose one
|
|
1714
|
+
* cause was certain at the third, 47 s in, and the note answers exactly that; X1 flash-m83 then
|
|
1715
|
+
* stopped eight runs at their first repeating cause in 14.5 minutes and never finished one, so
|
|
1716
|
+
* the note must not say to stop when the run still has other causes to name.
|
|
1244
1717
|
*
|
|
1245
1718
|
* @param found - What {@link repeatingFailure} found.
|
|
1246
|
-
* @param o - The run's facts: its `taskId`, its `log` path,
|
|
1247
|
-
* stops it
|
|
1719
|
+
* @param o - The run's facts: its `taskId`, its `log` path, `stop` — the shell command that
|
|
1720
|
+
* stops it, printed only for a {@link UniversalCause} — and `universal`, what
|
|
1721
|
+
* {@link universalCause} found (absent means the run is not failing whole).
|
|
1248
1722
|
* @returns The note.
|
|
1249
1723
|
*/
|
|
1250
1724
|
export function repeatingFailureNote(found, o) {
|
|
1251
1725
|
const listed = found.tests.slice(0, REPEATING_FAILURE_LISTED);
|
|
1726
|
+
// Every row names its test by the SAME path the re-run hint below uses (`shortWhere`
|
|
1727
|
+
// would print this one file under two names: X1 flash-m82 read `✘ e2e/auth.spec.ts:19`
|
|
1728
|
+
// here against `Re-run fixtures/realworld/specs/e2e/auth.spec.ts:19 alone` three lines
|
|
1729
|
+
// down, took the suite-relative form for "the project's own stale starter spec" and
|
|
1730
|
+
// rewrote its Playwright config over it, deleted the app's tests and wrote its own —
|
|
1731
|
+
// 4 minutes of a 60-minute run, on a run whose config was right). The path the log
|
|
1732
|
+
// wrote is relative to the command's own cwd, so it is the one the executor can paste
|
|
1733
|
+
// back into a command.
|
|
1252
1734
|
const rows = listed
|
|
1253
|
-
.map((t) => ` ✘ ${[t.where
|
|
1735
|
+
.map((t) => ` ✘ ${[t.where, t.title].filter(Boolean).join(' › ')}` +
|
|
1254
1736
|
(t.seconds === null ? '' : ` (${t.seconds}s)`))
|
|
1255
1737
|
.join('\n');
|
|
1256
1738
|
const more = found.count > listed.length ? `\n … and ${found.count - listed.length} more` : '';
|
|
1257
|
-
|
|
1739
|
+
// Two signatures, and only ONE of them is a cause. An error repeating verbatim is the cause
|
|
1740
|
+
// itself; a duration repeating is what a suite's own `actionTimeout` does to every failure that
|
|
1741
|
+
// waits for anything, so the note must not call it one cause (X1 flash-m84, 2026-10-16: four
|
|
1742
|
+
// failures at 6.8-7.4 s, four different missing elements, and this sentence told the executor
|
|
1743
|
+
// "one cause is repeating").
|
|
1744
|
+
const isCause = found.message !== null;
|
|
1745
|
+
const signature = isCause
|
|
1258
1746
|
? `every one with \`${found.message}\``
|
|
1259
1747
|
: `each after ~${found.seconds}s — the same duration to within half a second`;
|
|
1260
1748
|
const first = found.tests[0];
|
|
@@ -1263,10 +1751,33 @@ export function repeatingFailureNote(found, o) {
|
|
|
1263
1751
|
: first?.where
|
|
1264
1752
|
? ` Re-run \`${first.where}\` alone to see its error in full.`
|
|
1265
1753
|
: '';
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1754
|
+
const where = `task ${o.taskId}, log ${o.log}`;
|
|
1755
|
+
if (o.universal) {
|
|
1756
|
+
const files = o.universal.files.map(shortFile);
|
|
1757
|
+
const shown = files.slice(0, 4).join(', ');
|
|
1758
|
+
const moreFiles = files.length > 4 ? `, … and ${files.length - 4} more` : '';
|
|
1759
|
+
return (`Every test this run has walked so far has failed the same way — all ${found.count} of ` +
|
|
1760
|
+
`them, ${signature} — across ${files.length} files (${shown}${moreFiles}), with nothing ` +
|
|
1761
|
+
`passing since the first:\n${rows}${more}\n` +
|
|
1762
|
+
`Every remaining test will fail the same way too, so walking the rest of this run only ` +
|
|
1763
|
+
`repeats this one cause. Stop it and fix that cause first: \`exec_command: ${o.stop}\`, ` +
|
|
1764
|
+
`then run the whole suite again. It is still going (${where}).${reproduce}`);
|
|
1765
|
+
}
|
|
1766
|
+
const keepGoing = `Let the rest of the run finish — it is the only thing that will name the causes that are NOT ` +
|
|
1767
|
+
`this one, and a run stopped at its first repeating cause costs a second whole run to find ` +
|
|
1768
|
+
`them. Stop it only if every test it walks fails this same way (the app unreachable, a 5xx on ` +
|
|
1769
|
+
`every request, the same selector timing out on every test of every file so far). ` +
|
|
1770
|
+
`It is still going (${where}).`;
|
|
1771
|
+
if (isCause)
|
|
1772
|
+
return (`${found.count} different tests in this run have already failed the same way — ${signature} — ` +
|
|
1773
|
+
`so one cause is repeating:\n${rows}${more}\n` +
|
|
1774
|
+
`Start fixing it now, while the run keeps walking.${reproduce} ${keepGoing}`);
|
|
1775
|
+
return (`${found.count} different tests in this run each failed after ~${found.seconds}s — the same ` +
|
|
1776
|
+
`duration to within half a second. A shared duration is a lead, not a cause: a suite whose ` +
|
|
1777
|
+
`config sets a timeout spends that timeout on every failure that waits for anything, so ` +
|
|
1778
|
+
`different missing elements land on the same number. Read these failures before fixing one ` +
|
|
1779
|
+
`of them:\n${rows}${more}\n` +
|
|
1780
|
+
`Start on them now, while the run keeps walking.${reproduce} ${keepGoing}`);
|
|
1270
1781
|
}
|
|
1271
1782
|
/**
|
|
1272
1783
|
* Attempt a whitespace-tolerant replacement when an exact `old_string` match
|