@molecule/api-ai-tools 1.5.20 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +302 -38
- package/dist/command-position.d.ts +62 -0
- package/dist/command-position.d.ts.map +1 -0
- package/dist/command-position.js +352 -0
- package/dist/index.d.ts +8 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -3
- package/dist/no-test-run.d.ts +13 -0
- package/dist/no-test-run.d.ts.map +1 -1
- package/dist/no-test-run.js +31 -22
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +10 -4
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +113 -49
- package/dist/utilities.d.ts +189 -38
- package/dist/utilities.d.ts.map +1 -1
- package/dist/utilities.js +463 -85
- package/package.json +1 -1
package/dist/utilities.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* @module
|
|
6
6
|
*/
|
|
7
7
|
import { posix } from 'path';
|
|
8
|
+
import { commandSegments } from './command-position.js';
|
|
8
9
|
import { commandRunsNoTests } from './no-test-run.js';
|
|
9
10
|
/**
|
|
10
11
|
* Shell-safe quoting using single quotes. Unlike JSON.stringify (double quotes),
|
|
@@ -288,8 +289,43 @@ export function outputRedirectTarget(command) {
|
|
|
288
289
|
}
|
|
289
290
|
/** A trailing `| tail -N` / `| tail -n N` (stderr redirects before it stay). */
|
|
290
291
|
const TRAILING_TAIL = /\s*\|\s*tail(?:\s+-n)?\s+[-+]?\d+\s*$/;
|
|
291
|
-
/** `--reporter
|
|
292
|
-
const
|
|
292
|
+
/** `--reporter line` / `--reporter=line` / `--reporter="list,line"`, with its value. */
|
|
293
|
+
const REPORTER_FLAG = /--reporter(=|\s+)(?:"([^"]*)"|'([^']*)'|(\S+))/g;
|
|
294
|
+
/**
|
|
295
|
+
* The reporters a step's LAST `--reporter` names (`--reporter=list,line` → `['list', 'line']`).
|
|
296
|
+
*
|
|
297
|
+
* Playwright takes the last `--reporter`, so an earlier one is not what runs.
|
|
298
|
+
*
|
|
299
|
+
* @param step - The command's last step.
|
|
300
|
+
* @returns The flag's span and its reporters, or null when the step names none.
|
|
301
|
+
*/
|
|
302
|
+
function lastReporterFlag(step) {
|
|
303
|
+
REPORTER_FLAG.lastIndex = 0;
|
|
304
|
+
let last = null;
|
|
305
|
+
for (let m = REPORTER_FLAG.exec(step); m !== null; m = REPORTER_FLAG.exec(step))
|
|
306
|
+
last = m;
|
|
307
|
+
if (!last)
|
|
308
|
+
return null;
|
|
309
|
+
const value = last[2] ?? last[3] ?? last[4] ?? '';
|
|
310
|
+
return {
|
|
311
|
+
start: last.index,
|
|
312
|
+
end: last.index + last[0].length,
|
|
313
|
+
separator: last[1] ?? '=',
|
|
314
|
+
values: value
|
|
315
|
+
.split(',')
|
|
316
|
+
.map((v) => v.trim())
|
|
317
|
+
.filter(Boolean),
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* The reporters that print one line (or one mark) per test into the log, `list` among them.
|
|
322
|
+
*
|
|
323
|
+
* `list` is the only one of these the platform can read back — `suite-fail-fast.ts`'s fail-fast
|
|
324
|
+
* note and the Tests bar both parse its ` ✓/✘ <n> … (…)` lines — so it is the one a background
|
|
325
|
+
* run is put on. The rest are here because they print each failure TOO: a log with two of them
|
|
326
|
+
* carries every result twice.
|
|
327
|
+
*/
|
|
328
|
+
const PRINTING_REPORTERS = new Set(['list', 'line', 'dot', 'basic', 'spec']);
|
|
293
329
|
/**
|
|
294
330
|
* Rewrite a command about to run in the background so its log grows while it
|
|
295
331
|
* runs. Two shapes left the log empty until exit, and an empty log is what
|
|
@@ -302,19 +338,18 @@ const LINE_REPORTER_FLAG = /--reporter([= ])line\b/;
|
|
|
302
338
|
* often `html`, which prints nothing per test. `--reporter=list` is added
|
|
303
339
|
* when the playwright run is the command's last step and nothing is piped or
|
|
304
340
|
* redirected from it: it writes each result as it finishes, and it is the
|
|
305
|
-
* shape the platform reads a background suite back in
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
*
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
* `TimeoutError: page.waitForURL` from one missing `navigate('/')`.
|
|
341
|
+
* shape the platform reads a background suite back in.
|
|
342
|
+
*
|
|
343
|
+
* The run ends up with exactly ONE reporter that prints each failure — `list`. A run that names
|
|
344
|
+
* any other printing reporter has it REPLACED, never joined: two of them write every test result
|
|
345
|
+
* into the log twice, which on X1 flash-m83 (2026-10-16) made a `--reporter=list,line` run (its
|
|
346
|
+
* `line` added by this function one run earlier) report two distinct failures as "4 of this run's
|
|
347
|
+
* tests have already failed the same way", and the executor stopped a 139-test walk at test 21
|
|
348
|
+
* with a single cause in hand. `line` alone is no use either: it writes `[n/N] <title>` with no
|
|
349
|
+
* ✓/✘ mark and no duration, so on X1 flash-m62 an 8.3-minute walk of one suite could never trip
|
|
350
|
+
* the fail-fast note. A reporter set that writes a machine format to a file (`json`, `html`,
|
|
351
|
+
* `junit`) is the
|
|
352
|
+
* executor's own and is left exactly as it is — so is any reporter this build does not know.
|
|
318
353
|
*
|
|
319
354
|
* Only the command's last step is touched, and only its reporter, and only when that step runs
|
|
320
355
|
* something: `npx playwright test --list` takes two seconds and writes nothing per test, so
|
|
@@ -353,40 +388,54 @@ export function streamingBackgroundCommand(command) {
|
|
|
353
388
|
// which had dropped the `--retries=0` it used one command earlier. The uncapped proof run
|
|
354
389
|
// keeps the suite's retries.
|
|
355
390
|
const iterating = /--max-failures\b|--last-failed\b|(?:^|\s)(?:-g|--grep)\b|\S+\.(?:spec|test)\.[cm]?[jt]sx?(?::\d+)?\b/.test(lastStep);
|
|
356
|
-
//
|
|
357
|
-
// the
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
391
|
+
// The last step's reporters: one printer, and it is `list`. Every branch below only ever moves
|
|
392
|
+
// the command TO that shape — a reporter set is never left with two printers in it.
|
|
393
|
+
const reporters = finalPlaywrightRun ? lastReporterFlag(lastStep) : null;
|
|
394
|
+
if (reporters) {
|
|
395
|
+
const printing = reporters.values.filter((v) => PRINTING_REPORTERS.has(v));
|
|
396
|
+
// Already exactly `list`: the shape, and nothing printing beside it. Left verbatim — the
|
|
397
|
+
// executor's own spelling of the command is not rewritten for nothing.
|
|
398
|
+
if (printing.length > 0 && !(printing.length === 1 && printing[0] === 'list')) {
|
|
399
|
+
// The printing reporters become ONE `list`, in the place of the first of them; every other
|
|
400
|
+
// reporter (a `json`/`html`/`junit` file the executor asked for, or one this build does not
|
|
401
|
+
// know) is left where it was.
|
|
402
|
+
let placed = false;
|
|
403
|
+
const kept = reporters.values
|
|
404
|
+
.map((v) => {
|
|
405
|
+
if (!PRINTING_REPORTERS.has(v))
|
|
406
|
+
return v;
|
|
407
|
+
if (placed)
|
|
408
|
+
return null;
|
|
409
|
+
placed = true;
|
|
410
|
+
return 'list';
|
|
411
|
+
})
|
|
412
|
+
.filter((v) => v !== null);
|
|
413
|
+
// Only the last step is rewritten, and `lastStep` is a suffix of `out`.
|
|
414
|
+
out =
|
|
415
|
+
out.slice(0, out.length - lastStep.length) +
|
|
416
|
+
lastStep.slice(0, reporters.start) +
|
|
417
|
+
`--reporter${reporters.separator}${kept.join(',')}` +
|
|
418
|
+
lastStep.slice(reporters.end);
|
|
419
|
+
changes.push(
|
|
420
|
+
// `list` was already there and a second printer sat beside it: that one is dropped. When
|
|
421
|
+
// no `list` was named at all, the printer is SWITCHED to it (flash-m62's `line`).
|
|
422
|
+
printing.includes('list')
|
|
423
|
+
? `Dropped \`${printing
|
|
424
|
+
.filter((v) => v !== 'list')
|
|
425
|
+
.join(',')}\` from your \`--reporter=${reporters.values.join(',')}\`: ` +
|
|
426
|
+
'two reporters that both print each failure write every test result into the log ' +
|
|
427
|
+
'twice — the whole log doubled for nothing (X1 flash-m83) — so this run keeps ' +
|
|
428
|
+
'`list` alone.'
|
|
429
|
+
: `Switched the test reporters to \`--reporter${reporters.separator}${kept.join(',')}\`: ` +
|
|
430
|
+
'`list` writes each test result as it finishes and is the shape the platform reads ' +
|
|
431
|
+
'back from a background run, and it is the only printing reporter — two of them ' +
|
|
432
|
+
'write every result into the log twice (X1 flash-m83).');
|
|
433
|
+
}
|
|
382
434
|
}
|
|
383
|
-
else if (finalPlaywrightRun
|
|
384
|
-
out = `${out.trimEnd()} --reporter
|
|
385
|
-
changes.push(
|
|
386
|
-
'(the config reporter may print nothing until the end).'
|
|
387
|
-
(iterating
|
|
388
|
-
? ' `line` writes each failure’s error as it happens, which is what a mid-run look reads.'
|
|
389
|
-
: ''));
|
|
435
|
+
else if (finalPlaywrightRun) {
|
|
436
|
+
out = `${out.trimEnd()} --reporter=list`;
|
|
437
|
+
changes.push('Added `--reporter=list` so each test result is written to the log as it finishes ' +
|
|
438
|
+
'(the config reporter may print nothing until the end).');
|
|
390
439
|
}
|
|
391
440
|
if (finalPlaywrightRun && iterating && !/--retries\b/.test(lastStep)) {
|
|
392
441
|
out = `${out.trimEnd()} --retries=0`;
|
|
@@ -400,14 +449,18 @@ export function streamingBackgroundCommand(command) {
|
|
|
400
449
|
* of from a sandbox; `suiteRunKind` (molecule-dev `api/src/ai/suite-proof.ts`)
|
|
401
450
|
* is the platform's fuller list of URL-testing runners, and the shapes here are
|
|
402
451
|
* its shapes.
|
|
452
|
+
*
|
|
453
|
+
* Each is matched at the START of a pipeline segment's command-position text
|
|
454
|
+
* (`commandSegments`), so a runner named as an argument is not an invocation:
|
|
455
|
+
* `grep -n playwright e2e/README.md` reads a file and starts nothing.
|
|
403
456
|
*/
|
|
404
457
|
const SUITE_RUN_COMMANDS = [
|
|
405
458
|
// `playwright test`, not a bare `playwright` (`--version` is not a run).
|
|
406
|
-
|
|
459
|
+
/^playwright\s+test\b/,
|
|
407
460
|
// Vitest, however it is invoked (`vitest`, `npx vitest run`).
|
|
408
|
-
|
|
461
|
+
/^vitest\b/,
|
|
409
462
|
// `hurl --test …` or `hurl … x.hurl`.
|
|
410
|
-
|
|
463
|
+
/^hurl\s+[^|\n]*(?:--test|\.hurl)/,
|
|
411
464
|
];
|
|
412
465
|
/**
|
|
413
466
|
* Whether a command runs a whole test suite — a Playwright, Vitest or Hurl run
|
|
@@ -421,21 +474,29 @@ const SUITE_RUN_COMMANDS = [
|
|
|
421
474
|
* @returns True when the command starts one of those runners.
|
|
422
475
|
*/
|
|
423
476
|
export function isSuiteRunCommand(command) {
|
|
424
|
-
|
|
477
|
+
if (commandRunsNoTests(command))
|
|
478
|
+
return false;
|
|
479
|
+
return commandSegments(command).some((segment) => SUITE_RUN_COMMANDS.some((re) => re.test(segment.text)));
|
|
425
480
|
}
|
|
426
481
|
/**
|
|
427
|
-
* Whether a command starts a Playwright run — `playwright test` in
|
|
482
|
+
* Whether a command starts a Playwright run — a `playwright test` in command position, in a form
|
|
483
|
+
* that runs something.
|
|
428
484
|
*
|
|
429
485
|
* `npx playwright test --list` matched every `\bplaywright\s+test\b` test in the platform and was
|
|
430
486
|
* then treated as a run: held (flash-m79's config check), backgrounded for a run that outlasts
|
|
431
487
|
* the tool's ceiling, and rewritten with `--reporter`. A list takes two seconds, prints its lines
|
|
432
488
|
* and exits, so it is none of those; {@link commandRunsNoTests} is what decides.
|
|
433
489
|
*
|
|
490
|
+
* A `playwright test` named as an argument is not a run either (`head -30 run-tests.sh`,
|
|
491
|
+
* `grep -n "playwright test" e2e/README.md`): the command that runs is the reader.
|
|
492
|
+
*
|
|
434
493
|
* @param command - The shell command.
|
|
435
494
|
* @returns True when the command starts a Playwright run.
|
|
436
495
|
*/
|
|
437
496
|
export function isPlaywrightRunCommand(command) {
|
|
438
|
-
|
|
497
|
+
if (commandRunsNoTests(command))
|
|
498
|
+
return false;
|
|
499
|
+
return commandSegments(command).some((segment) => /^playwright\s+test\b/.test(segment.text));
|
|
439
500
|
}
|
|
440
501
|
/**
|
|
441
502
|
* Start a background suite run at lower CPU and IO priority, so the sandbox it
|
|
@@ -740,31 +801,96 @@ export function sliceLines(content, window) {
|
|
|
740
801
|
truncated: offset > 1 || offset - 1 + picked.length < totalLines,
|
|
741
802
|
};
|
|
742
803
|
}
|
|
743
|
-
/**
|
|
804
|
+
/**
|
|
805
|
+
* A file's text as the request counts it: the length of its JSON string.
|
|
806
|
+
*
|
|
807
|
+
* JSON escaping (`\n` → `\\n`, `"` → `\"`) costs characters, and the request
|
|
808
|
+
* side counts the SERIALIZED tool result. A window of 59,000 characters of
|
|
809
|
+
* source — 1,200 lines — arrives as ~60,200 characters of JSON, over the
|
|
810
|
+
* ceiling it was sized for, so a window is measured with this rather than by
|
|
811
|
+
* its raw text length.
|
|
812
|
+
*
|
|
813
|
+
* @param text - The text.
|
|
814
|
+
* @returns Its serialized length, without the surrounding quotes.
|
|
815
|
+
*/
|
|
816
|
+
export function serializedChars(text) {
|
|
817
|
+
return JSON.stringify(text).length - 2;
|
|
818
|
+
}
|
|
819
|
+
/**
|
|
820
|
+
* How many leading lines of a file fit inside a budget of SERIALIZED chars,
|
|
821
|
+
* the way `read_file` sizes a window (see {@link serializedChars}).
|
|
822
|
+
*
|
|
823
|
+
* @param content - The file's full text.
|
|
824
|
+
* @param chars - The budget in serialized chars.
|
|
825
|
+
* @returns The line count, at least one.
|
|
826
|
+
*/
|
|
827
|
+
export function linesWithinSerialized(content, chars) {
|
|
828
|
+
let used = 0;
|
|
829
|
+
let lines = 0;
|
|
830
|
+
for (const line of content.split('\n')) {
|
|
831
|
+
// +2: the newline between two lines escapes to `\n`. The last line of a
|
|
832
|
+
// window has no trailing newline, so this counts 2 chars it does not use —
|
|
833
|
+
// a window that errs small is the safe direction here.
|
|
834
|
+
used += serializedChars(line) + 2;
|
|
835
|
+
if (used > chars && lines > 0)
|
|
836
|
+
break;
|
|
837
|
+
lines++;
|
|
838
|
+
}
|
|
839
|
+
return Math.max(1, lines);
|
|
840
|
+
}
|
|
744
841
|
/**
|
|
745
842
|
* Most files one batched `read_file` call returns. The cap exists so a survey
|
|
746
|
-
* of a large tree cannot swallow the turn's context in one result;
|
|
747
|
-
*
|
|
843
|
+
* of a large tree cannot swallow the turn's context in one result; every path
|
|
844
|
+
* past it is named in the result's `note` and listed with its own `error`, so
|
|
845
|
+
* the model continues from there.
|
|
748
846
|
*/
|
|
749
847
|
export const MAX_BATCH_READ_FILES = 25;
|
|
750
848
|
/**
|
|
751
|
-
*
|
|
752
|
-
*
|
|
849
|
+
* The most one `read_file` RESULT may carry, in chars of its JSON — every file
|
|
850
|
+
* the call returns, the envelope and the note included. That is how the request
|
|
851
|
+
* side counts a tool result, so it is how this tool counts one too
|
|
852
|
+
* ({@link serializedChars}).
|
|
853
|
+
*
|
|
854
|
+
* One tool result is ONE block in the request, and the request side caps a
|
|
855
|
+
* block: molecule-dev cuts any block over `MAX_TAIL_BLOCK_CHARS` (60,000) in
|
|
856
|
+
* place, keeping ~30 KB of each end. A batched read that crossed it lost its
|
|
857
|
+
* MIDDLE — the cut lands between two `"content"` values, so whole files fell
|
|
858
|
+
* out and the note in their place said only "N characters cut here" — while
|
|
859
|
+
* this tool's own note had promised each of those files that it "fits in one
|
|
860
|
+
* call, so it is not paged": X0 flash-m81 (2026-10-16) read ten e2e specs in
|
|
861
|
+
* one call (121,467 chars), lost the middle six, and bought them again in six
|
|
862
|
+
* further calls (52,908 chars).
|
|
863
|
+
*
|
|
864
|
+
* So the CALL gets the ceiling the request gives it. Whole files are returned
|
|
865
|
+
* up to it; the paths that did not fit are named in `note` and listed in
|
|
866
|
+
* `files` with their own `error`, the shape every other entry has. No file is
|
|
867
|
+
* ever cut, and no read result reaches the request-side cut. A test in
|
|
868
|
+
* molecule-dev's `context-fit` asserts the two ceilings still agree.
|
|
869
|
+
*/
|
|
870
|
+
export const MAX_READ_CALL_CHARS = 60_000;
|
|
871
|
+
/**
|
|
872
|
+
* Room inside one read result for what is not file content: the `path`, the
|
|
873
|
+
* offset/line fields, and the note.
|
|
874
|
+
*
|
|
875
|
+
* A batch sizes each file against what the call has left
|
|
876
|
+
* ({@link MAX_READ_CALL_CHARS} less this), so the entry that does not fit is
|
|
877
|
+
* caught here rather than by the request-side cut.
|
|
753
878
|
*/
|
|
754
|
-
export const
|
|
879
|
+
export const READ_RESULT_ENVELOPE_CHARS = 1_000;
|
|
755
880
|
/**
|
|
756
881
|
* The most one read_file call returns for ONE file when no window was asked
|
|
757
|
-
* for, in chars
|
|
758
|
-
* saying how to read the rest.
|
|
882
|
+
* for, in serialized chars: larger files come back as their first window, with
|
|
883
|
+
* a note saying how to read the rest.
|
|
759
884
|
*
|
|
760
885
|
* MAX_READ_SIZE (5 MB) is a bound on what the tool will open, not on what it
|
|
761
886
|
* should hand a model: a single batched read that returned a 1.2 MB README
|
|
762
887
|
* put the next three provider calls at 366k tokens (~330k of them fresh)
|
|
763
|
-
* against a 120k-token context cap (X0 run x5, call 26).
|
|
764
|
-
*
|
|
765
|
-
*
|
|
888
|
+
* against a 120k-token context cap (X0 run x5, call 26). This is the call's
|
|
889
|
+
* own ceiling ({@link MAX_READ_CALL_CHARS}) less the room its envelope takes,
|
|
890
|
+
* so one read result — one file or twenty-five — fits inside what the request
|
|
891
|
+
* side carries without being cut.
|
|
766
892
|
*/
|
|
767
|
-
export const MAX_READ_RETURN_CHARS =
|
|
893
|
+
export const MAX_READ_RETURN_CHARS = MAX_READ_CALL_CHARS - READ_RESULT_ENVELOPE_CHARS;
|
|
768
894
|
/**
|
|
769
895
|
* A file this small is returned whole on a second window of it (paging it costs a call each).
|
|
770
896
|
* The same bound as {@link MAX_READ_RETURN_CHARS}: the windowed-read note promises that a
|
|
@@ -1065,7 +1191,10 @@ export function runningCauseCommand(log, own) {
|
|
|
1065
1191
|
const read = `cat ${log}${own ? ` ${shellQuote(own)}` : ''} 2>/dev/null`;
|
|
1066
1192
|
return `${read} | tail -c ${REPEATING_CAUSE_TAIL_BYTES}`;
|
|
1067
1193
|
}
|
|
1068
|
-
/**
|
|
1194
|
+
/**
|
|
1195
|
+
* DISTINCT tests that must share one signature before a running suite's repeating cause is named.
|
|
1196
|
+
* A test the log prints twice, or a test that failed twice, is still one test.
|
|
1197
|
+
*/
|
|
1069
1198
|
export const REPEATING_FAILURE_MIN = 3;
|
|
1070
1199
|
/** ANSI SGR sequences — the `line` reporter writes cursor moves even into a file. */
|
|
1071
1200
|
// eslint-disable-next-line no-control-regex -- an escape sequence IS a control character
|
|
@@ -1162,28 +1291,63 @@ function largestGroup(groups) {
|
|
|
1162
1291
|
}
|
|
1163
1292
|
return best;
|
|
1164
1293
|
}
|
|
1294
|
+
/**
|
|
1295
|
+
* One test's identity in a log: where it sits and what it is called.
|
|
1296
|
+
*
|
|
1297
|
+
* A test printed twice by two reporters, or failed twice by a retry, is the same test — the
|
|
1298
|
+
* identity is what it is called and where it lives, never which line of the log it came from.
|
|
1299
|
+
*
|
|
1300
|
+
* @param t - A failed test.
|
|
1301
|
+
* @returns The key two prints of one test share.
|
|
1302
|
+
*/
|
|
1303
|
+
function testKey(t) {
|
|
1304
|
+
return `${t.where}|${t.title}`;
|
|
1305
|
+
}
|
|
1306
|
+
/**
|
|
1307
|
+
* The same tests with each one kept once, in first-seen order.
|
|
1308
|
+
*
|
|
1309
|
+
* @param tests - Failed tests, as the log printed them (possibly more than once each).
|
|
1310
|
+
* @returns The distinct tests.
|
|
1311
|
+
*/
|
|
1312
|
+
function distinctTests(tests) {
|
|
1313
|
+
const seen = new Set();
|
|
1314
|
+
return tests.filter((t) => {
|
|
1315
|
+
const key = testKey(t);
|
|
1316
|
+
if (seen.has(key))
|
|
1317
|
+
return false;
|
|
1318
|
+
seen.add(key);
|
|
1319
|
+
return true;
|
|
1320
|
+
});
|
|
1321
|
+
}
|
|
1165
1322
|
/**
|
|
1166
1323
|
* The cause repeating among a running suite's failures, or null.
|
|
1167
1324
|
*
|
|
1168
1325
|
* Playwright's `list` reporter prints one line per test as it finishes — its mark, its title
|
|
1169
1326
|
* path and how long it ran — and defers each failure's error to the summary at the end, so
|
|
1170
|
-
* mid-run the only signature those lines carry is the duration.
|
|
1171
|
-
* failure bodies
|
|
1172
|
-
* which is stronger (it needs no assumption about how long a
|
|
1173
|
-
* message is looked for first.
|
|
1327
|
+
* mid-run the only signature those lines carry is the duration. Where the log carries the
|
|
1328
|
+
* failure bodies (a `line` reporter's output, or a finished run's summary) the repeated error
|
|
1329
|
+
* message itself is the signature, which is stronger (it needs no assumption about how long a
|
|
1330
|
+
* failure should take), so the message is looked for first.
|
|
1174
1331
|
*
|
|
1175
1332
|
* Either way the point is the same: a suite whose failures repeat one cause spends its whole
|
|
1176
1333
|
* runtime proving that one cause, and the executor watching it can be told so at the third
|
|
1177
1334
|
* failure instead of at the last. X1 flash-m76 walked 15 tests in 4.0 minutes where 15.6 s
|
|
1178
1335
|
* each was the test timeout and one missing `navigate('/')` was the cause of all of them.
|
|
1179
1336
|
*
|
|
1337
|
+
* The quorum is of DISTINCT tests — {@link REPEATING_FAILURE_MIN} of them — because a log that
|
|
1338
|
+
* prints each failure twice (a second reporter, a retry) had X1 flash-m83 told that four of its
|
|
1339
|
+
* tests had failed the same way when two had, so the executor stopped a 139-test walk at test 21
|
|
1340
|
+
* with one cause in hand (2026-10-16).
|
|
1341
|
+
*
|
|
1180
1342
|
* @param log - The log so far.
|
|
1181
1343
|
* @returns The repeating cause, or null when no signature has {@link REPEATING_FAILURE_MIN}
|
|
1182
|
-
*
|
|
1344
|
+
* distinct tests behind it.
|
|
1183
1345
|
*/
|
|
1184
1346
|
export function repeatingFailure(log) {
|
|
1185
1347
|
const text = log.replace(ANSI_SGR, '');
|
|
1186
|
-
// The failure BODIES first: a message repeating verbatim names the cause itself.
|
|
1348
|
+
// The failure BODIES first: a message repeating verbatim names the cause itself. Grouped by
|
|
1349
|
+
// DISTINCT test, so the largest group is the cause the most tests share — not the one whose
|
|
1350
|
+
// failures happen to have been printed the most times.
|
|
1187
1351
|
const byMessage = new Map();
|
|
1188
1352
|
for (const block of failureBlocks(text)) {
|
|
1189
1353
|
const message = failureMessage(block);
|
|
@@ -1197,7 +1361,10 @@ export function repeatingFailure(log) {
|
|
|
1197
1361
|
else
|
|
1198
1362
|
byMessage.set(message, [{ where, title, seconds: null }]);
|
|
1199
1363
|
}
|
|
1200
|
-
const
|
|
1364
|
+
const distinctByMessage = new Map();
|
|
1365
|
+
for (const [message, members] of byMessage)
|
|
1366
|
+
distinctByMessage.set(message, distinctTests(members));
|
|
1367
|
+
const messageGroup = largestGroup(distinctByMessage);
|
|
1201
1368
|
if (messageGroup && messageGroup.members.length >= REPEATING_FAILURE_MIN)
|
|
1202
1369
|
return {
|
|
1203
1370
|
count: messageGroup.members.length,
|
|
@@ -1208,7 +1375,7 @@ export function repeatingFailure(log) {
|
|
|
1208
1375
|
// Otherwise the durations: the largest set of failures that took the same time to within the
|
|
1209
1376
|
// band. A test that runs to its timeout prints slightly different durations on each attempt
|
|
1210
1377
|
// (15.6 s, 15.1 s, 15.0 s), so the band is 0.5 s or 5% of the duration, whichever is larger.
|
|
1211
|
-
const tests = failedTests(text);
|
|
1378
|
+
const tests = distinctTests(failedTests(text));
|
|
1212
1379
|
const withSeconds = tests.filter((t) => t.seconds !== null);
|
|
1213
1380
|
let best = [];
|
|
1214
1381
|
for (const t of withSeconds) {
|
|
@@ -1224,27 +1391,222 @@ export function repeatingFailure(log) {
|
|
|
1224
1391
|
}
|
|
1225
1392
|
/** How many of a repeating cause's failures the note lists before it says "and N more". */
|
|
1226
1393
|
export const REPEATING_FAILURE_LISTED = 5;
|
|
1394
|
+
/** How many causes a finished run's summary lists before it says "and N more". */
|
|
1395
|
+
export const FAILURE_CAUSES_LISTED = 5;
|
|
1396
|
+
/** How many of one cause's tests that summary names before it says "and N more". */
|
|
1397
|
+
export const FAILURE_CAUSE_TESTS_LISTED = 6;
|
|
1227
1398
|
/**
|
|
1228
1399
|
* The shortest unambiguous tail of a logged path (`…/specs/e2e/articles.spec.ts:32` →
|
|
1229
1400
|
* `e2e/articles.spec.ts:32`).
|
|
1230
1401
|
*
|
|
1231
1402
|
* @param where - The location the log wrote.
|
|
1232
|
-
* @returns The last two path segments.
|
|
1403
|
+
* @returns The last two path segments, with the line number.
|
|
1233
1404
|
*/
|
|
1234
1405
|
function shortWhere(where) {
|
|
1235
1406
|
const parts = where.split('/');
|
|
1236
1407
|
return parts.slice(-2).join('/');
|
|
1237
1408
|
}
|
|
1238
1409
|
/**
|
|
1239
|
-
* The
|
|
1240
|
-
*
|
|
1410
|
+
* The shortest unambiguous tail of a logged path's FILE (`…/specs/e2e/articles.spec.ts` →
|
|
1411
|
+
* `e2e/articles.spec.ts`).
|
|
1412
|
+
*
|
|
1413
|
+
* @param file - A path as {@link specFile} returns it.
|
|
1414
|
+
* @returns The last two path segments.
|
|
1415
|
+
*/
|
|
1416
|
+
function shortFile(file) {
|
|
1417
|
+
return file.split('/').slice(-2).join('/');
|
|
1418
|
+
}
|
|
1419
|
+
/**
|
|
1420
|
+
* The spec file a logged location names (`…/e2e/articles.spec.ts:32` → `…/e2e/articles.spec.ts`).
|
|
1421
|
+
*
|
|
1422
|
+
* @param where - A location from {@link FailedTest.where}.
|
|
1423
|
+
* @returns The path without its line number.
|
|
1424
|
+
*/
|
|
1425
|
+
function specFile(where) {
|
|
1426
|
+
return where.replace(/:\d+$/, '');
|
|
1427
|
+
}
|
|
1428
|
+
/**
|
|
1429
|
+
* The mark of every test result a log names, in the order it printed them (`✘` failed, `✓`
|
|
1430
|
+
* passed).
|
|
1431
|
+
*
|
|
1432
|
+
* A `line`-only log writes none: `[n/N] <title>` carries no mark. An empty list therefore means
|
|
1433
|
+
* the log cannot say whether anything passed, not that nothing did.
|
|
1434
|
+
*
|
|
1435
|
+
* @param text - The log (ANSI already stripped).
|
|
1436
|
+
* @returns The marks.
|
|
1437
|
+
*/
|
|
1438
|
+
function resultMarks(text) {
|
|
1439
|
+
const out = [];
|
|
1440
|
+
for (const line of text.split('\n')) {
|
|
1441
|
+
const m = LIST_RESULT_LINE.exec(line);
|
|
1442
|
+
if (m)
|
|
1443
|
+
out.push(m[1]);
|
|
1444
|
+
}
|
|
1445
|
+
return out;
|
|
1446
|
+
}
|
|
1447
|
+
/**
|
|
1448
|
+
* Every test the log's numbered failure blocks name, in the order it names them.
|
|
1449
|
+
*
|
|
1450
|
+
* @param text - The log (ANSI already stripped).
|
|
1451
|
+
* @returns The failed tests, with the location and title their header carries.
|
|
1452
|
+
*/
|
|
1453
|
+
function failureBlockTests(text) {
|
|
1454
|
+
return failureBlocks(text).map((block) => {
|
|
1455
|
+
const m = FAILURE_BLOCK_HEADER.exec(block.split('\n')[0] ?? '');
|
|
1456
|
+
const { where, title } = splitTitlePath(m?.[1] ?? '');
|
|
1457
|
+
return { where, title, seconds: null };
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
/**
|
|
1461
|
+
* Distinct spec FILES a repeating cause must span — with every failure sharing it and nothing
|
|
1462
|
+
* passing since — before a run is called broken for the whole suite.
|
|
1463
|
+
*
|
|
1464
|
+
* One file's failures can be that file's own setup, so a cause confined to one file is never
|
|
1465
|
+
* this. Across {@link UNIVERSAL_CAUSE_FILES} files, on the other hand, whatever it is is a
|
|
1466
|
+
* property of the run rather than of a page.
|
|
1467
|
+
*/
|
|
1468
|
+
export const UNIVERSAL_CAUSE_FILES = 3;
|
|
1469
|
+
/**
|
|
1470
|
+
* Whether a repeating cause is failing the whole run — the one case where stopping it is the
|
|
1471
|
+
* right call.
|
|
1472
|
+
*
|
|
1473
|
+
* The test is: the cause accounts for every failure the log names, it has hit
|
|
1474
|
+
* {@link UNIVERSAL_CAUSE_FILES} spec files or more, and nothing has passed since its first
|
|
1475
|
+
* failure. An app that is not up, a 5xx on every request, or a selector that is missing on every
|
|
1476
|
+
* page all look like that, and every remaining test will fail the same way — so the run's
|
|
1477
|
+
* remaining minutes measure nothing. Anything less is NOT this: a cause that stops at one file,
|
|
1478
|
+
* or that leaves other tests passing, or that shares the run with failures of another shape,
|
|
1479
|
+
* leaves the run with something to say, and the run is the only thing that will say it. X1
|
|
1480
|
+
* flash-m83 (2026-10-16) stopped eight local runs at their first repeating cause over 14.5
|
|
1481
|
+
* minutes and never finished one; three of its deploys each fixed a single cause that a finished
|
|
1482
|
+
* run would have listed alongside the others.
|
|
1483
|
+
*
|
|
1484
|
+
* @param log - The log so far.
|
|
1485
|
+
* @param found - The repeating cause {@link repeatingFailure} found in it.
|
|
1486
|
+
* @returns The cause's count and files, or null when the run is not failing whole.
|
|
1487
|
+
*/
|
|
1488
|
+
export function universalCause(log, found) {
|
|
1489
|
+
const text = log.replace(ANSI_SGR, '');
|
|
1490
|
+
const files = [...new Set(found.tests.map((t) => specFile(t.where)).filter(Boolean))];
|
|
1491
|
+
if (files.length < UNIVERSAL_CAUSE_FILES)
|
|
1492
|
+
return null;
|
|
1493
|
+
// Every failure the log names has to be this cause: another cause anywhere in the run means
|
|
1494
|
+
// the run still has something to say, and stopping it throws that away.
|
|
1495
|
+
const named = Math.max(distinctTests(failedTests(text)).length, distinctTests(failureBlockTests(text)).length);
|
|
1496
|
+
if (found.count < named)
|
|
1497
|
+
return null;
|
|
1498
|
+
// And nothing may have passed since its first failure. A pass after it means the app answered
|
|
1499
|
+
// a request, so a test that still fails the same way fails for a reason this run has not seen.
|
|
1500
|
+
const marks = resultMarks(text);
|
|
1501
|
+
const firstFailure = marks.indexOf('✘');
|
|
1502
|
+
if (firstFailure === -1 || marks.slice(firstFailure).includes('✓'))
|
|
1503
|
+
return null;
|
|
1504
|
+
return { count: found.count, files };
|
|
1505
|
+
}
|
|
1506
|
+
/**
|
|
1507
|
+
* Every failure a run names, grouped by the cause behind it — one group per cause, with the
|
|
1508
|
+
* tests it hit.
|
|
1509
|
+
*
|
|
1510
|
+
* The point is that ONE run yields EVERY cause. A Playwright report prints its failures one
|
|
1511
|
+
* after another with no statement of which of them share a cause, so a run of 19 failures is 19
|
|
1512
|
+
* bodies to read; X1 flash-m83 (2026-10-16) spent three deploys (6.7 minutes) each fixing a
|
|
1513
|
+
* single cause, two of them found by runs stopped before they walked the rest.
|
|
1514
|
+
*
|
|
1515
|
+
* The signature is the failures' own error line where the log carries their bodies (a finished
|
|
1516
|
+
* run, or a `line` reporter), and how long they took where it does not (a still-running `list`
|
|
1517
|
+
* log). A test the log printed twice, or failed twice, is one test.
|
|
1518
|
+
*
|
|
1519
|
+
* @param log - The log of a finished or running run.
|
|
1520
|
+
* @returns The causes, the one with the most tests first.
|
|
1521
|
+
*/
|
|
1522
|
+
export function failuresByCause(log) {
|
|
1523
|
+
const text = log.replace(ANSI_SGR, '');
|
|
1524
|
+
const byMessage = new Map();
|
|
1525
|
+
for (const block of failureBlocks(text)) {
|
|
1526
|
+
const message = failureMessage(block);
|
|
1527
|
+
if (message === null)
|
|
1528
|
+
continue;
|
|
1529
|
+
const m = FAILURE_BLOCK_HEADER.exec(block.split('\n')[0] ?? '');
|
|
1530
|
+
const { where, title } = splitTitlePath(m?.[1] ?? '');
|
|
1531
|
+
const group = byMessage.get(message);
|
|
1532
|
+
if (group)
|
|
1533
|
+
group.push({ where, title, seconds: null });
|
|
1534
|
+
else
|
|
1535
|
+
byMessage.set(message, [{ where, title, seconds: null }]);
|
|
1536
|
+
}
|
|
1537
|
+
const causes = [];
|
|
1538
|
+
for (const [message, members] of byMessage) {
|
|
1539
|
+
const tests = distinctTests(members);
|
|
1540
|
+
if (tests.length > 0)
|
|
1541
|
+
causes.push({ message, seconds: null, tests });
|
|
1542
|
+
}
|
|
1543
|
+
// The failures with no body of their own — every failure of a still-running `list` log — are
|
|
1544
|
+
// grouped by how long they took: a band is opened by its first test and takes the ones that
|
|
1545
|
+
// then ran the same length, the same band {@link repeatingFailure} uses.
|
|
1546
|
+
const claimed = new Set(causes.flatMap((c) => c.tests.map(testKey)));
|
|
1547
|
+
const unclaimed = distinctTests(failedTests(text)).filter((t) => t.seconds !== null && !claimed.has(testKey(t)));
|
|
1548
|
+
const byDuration = [];
|
|
1549
|
+
for (const t of unclaimed) {
|
|
1550
|
+
const band = Math.max(0.5, t.seconds * 0.05);
|
|
1551
|
+
const open = byDuration.find((g) => Math.abs(g.seconds - t.seconds) <= Math.max(band, g.seconds * 0.05));
|
|
1552
|
+
if (open)
|
|
1553
|
+
open.tests.push(t);
|
|
1554
|
+
else
|
|
1555
|
+
byDuration.push({ seconds: t.seconds, tests: [t] });
|
|
1556
|
+
}
|
|
1557
|
+
for (const g of byDuration)
|
|
1558
|
+
causes.push({ message: null, seconds: g.seconds, tests: g.tests });
|
|
1559
|
+
return causes.sort((a, b) => b.tests.length - a.tests.length);
|
|
1560
|
+
}
|
|
1561
|
+
/**
|
|
1562
|
+
* The one-line-per-cause summary of a finished run's failures, or null when the run named fewer
|
|
1563
|
+
* than two of them.
|
|
1564
|
+
*
|
|
1565
|
+
* This is what makes a run worth finishing: every cause the run found, in the result the
|
|
1566
|
+
* executor reads, so one round of fixes covers all of them rather than the one cause a run
|
|
1567
|
+
* stopped early happened to reach.
|
|
1568
|
+
*
|
|
1569
|
+
* @param log - The finished run's output.
|
|
1570
|
+
* @returns The note, or null.
|
|
1571
|
+
*/
|
|
1572
|
+
export function failureCausesNote(log) {
|
|
1573
|
+
const causes = failuresByCause(log);
|
|
1574
|
+
const total = causes.reduce((n, c) => n + c.tests.length, 0);
|
|
1575
|
+
if (total < 2)
|
|
1576
|
+
return null;
|
|
1577
|
+
const listed = causes.slice(0, FAILURE_CAUSES_LISTED);
|
|
1578
|
+
const lines = listed.map((c, i) => {
|
|
1579
|
+
const tests = c.tests
|
|
1580
|
+
.slice(0, FAILURE_CAUSE_TESTS_LISTED)
|
|
1581
|
+
.map((t) => shortWhere(t.where) || t.title);
|
|
1582
|
+
const more = c.tests.length > tests.length ? `, … and ${c.tests.length - tests.length} more` : '';
|
|
1583
|
+
const signature = c.message !== null ? `\`${c.message}\`` : `each after ~${c.seconds}s`;
|
|
1584
|
+
const files = new Set(c.tests.map((t) => specFile(t.where)).filter(Boolean)).size;
|
|
1585
|
+
return (` ${i + 1}. ${signature} — ${c.tests.length} test${c.tests.length === 1 ? '' : 's'} ` +
|
|
1586
|
+
`in ${files} file${files === 1 ? '' : 's'}: ${tests.join(', ')}${more}`);
|
|
1587
|
+
});
|
|
1588
|
+
const moreCauses = causes.length > listed.length ? `\n … and ${causes.length - listed.length} more causes` : '';
|
|
1589
|
+
return (`${total} failure${total === 1 ? '' : 's'} in this run, grouped by cause — fix every cause ` +
|
|
1590
|
+
`below before the next deploy:\n${lines.join('\n')}${moreCauses}`);
|
|
1591
|
+
}
|
|
1592
|
+
/**
|
|
1593
|
+
* The sentence that names a cause repeating among a running suite's failures, mid-run.
|
|
1241
1594
|
*
|
|
1242
1595
|
* Kept here, beside {@link repeatingFailure}, so the two stay in step: every word below is about
|
|
1243
|
-
* what that function could actually see in the log
|
|
1596
|
+
* what that function could actually see in the log — including the count, which is of DISTINCT
|
|
1597
|
+
* tests, so the note never names one or two tests as a cause.
|
|
1598
|
+
*
|
|
1599
|
+
* The note names the cause early so the executor can START FIXING while the run keeps walking,
|
|
1600
|
+
* and it asks for the run to be stopped only when {@link universalCause} says the cause is
|
|
1601
|
+
* failing the whole run. X1 flash-m76 (2026-10-16) spent 4.0 minutes walking 15 tests whose one
|
|
1602
|
+
* cause was certain at the third, 47 s in, and the note answers exactly that; X1 flash-m83 then
|
|
1603
|
+
* stopped eight runs at their first repeating cause in 14.5 minutes and never finished one, so
|
|
1604
|
+
* the note must not say to stop when the run still has other causes to name.
|
|
1244
1605
|
*
|
|
1245
1606
|
* @param found - What {@link repeatingFailure} found.
|
|
1246
|
-
* @param o - The run's facts: its `taskId`, its `log` path,
|
|
1247
|
-
* stops it
|
|
1607
|
+
* @param o - The run's facts: its `taskId`, its `log` path, `stop` — the shell command that
|
|
1608
|
+
* stops it, printed only for a {@link UniversalCause} — and `universal`, what
|
|
1609
|
+
* {@link universalCause} found (absent means the run is not failing whole).
|
|
1248
1610
|
* @returns The note.
|
|
1249
1611
|
*/
|
|
1250
1612
|
export function repeatingFailureNote(found, o) {
|
|
@@ -1263,10 +1625,26 @@ export function repeatingFailureNote(found, o) {
|
|
|
1263
1625
|
: first?.where
|
|
1264
1626
|
? ` Re-run \`${first.where}\` alone to see its error in full.`
|
|
1265
1627
|
: '';
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1628
|
+
const where = `task ${o.taskId}, log ${o.log}`;
|
|
1629
|
+
if (o.universal) {
|
|
1630
|
+
const files = o.universal.files.map(shortFile);
|
|
1631
|
+
const shown = files.slice(0, 4).join(', ');
|
|
1632
|
+
const moreFiles = files.length > 4 ? `, … and ${files.length - 4} more` : '';
|
|
1633
|
+
return (`Every test this run has walked so far has failed the same way — all ${found.count} of ` +
|
|
1634
|
+
`them, ${signature} — across ${files.length} files (${shown}${moreFiles}), with nothing ` +
|
|
1635
|
+
`passing since the first:\n${rows}${more}\n` +
|
|
1636
|
+
`Every remaining test will fail the same way too, so walking the rest of this run only ` +
|
|
1637
|
+
`repeats this one cause. Stop it and fix that cause first: \`exec_command: ${o.stop}\`, ` +
|
|
1638
|
+
`then run the whole suite again. It is still going (${where}).${reproduce}`);
|
|
1639
|
+
}
|
|
1640
|
+
return (`${found.count} different tests in this run have already failed the same way — ${signature} — ` +
|
|
1641
|
+
`so one cause is repeating:\n${rows}${more}\n` +
|
|
1642
|
+
`Start fixing it now, while the run keeps walking.${reproduce} ` +
|
|
1643
|
+
`Let the rest of the run finish — it is the only thing that will name the causes that are NOT ` +
|
|
1644
|
+
`this one, and a run stopped at its first repeating cause costs a second whole run to find ` +
|
|
1645
|
+
`them. Stop it only if every test it walks fails this same way (the app unreachable, a 5xx on ` +
|
|
1646
|
+
`every request, the same selector timing out on every test of every file so far). ` +
|
|
1647
|
+
`It is still going (${where}).`);
|
|
1270
1648
|
}
|
|
1271
1649
|
/**
|
|
1272
1650
|
* Attempt a whitespace-tolerant replacement when an exact `old_string` match
|