@molecule/api-ai-tools 1.7.0 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +324 -25
- package/dist/command-position.d.ts +42 -0
- package/dist/command-position.d.ts.map +1 -1
- package/dist/command-position.js +271 -41
- package/dist/cut-run.d.ts +98 -8
- package/dist/cut-run.d.ts.map +1 -1
- package/dist/cut-run.js +182 -29
- package/dist/suite-overlap.d.ts +5 -1
- package/dist/suite-overlap.d.ts.map +1 -1
- package/dist/suite-overlap.js +16 -2
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +78 -13
- package/dist/trace-artifacts.d.ts.map +1 -1
- package/dist/trace-artifacts.js +45 -1
- package/dist/types.d.ts +13 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utilities.d.ts +121 -8
- package/dist/utilities.d.ts.map +1 -1
- package/dist/utilities.js +482 -41
- package/package.json +1 -1
package/dist/utilities.js
CHANGED
|
@@ -349,11 +349,15 @@ export const PLAYWRIGHT_FAILURES_INLINE_ENV = 'PLAYWRIGHT_LIST_PRINT_FAILURES_IN
|
|
|
349
349
|
*
|
|
350
350
|
* - a trailing `| tail -N`, which holds everything back until the command ends.
|
|
351
351
|
* It is dropped: `wait_for_task` already returns the end of the log.
|
|
352
|
+
* - any output-only filter piped after a SUITE runner (`| grep … | tail -20`), which hides the
|
|
353
|
+
* failing test's own error and leaves the pipeline's exit status — the filter's — as the run's.
|
|
354
|
+
* Every such stage is dropped ({@link stripTrailingSuiteFilters}); `wait_for_task` already
|
|
355
|
+
* returns the end of the full log, and a run whose tally shows a failure must never report 0
|
|
356
|
+
* (xr-molecule-flash-m101 +25:54).
|
|
352
357
|
* - `playwright test` with no `--reporter`, which runs the config's reporter —
|
|
353
358
|
* often `html`, which prints nothing per test. `--reporter=list` is added
|
|
354
|
-
* when the playwright run is the command's last step
|
|
355
|
-
*
|
|
356
|
-
* shape the platform reads a background suite back in.
|
|
359
|
+
* when the playwright run is the command's last step, piped or not: it writes each result as
|
|
360
|
+
* it finishes, and it is the shape the platform reads a background suite back in.
|
|
357
361
|
*
|
|
358
362
|
* The run ends up with exactly ONE reporter that prints each failure — `list`. A run that names
|
|
359
363
|
* any other printing reporter has it REPLACED, never joined: two of them write every test result
|
|
@@ -386,6 +390,24 @@ export function streamingBackgroundCommand(command) {
|
|
|
386
390
|
return { command, changes: [] };
|
|
387
391
|
const changes = [];
|
|
388
392
|
let out = command;
|
|
393
|
+
// A filter after the suite runner is not output the platform can read: it hides the failing
|
|
394
|
+
// test's own error, and its exit status replaces the run's with the filter's — m101's
|
|
395
|
+
// `… | grep -E "✘|passed|failed|skipped" | tail -20` reported `exitCode: 0` for a run with a
|
|
396
|
+
// failure in it. Every stage the parse proves is output-only is dropped; the end of the FULL
|
|
397
|
+
// log comes back anyway.
|
|
398
|
+
const filtered = stripTrailingSuiteFilters(out);
|
|
399
|
+
if (filtered.dropped) {
|
|
400
|
+
out = filtered.command;
|
|
401
|
+
// A lone trailing `| tail -N` keeps its own sentence: the reason it goes is that it holds all
|
|
402
|
+
// output back until the command ends, not that a filter cannot be read back.
|
|
403
|
+
const tailOnly = TRAILING_TAIL.exec(filtered.dropped);
|
|
404
|
+
changes.push(tailOnly && tailOnly[0].trim() === filtered.dropped
|
|
405
|
+
? `Dropped the trailing \`${filtered.dropped}\`: it holds all output back until the command ends, ` +
|
|
406
|
+
'and wait_for_task returns the end of the log anyway.'
|
|
407
|
+
: `Dropped \`${filtered.dropped}\` from after the test runner: a filtered log cannot be ` +
|
|
408
|
+
'diagnosed — the failing test’s own error is not in it — and wait_for_task returns the ' +
|
|
409
|
+
'end of the full log anyway.');
|
|
410
|
+
}
|
|
389
411
|
const tail = TRAILING_TAIL.exec(out);
|
|
390
412
|
if (tail) {
|
|
391
413
|
out = out.slice(0, tail.index);
|
|
@@ -400,9 +422,25 @@ export function streamingBackgroundCommand(command) {
|
|
|
400
422
|
changes.push(`Dropped the trailing \`${head[0].trim()}\`: Playwright prints its pass/fail count last, and it would be cut.`);
|
|
401
423
|
}
|
|
402
424
|
const lastStep = out.split(/&&|\|\||;/).at(-1) ?? '';
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
425
|
+
// The last step runs Playwright — whether or not anything is piped from it. Keying this on
|
|
426
|
+
// "nothing is piped out of the last step" withheld the reporter, the inline-failure switch and
|
|
427
|
+
// `--retries=0` from exactly the runs whose output was filtered (xr-molecule-flash-m101
|
|
428
|
+
// +25:54); what the runner writes into its log does not depend on what reads the pipe. A
|
|
429
|
+
// REDIRECT still sends the output somewhere this tool cannot read it (`> out.txt`); `2>&1` only
|
|
430
|
+
// joins stderr to it.
|
|
431
|
+
const finalPlaywrightRun = isPlaywrightRunCommand(lastStep) && !/>/.test(lastStep.replace(/\d?>&\d/g, ''));
|
|
432
|
+
/**
|
|
433
|
+
* Add a word to the suite runner's own arguments. A pipe this rewrite could not drop still
|
|
434
|
+
* follows the runner (`| tee run.log`), so appending at the end of the whole command would make
|
|
435
|
+
* the word an operand of the filter — m102's dangling `--trace` flag, in the same shape.
|
|
436
|
+
*/
|
|
437
|
+
const addToRun = (flag) => {
|
|
438
|
+
const pipeline = lastSuitePipeline(out);
|
|
439
|
+
const runner = pipeline ? pipeline.located[pipeline.at] : undefined;
|
|
440
|
+
out = runner
|
|
441
|
+
? `${out.slice(0, runner.end)} ${flag}${out.slice(runner.end)}`
|
|
442
|
+
: `${out.trimEnd()} ${flag}`;
|
|
443
|
+
};
|
|
406
444
|
// A diagnosis or a re-run (capped, last-failed, grepped, or named files) walks each failure
|
|
407
445
|
// once: the suite's own `retries` doubled every failure of X1 flash-m71's first whole run,
|
|
408
446
|
// which had dropped the `--retries=0` it used one command earlier. The uncapped proof run
|
|
@@ -456,7 +494,7 @@ export function streamingBackgroundCommand(command) {
|
|
|
456
494
|
}
|
|
457
495
|
}
|
|
458
496
|
else if (finalPlaywrightRun) {
|
|
459
|
-
|
|
497
|
+
addToRun('--reporter=list');
|
|
460
498
|
changes.push('Added `--reporter=list` so each test result is written to the log as it finishes ' +
|
|
461
499
|
'(the config reporter may print nothing until the end).');
|
|
462
500
|
}
|
|
@@ -475,7 +513,7 @@ export function streamingBackgroundCommand(command) {
|
|
|
475
513
|
'and you need not wait for it to end to start on them.');
|
|
476
514
|
}
|
|
477
515
|
if (finalPlaywrightRun && diagnosis && !/--retries\b/.test(lastStep)) {
|
|
478
|
-
|
|
516
|
+
addToRun('--retries=0');
|
|
479
517
|
changes.push('Added `--retries=0`: a diagnosis or re-run walks each failure once; keep the suite’s retries for the proof run.');
|
|
480
518
|
}
|
|
481
519
|
return { command: out, changes };
|
|
@@ -535,6 +573,185 @@ export function isPlaywrightRunCommand(command) {
|
|
|
535
573
|
return false;
|
|
536
574
|
return commandSegments(command).some((segment) => /^playwright\s+test\b/.test(segment.text));
|
|
537
575
|
}
|
|
576
|
+
/**
|
|
577
|
+
* The programs a pipeline may end with when all they do is read what the runner printed and print
|
|
578
|
+
* it again — the filters a background suite may lose without changing what it ran.
|
|
579
|
+
*
|
|
580
|
+
* Everything else stays: `tee` writes a file, `xargs` runs a command, and an unknown program may
|
|
581
|
+
* do either. Dropping a filter is only sound where the parse can PROVE it is output-only.
|
|
582
|
+
*/
|
|
583
|
+
const OUTPUT_ONLY_FILTERS = new Set([
|
|
584
|
+
'awk',
|
|
585
|
+
'cat',
|
|
586
|
+
'column',
|
|
587
|
+
'cut',
|
|
588
|
+
'egrep',
|
|
589
|
+
'fgrep',
|
|
590
|
+
'fmt',
|
|
591
|
+
'fold',
|
|
592
|
+
'grep',
|
|
593
|
+
'head',
|
|
594
|
+
'jq',
|
|
595
|
+
'nl',
|
|
596
|
+
'rg',
|
|
597
|
+
'sed',
|
|
598
|
+
'sort',
|
|
599
|
+
'tail',
|
|
600
|
+
'tr',
|
|
601
|
+
'uniq',
|
|
602
|
+
'wc',
|
|
603
|
+
]);
|
|
604
|
+
/** A word that sends output to a file (`> f`, `>> f`, `2> f`); `2>&1` only joins stderr, so it is not one. */
|
|
605
|
+
const OUTPUT_REDIRECT = /^(?:\d*|&)>>?(?!&)/;
|
|
606
|
+
/**
|
|
607
|
+
* The shell parse's segments with their offsets in the command, or null when one cannot be placed.
|
|
608
|
+
*
|
|
609
|
+
* {@link commandSegments} answers which program each stage runs, but re-joins the words with single
|
|
610
|
+
* spaces — so a segment written with irregular whitespace, or a program spelled with a directory,
|
|
611
|
+
* has no offset a caller can cut at. Null is the honest answer there: every caller then leaves the
|
|
612
|
+
* command exactly as written rather than cutting into a guess.
|
|
613
|
+
*
|
|
614
|
+
* @param command - The shell command.
|
|
615
|
+
* @returns The located segments, in order, or null.
|
|
616
|
+
*/
|
|
617
|
+
function locatedSegments(command) {
|
|
618
|
+
const out = [];
|
|
619
|
+
let cursor = 0;
|
|
620
|
+
let previousEnd = null;
|
|
621
|
+
for (const segment of commandSegments(command)) {
|
|
622
|
+
const at = command.indexOf(segment.text, cursor);
|
|
623
|
+
if (at < 0)
|
|
624
|
+
return null;
|
|
625
|
+
out.push({
|
|
626
|
+
segment,
|
|
627
|
+
start: at,
|
|
628
|
+
end: at + segment.text.length,
|
|
629
|
+
between: previousEnd === null ? '' : command.slice(previousEnd, at),
|
|
630
|
+
});
|
|
631
|
+
cursor = at + segment.text.length;
|
|
632
|
+
previousEnd = cursor;
|
|
633
|
+
}
|
|
634
|
+
return out;
|
|
635
|
+
}
|
|
636
|
+
/**
|
|
637
|
+
* Whether a pipeline stage only reads the output it is handed and prints it again.
|
|
638
|
+
*
|
|
639
|
+
* `sed`/`awk` are listed for the filters they usually are (`sed -n '1,20p'`, `awk '{print $1}'`),
|
|
640
|
+
* but a program that can WRITE is not output-only: `-i` edits a file in place, and a program that
|
|
641
|
+
* names `>`, `w <file>` or `system(` writes a file or runs a command. Those keep their pipe.
|
|
642
|
+
*
|
|
643
|
+
* @param segment - One pipeline stage of the shell parse.
|
|
644
|
+
* @returns True when dropping the stage can only change what is printed.
|
|
645
|
+
*/
|
|
646
|
+
function isOutputOnlyFilter(segment) {
|
|
647
|
+
if (!OUTPUT_ONLY_FILTERS.has(segment.program))
|
|
648
|
+
return false;
|
|
649
|
+
if (segment.args.some((arg) => OUTPUT_REDIRECT.test(arg)))
|
|
650
|
+
return false;
|
|
651
|
+
if (segment.program === 'sed' &&
|
|
652
|
+
segment.args.some((arg) => /^(?:--in-place\b|-[A-Za-z]*i)/.test(arg)))
|
|
653
|
+
return false;
|
|
654
|
+
if (segment.program === 'awk' || segment.program === 'sed') {
|
|
655
|
+
const program = segment.args.map((arg) => arg.replace(/^['"]|['"]$/g, '')).join(' ');
|
|
656
|
+
if (/\bsystem\s*\(|>|\bw\s+\S/.test(program))
|
|
657
|
+
return false;
|
|
658
|
+
}
|
|
659
|
+
return true;
|
|
660
|
+
}
|
|
661
|
+
/**
|
|
662
|
+
* The command's last step, when that step runs a suite: the suite's own segment and every stage
|
|
663
|
+
* piped after it. Null when no suite runs, or when the suite is not the last step — a `;` or `&&`
|
|
664
|
+
* after it starts another statement, whose output and status are then what the command describes.
|
|
665
|
+
*
|
|
666
|
+
* A stage belongs to the same statement as its predecessor only when the two are joined by a
|
|
667
|
+
* single `|` ({@link LocatedSegment.between}); `||` is a different separator and is not one.
|
|
668
|
+
*
|
|
669
|
+
* @param command - The shell command.
|
|
670
|
+
* @returns The located segments and the last suite stage's index among them, or null.
|
|
671
|
+
*/
|
|
672
|
+
function lastSuitePipeline(command) {
|
|
673
|
+
if (commandRunsNoTests(command))
|
|
674
|
+
return null;
|
|
675
|
+
const located = locatedSegments(command);
|
|
676
|
+
if (located === null)
|
|
677
|
+
return null;
|
|
678
|
+
let at = -1;
|
|
679
|
+
for (let i = 0; i < located.length; i += 1) {
|
|
680
|
+
const segment = located[i]?.segment;
|
|
681
|
+
if (segment && SUITE_RUN_COMMANDS.some((re) => re.test(segment.text)))
|
|
682
|
+
at = i;
|
|
683
|
+
}
|
|
684
|
+
if (at < 0)
|
|
685
|
+
return null;
|
|
686
|
+
for (let i = at + 1; i < located.length; i += 1) {
|
|
687
|
+
if ((located[i]?.between ?? '').trim() !== '|')
|
|
688
|
+
return null;
|
|
689
|
+
}
|
|
690
|
+
return { located, at };
|
|
691
|
+
}
|
|
692
|
+
/**
|
|
693
|
+
* Drop the output-only filters piped after the command's suite runner.
|
|
694
|
+
*
|
|
695
|
+
* A filter after the runner is not "the end of the log" the executor asked for: it hides the
|
|
696
|
+
* failing test's own error (it usually matches none of `✘|passed|failed`), and the pipeline's exit
|
|
697
|
+
* status becomes the FILTER's, so `… 2>&1 | grep -E "✘|passed|failed|skipped" | tail -20` came back
|
|
698
|
+
* `exitCode: 0` for a run with a failure in it (xr-molecule-flash-m101 +25:54). The tool returns
|
|
699
|
+
* the end of the full log anyway, so the filter only costs diagnosability.
|
|
700
|
+
*
|
|
701
|
+
* Only a contiguous suffix of the stages after the runner is cut, and only stages the parse proves
|
|
702
|
+
* are output-only ({@link isOutputOnlyFilter}): a `| tee` or `| xargs` keeps its place, and a
|
|
703
|
+
* caller that reports the run's exit code must then treat it as unknown
|
|
704
|
+
* ({@link suiteRunPipedAfterRunner}).
|
|
705
|
+
*
|
|
706
|
+
* @param command - The shell command.
|
|
707
|
+
* @returns The rewritten command and the stages dropped.
|
|
708
|
+
*/
|
|
709
|
+
export function stripTrailingSuiteFilters(command) {
|
|
710
|
+
const pipeline = lastSuitePipeline(command);
|
|
711
|
+
if (pipeline === null)
|
|
712
|
+
return { command, dropped: '' };
|
|
713
|
+
const { located, at } = pipeline;
|
|
714
|
+
let first = located.length;
|
|
715
|
+
for (let i = located.length - 1; i > at; i -= 1) {
|
|
716
|
+
const stage = located[i];
|
|
717
|
+
if (!stage || !isOutputOnlyFilter(stage.segment))
|
|
718
|
+
break;
|
|
719
|
+
first = i;
|
|
720
|
+
}
|
|
721
|
+
if (first === located.length)
|
|
722
|
+
return { command, dropped: '' };
|
|
723
|
+
const previous = located[first - 1];
|
|
724
|
+
const stage = located[first];
|
|
725
|
+
if (!previous || !stage)
|
|
726
|
+
return { command, dropped: '' };
|
|
727
|
+
const pipe = stage.between.indexOf('|');
|
|
728
|
+
if (pipe < 0)
|
|
729
|
+
return { command, dropped: '' };
|
|
730
|
+
const cut = previous.end + pipe;
|
|
731
|
+
return { command: command.slice(0, cut).trimEnd(), dropped: command.slice(cut).trim() };
|
|
732
|
+
}
|
|
733
|
+
/**
|
|
734
|
+
* Whether the command's last step runs a suite whose output still passes through a stage after
|
|
735
|
+
* the runner — one {@link stripTrailingSuiteFilters} could not prove is output-only (`| tee log`,
|
|
736
|
+
* `| xargs …`). The pipeline's exit status is that stage's, not the run's.
|
|
737
|
+
*
|
|
738
|
+
* A command whose stages cannot be placed in the text (irregular whitespace inside the run) is
|
|
739
|
+
* reported as piped too: whether a pipe follows the runner is then UNKNOWN, and an unknown exit
|
|
740
|
+
* code that gets trusted is a failing run read as a pass — the failure this rule exists to stop.
|
|
741
|
+
*
|
|
742
|
+
* @param command - The shell command, after the streaming rewrite when one was applied.
|
|
743
|
+
* @returns True when a pipe this tool must not drop still follows the suite runner, or when the
|
|
744
|
+
* command runs a suite the parse cannot place.
|
|
745
|
+
*/
|
|
746
|
+
export function suiteRunPipedAfterRunner(command) {
|
|
747
|
+
const pipeline = lastSuitePipeline(command);
|
|
748
|
+
if (pipeline !== null)
|
|
749
|
+
return pipeline.located.length > pipeline.at + 1;
|
|
750
|
+
// No placed pipeline: either there is no suite at all, or its stages cannot be located.
|
|
751
|
+
if (commandRunsNoTests(command) || locatedSegments(command) !== null)
|
|
752
|
+
return false;
|
|
753
|
+
return commandSegments(command).some((segment) => SUITE_RUN_COMMANDS.some((re) => re.test(segment.text)));
|
|
754
|
+
}
|
|
538
755
|
/**
|
|
539
756
|
* Start a background suite run at lower CPU and IO priority, so the sandbox it
|
|
540
757
|
* shares with everything else stays responsive to what the executor does next.
|
|
@@ -1474,6 +1691,44 @@ function failureDetail(body) {
|
|
|
1474
1691
|
];
|
|
1475
1692
|
return parts.filter((p) => p !== undefined).join(' · ');
|
|
1476
1693
|
}
|
|
1694
|
+
/** The longest one body line a cause's recap carries before it is shortened. */
|
|
1695
|
+
const FAILURE_EVIDENCE_CHARS = 200;
|
|
1696
|
+
/**
|
|
1697
|
+
* The two body lines worth carrying beside a cause's signature: the values that differ
|
|
1698
|
+
* (`Expected:` / `Received:`), or — where the failure asserts no pair — the locator from the
|
|
1699
|
+
* `Call log` and the error line that follows it.
|
|
1700
|
+
*
|
|
1701
|
+
* The signature is normalised ({@link normalizedSignature}): `Expected: 100000` becomes
|
|
1702
|
+
* `Expected: <n>`, and a signature built from a `waiting for` line never carries the assertion
|
|
1703
|
+
* values at all. Those values are what a fix round needs, and the run's stdout they came from is
|
|
1704
|
+
* cut in the MIDDLE by the tool-result cap, so a listed cause that does not carry them sends the
|
|
1705
|
+
* executor back to the log — xr-molecule-flash-m102 (+12:43) made four `grep -A` round trips
|
|
1706
|
+
* (~25 s) for the bodies of causes the note had already listed.
|
|
1707
|
+
*
|
|
1708
|
+
* @param body - The block's lines, from {@link failureBody}.
|
|
1709
|
+
* @param message - The cause's signature: a line it already carries is not repeated.
|
|
1710
|
+
* @returns Up to two lines, most useful first (empty when the body names none).
|
|
1711
|
+
*/
|
|
1712
|
+
function failureEvidence(body, message) {
|
|
1713
|
+
const carried = (line) => message !== null && message.includes(line);
|
|
1714
|
+
const expected = body.find((l) => /^Expected\b/.test(l));
|
|
1715
|
+
const received = body.find((l) => /^Received\b/.test(l));
|
|
1716
|
+
if (expected !== undefined && received !== undefined)
|
|
1717
|
+
return [expected, received];
|
|
1718
|
+
const picked = [];
|
|
1719
|
+
const waiting = body.find((l) => /^-?\s*waiting for (?:locator\(|navigation|element|selector)/.test(l));
|
|
1720
|
+
if (waiting !== undefined)
|
|
1721
|
+
picked.push(waiting.replace(/^-\s*/, ''));
|
|
1722
|
+
const locator = body.find((l) => /^Locator:\s/.test(l));
|
|
1723
|
+
if (locator !== undefined)
|
|
1724
|
+
picked.push(locator);
|
|
1725
|
+
// The first body line IS the error the signature already names; a later `Error:` line is the
|
|
1726
|
+
// detail that explains it (a strict-mode violation, the two elements it resolved to).
|
|
1727
|
+
const detail = body.slice(1).find((l) => /^Error:\s/.test(l));
|
|
1728
|
+
if (detail !== undefined)
|
|
1729
|
+
picked.push(detail);
|
|
1730
|
+
return picked.filter((line) => !carried(line)).slice(0, 2);
|
|
1731
|
+
}
|
|
1477
1732
|
/**
|
|
1478
1733
|
* A cause signature with the values that differ between two tests of ONE cause replaced by `<n>`:
|
|
1479
1734
|
* a duration, a generated id or timestamp, a host's port.
|
|
@@ -1644,8 +1899,31 @@ export function repeatingFailure(log) {
|
|
|
1644
1899
|
}
|
|
1645
1900
|
/** How many of a repeating cause's failures the note lists before it says "and N more". */
|
|
1646
1901
|
export const REPEATING_FAILURE_LISTED = 5;
|
|
1647
|
-
/**
|
|
1648
|
-
|
|
1902
|
+
/**
|
|
1903
|
+
* The platform's cap on one tool result's text: `MAX_TOOL_OUTPUT_CHARS` in molecule-dev's
|
|
1904
|
+
* `api/src/ai/chat-handler.ts`, which cuts a longer string field's MIDDLE. A finished run's
|
|
1905
|
+
* stdout is cut to it, so the cause list must not depend on reading the log back — and must
|
|
1906
|
+
* itself fit inside it, or the cut lands in the list.
|
|
1907
|
+
*/
|
|
1908
|
+
export const TOOL_RESULT_CHARS = 8000;
|
|
1909
|
+
/**
|
|
1910
|
+
* The characters a finished run's cause list may use of {@link TOOL_RESULT_CHARS}. The recap
|
|
1911
|
+
* measured by xr-molecule-flash-m102 (+12:43) was `12 failures, 8 causes`: one cause line is
|
|
1912
|
+
* ~200 chars and ~80 more with its two body lines, so 8 causes is ~2.3 k against the 8 k cap —
|
|
1913
|
+
* the old 5-cause cap withheld 3 causes that the same result had room for.
|
|
1914
|
+
*
|
|
1915
|
+
* The rest of the cap is left to the header, the narrowed re-run command and the exclusion
|
|
1916
|
+
* line, which ride the same note.
|
|
1917
|
+
*/
|
|
1918
|
+
export const FAILURE_CAUSES_CHARS = TOOL_RESULT_CHARS - 2600;
|
|
1919
|
+
/**
|
|
1920
|
+
* How many causes a finished run's summary lists at most. The character budget above is the
|
|
1921
|
+
* bound that must hold (it is what keeps the note on the surviving side of the tool-result
|
|
1922
|
+
* cut); this is the bound past which a list stops being read at all.
|
|
1923
|
+
*/
|
|
1924
|
+
export const FAILURE_CAUSES_LISTED = 12;
|
|
1925
|
+
/** The room the "and N more causes" sentence gets inside {@link FAILURE_CAUSES_CHARS}. */
|
|
1926
|
+
const FAILURE_CAUSES_OMITTED_CHARS = 800;
|
|
1649
1927
|
/** How many of one cause's tests that summary names before it says "and N more". */
|
|
1650
1928
|
export const FAILURE_CAUSE_TESTS_LISTED = 6;
|
|
1651
1929
|
/**
|
|
@@ -1787,6 +2065,36 @@ export function earlyReturnCause(found, universal) {
|
|
|
1787
2065
|
return null;
|
|
1788
2066
|
return universal ?? null;
|
|
1789
2067
|
}
|
|
2068
|
+
/**
|
|
2069
|
+
* The titles a request excludes that a test's heading names — the ONE matcher for "is this
|
|
2070
|
+
* failure one the request does not count?".
|
|
2071
|
+
*
|
|
2072
|
+
* The request quotes a test's own title (`… except "should require login to post comment"`,
|
|
2073
|
+
* extracted by molecule-dev's `excludedTestTitles`) while a run's heading carries more of it
|
|
2074
|
+
* (`comments.spec.ts › Comments › should require login to post comment`), so containment is the
|
|
2075
|
+
* comparison. Whitespace is normalised and case is ignored: a brief may wrap the title across a
|
|
2076
|
+
* line (`xr-molecule-flash-m44`), and the reporter's own casing never has to match the brief's.
|
|
2077
|
+
*
|
|
2078
|
+
* Two readers used to answer this separately — the causes note (which therefore asked the
|
|
2079
|
+
* executor to fix a test the same result said not to) and molecule-dev's exclusion notes — and
|
|
2080
|
+
* xr-molecule-flash-m101 paid for the gap (+12:00: a turn and an edit spent on the excluded
|
|
2081
|
+
* test).
|
|
2082
|
+
*
|
|
2083
|
+
* @param heading - The text to match against: a test's title, its whole `file › describe › title`
|
|
2084
|
+
* path, or (for a note that only has the run's output) the run's text.
|
|
2085
|
+
* @param excluded - The titles the request carves out, if the caller knows them.
|
|
2086
|
+
* @returns The excluded titles this heading names, in the request's order.
|
|
2087
|
+
*/
|
|
2088
|
+
export function matchingExcludedTitles(heading, excluded) {
|
|
2089
|
+
const normalise = (s) => s.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
2090
|
+
const haystack = normalise(heading);
|
|
2091
|
+
if (!haystack)
|
|
2092
|
+
return [];
|
|
2093
|
+
return excluded.filter((title) => {
|
|
2094
|
+
const needle = normalise(title);
|
|
2095
|
+
return needle.length > 0 && haystack.includes(needle);
|
|
2096
|
+
});
|
|
2097
|
+
}
|
|
1790
2098
|
/**
|
|
1791
2099
|
* Every failure a run names, grouped by the cause behind it — one group per cause, with the
|
|
1792
2100
|
* tests it hit.
|
|
@@ -1802,36 +2110,70 @@ export function earlyReturnCause(found, universal) {
|
|
|
1802
2110
|
* they took where it does not, which is why a duration group is labelled as a duration and not as a
|
|
1803
2111
|
* proven cause. A test the log printed twice, or failed twice, is one test.
|
|
1804
2112
|
*
|
|
2113
|
+
* A test the request EXCLUDES by name is not a cause to fix: its failure does not count for
|
|
2114
|
+
* this request ({@link matchingExcludedTitles}), so it is dropped here rather than handed to the
|
|
2115
|
+
* executor as work — a result that names it beside "it does not count" is the contradiction
|
|
2116
|
+
* xr-molecule-flash-m101 filed (+12:00: the model spent a turn and an edit on the excluded test,
|
|
2117
|
+
* on a page that was passing).
|
|
2118
|
+
*
|
|
1805
2119
|
* @param log - The log of a finished or running run.
|
|
2120
|
+
* @param excluded - The test titles the request carves out, if the caller knows them.
|
|
1806
2121
|
* @returns The causes, the one with the most tests first.
|
|
1807
2122
|
*/
|
|
1808
|
-
export function failuresByCause(log) {
|
|
2123
|
+
export function failuresByCause(log, excluded = []) {
|
|
2124
|
+
return groupFailures(log, excluded).causes;
|
|
2125
|
+
}
|
|
2126
|
+
/**
|
|
2127
|
+
* The causes a run's log names, and the failures the request excludes out of them.
|
|
2128
|
+
*
|
|
2129
|
+
* One grouping pass, because a note that lists the causes must know what it left out to say so
|
|
2130
|
+
* ({@link failureCausesNote}).
|
|
2131
|
+
*
|
|
2132
|
+
* @param log - The log of a finished or running run.
|
|
2133
|
+
* @param excluded - The test titles the request carves out.
|
|
2134
|
+
* @returns The causes to fix, and the distinct tests dropped as excluded.
|
|
2135
|
+
*/
|
|
2136
|
+
function groupFailures(log, excluded) {
|
|
1809
2137
|
const text = log.replace(ANSI_SGR, '');
|
|
1810
2138
|
const byMessage = new Map();
|
|
2139
|
+
const named = [];
|
|
1811
2140
|
for (const block of failureBlocks(text)) {
|
|
1812
2141
|
const message = failureMessage(block);
|
|
1813
2142
|
if (message === null)
|
|
1814
2143
|
continue;
|
|
1815
2144
|
const m = FAILURE_BLOCK_HEADER.exec(block.split('\n')[0] ?? '');
|
|
1816
2145
|
const { where, title } = splitTitlePath(m?.[1] ?? '');
|
|
2146
|
+
const test = { where, title, seconds: null };
|
|
2147
|
+
named.push(test);
|
|
2148
|
+
if (matchingExcludedTitles(title, excluded).length > 0)
|
|
2149
|
+
continue;
|
|
1817
2150
|
const group = byMessage.get(message);
|
|
1818
2151
|
if (group)
|
|
1819
|
-
group.push(
|
|
2152
|
+
group.tests.push(test);
|
|
2153
|
+
// The FIRST block with this signature carries the evidence: a later block of the same cause
|
|
2154
|
+
// says the same thing, and the first is the one the note's own line refers to.
|
|
1820
2155
|
else
|
|
1821
|
-
byMessage.set(message,
|
|
2156
|
+
byMessage.set(message, {
|
|
2157
|
+
tests: [test],
|
|
2158
|
+
evidence: failureEvidence(failureBody(block), message),
|
|
2159
|
+
});
|
|
1822
2160
|
}
|
|
1823
2161
|
const causes = [];
|
|
1824
|
-
for (const [message,
|
|
1825
|
-
const tests = distinctTests(
|
|
2162
|
+
for (const [message, group] of byMessage) {
|
|
2163
|
+
const tests = distinctTests(group.tests);
|
|
1826
2164
|
if (tests.length > 0)
|
|
1827
|
-
causes.push({ message, seconds: null, tests });
|
|
2165
|
+
causes.push({ message, seconds: null, tests, evidence: group.evidence });
|
|
1828
2166
|
}
|
|
1829
2167
|
// The failures with no body of their own — a `list` log with no error blocks in it, from a run
|
|
1830
2168
|
// not started in the background — are grouped by how long they took: a band is opened by its
|
|
1831
2169
|
// first test and takes the ones that
|
|
1832
2170
|
// then ran the same length, the same band {@link repeatingFailure} uses.
|
|
2171
|
+
const listed = failedTests(text);
|
|
2172
|
+
named.push(...listed);
|
|
1833
2173
|
const claimed = new Set(causes.flatMap((c) => c.tests.map(testKey)));
|
|
1834
|
-
const unclaimed = distinctTests(
|
|
2174
|
+
const unclaimed = distinctTests(listed).filter((t) => t.seconds !== null &&
|
|
2175
|
+
!claimed.has(testKey(t)) &&
|
|
2176
|
+
matchingExcludedTitles(t.title, excluded).length === 0);
|
|
1835
2177
|
const byDuration = [];
|
|
1836
2178
|
for (const t of unclaimed) {
|
|
1837
2179
|
const band = Math.max(0.5, t.seconds * 0.05);
|
|
@@ -1842,8 +2184,9 @@ export function failuresByCause(log) {
|
|
|
1842
2184
|
byDuration.push({ seconds: t.seconds, tests: [t] });
|
|
1843
2185
|
}
|
|
1844
2186
|
for (const g of byDuration)
|
|
1845
|
-
causes.push({ message: null, seconds: g.seconds, tests: g.tests });
|
|
1846
|
-
|
|
2187
|
+
causes.push({ message: null, seconds: g.seconds, tests: g.tests, evidence: [] });
|
|
2188
|
+
const ignored = distinctTests(named).filter((t) => matchingExcludedTitles(t.title, excluded).length > 0);
|
|
2189
|
+
return { causes: causes.sort((a, b) => b.tests.length - a.tests.length), ignored };
|
|
1847
2190
|
}
|
|
1848
2191
|
/**
|
|
1849
2192
|
* The one-line-per-cause summary of a finished run's failures, or null when the run named fewer
|
|
@@ -1853,32 +2196,128 @@ export function failuresByCause(log) {
|
|
|
1853
2196
|
* executor reads, so one round of fixes covers all of them rather than the one cause a run
|
|
1854
2197
|
* stopped early happened to reach.
|
|
1855
2198
|
*
|
|
2199
|
+
* A test the request excludes is not one of the causes — it is dropped and named once, at the
|
|
2200
|
+
* end, so a result never asks to fix a test it also says to ignore. Where the caller knows how
|
|
2201
|
+
* the run was started, the note also hands over the narrowed re-run of exactly those tests
|
|
2202
|
+
* ({@link rerunOfRunningSuite}) instead of leaving the executor to re-run the whole suite.
|
|
2203
|
+
*
|
|
1856
2204
|
* @param log - The finished run's output.
|
|
2205
|
+
* @param excluded - The test titles the request carves out, if the caller knows them.
|
|
2206
|
+
* @param run - How the run was started, when the caller knows it: the narrowed re-run is built
|
|
2207
|
+
* from `command`/`cwd`, and `log` (the path the output was written to) is what the "left out"
|
|
2208
|
+
* sentence greps when more causes exist than the result has room for.
|
|
1857
2209
|
* @returns The note, or null.
|
|
1858
2210
|
*/
|
|
1859
|
-
export function failureCausesNote(log) {
|
|
1860
|
-
const causes =
|
|
2211
|
+
export function failureCausesNote(log, excluded = [], run) {
|
|
2212
|
+
const { causes, ignored } = groupFailures(log, excluded);
|
|
1861
2213
|
const total = causes.reduce((n, c) => n + c.tests.length, 0);
|
|
1862
2214
|
if (total < 2)
|
|
1863
2215
|
return null;
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
})
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
2216
|
+
// The narrowed run, so the next fix round costs two tests instead of the whole suite
|
|
2217
|
+
// (xr-molecule-flash-m101 +12:13 re-ran all 139, ~4 minutes, to check two). The command is the
|
|
2218
|
+
// run's own invocation narrowed by the ONE composer the mid-run note also uses
|
|
2219
|
+
// ({@link rerunOfRunningSuite}), with the request's exclusions inverted out so the re-run
|
|
2220
|
+
// cannot report them again.
|
|
2221
|
+
const locations = [...new Set(causes.flatMap((c) => c.tests.map((t) => t.where)).filter(Boolean))];
|
|
2222
|
+
const rerun = run ? rerunOfRunningSuite(run, locations, excluded) : null;
|
|
2223
|
+
const ignoredTitles = [...new Set(ignored.map((t) => t.title || t.where).filter(Boolean))];
|
|
2224
|
+
const header = `${total} failure${total === 1 ? '' : 's'} in this run, grouped by cause — fix every cause ` +
|
|
2225
|
+
`below before the next deploy:\n`;
|
|
2226
|
+
const tail = (rerun ? ` Re-run just these once they are fixed: \`${rerun}\`` : '') +
|
|
2227
|
+
(ignoredTitles.length ? ` Ignored (excluded by the request): ${ignoredTitles.join(', ')}.` : '');
|
|
2228
|
+
// This list is the part of the result that SURVIVES the platform's tool-output cut: the stdout
|
|
2229
|
+
// above it is cut in the MIDDLE at {@link TOOL_RESULT_CHARS}, which is why the recap must fit
|
|
2230
|
+
// whole ({@link FAILURE_CAUSES_CHARS}) instead of being capped at a line count that left 3 of
|
|
2231
|
+
// xr-molecule-flash-m102's 8 causes out of a result that had room for them (+12:43).
|
|
2232
|
+
const room = Math.min(FAILURE_CAUSES_CHARS, TOOL_RESULT_CHARS - header.length - tail.length - FAILURE_CAUSES_OMITTED_CHARS);
|
|
2233
|
+
const lines = [];
|
|
2234
|
+
let used = 0;
|
|
2235
|
+
for (const cause of causes) {
|
|
2236
|
+
if (lines.length >= FAILURE_CAUSES_LISTED)
|
|
2237
|
+
break;
|
|
2238
|
+
const line = failureCauseLine(cause, lines.length);
|
|
2239
|
+
if (lines.length > 0 && used + line.length + 1 > room)
|
|
2240
|
+
break;
|
|
2241
|
+
lines.push(line);
|
|
2242
|
+
used += line.length + 1;
|
|
2243
|
+
}
|
|
2244
|
+
return `${header}${lines.join('\n')}${omittedCausesRecap(causes.slice(lines.length), run, excluded)}${tail}`;
|
|
2245
|
+
}
|
|
2246
|
+
/**
|
|
2247
|
+
* One cause's line in a finished run's recap: its signature, the tests it hit, and the first
|
|
2248
|
+
* failure's two most useful body lines underneath.
|
|
2249
|
+
*
|
|
2250
|
+
* The path is the one the log itself wrote, never a shortened one: it is relative to the
|
|
2251
|
+
* command's own cwd, so it is the form the executor can paste back (X1 flash-m82 read a shortened
|
|
2252
|
+
* `e2e/auth.spec.ts:19` as the suite's own stale starter spec, rewrote its Playwright config over
|
|
2253
|
+
* it and deleted the app's tests — 4 minutes of a 60-minute run on a working config).
|
|
2254
|
+
*
|
|
2255
|
+
* @param cause - The cause to write.
|
|
2256
|
+
* @param index - Its position in the recap (0-based; printed 1-based).
|
|
2257
|
+
* @returns The line, with its body line when it has one.
|
|
2258
|
+
*/
|
|
2259
|
+
function failureCauseLine(cause, index) {
|
|
2260
|
+
const tests = cause.tests.slice(0, FAILURE_CAUSE_TESTS_LISTED).map((t) => t.where || t.title);
|
|
2261
|
+
const more = cause.tests.length > tests.length ? `, … and ${cause.tests.length - tests.length} more` : '';
|
|
2262
|
+
const signature = cause.message !== null
|
|
2263
|
+
? `\`${cause.message}\``
|
|
2264
|
+
: `each after ~${cause.seconds}s (a shared duration, not a proven cause)`;
|
|
2265
|
+
const files = new Set(cause.tests.map((t) => specFile(t.where)).filter(Boolean)).size;
|
|
2266
|
+
// A line the signature already carries (a timeout's `waiting for` locator, say) is not printed
|
|
2267
|
+
// twice; the rest are the values the normalised signature replaced with `<n>`.
|
|
2268
|
+
const body = cause.evidence.filter((line) => cause.message === null || !cause.message.includes(line));
|
|
2269
|
+
const shortened = body.map((line) => line.length > FAILURE_EVIDENCE_CHARS ? `${line.slice(0, FAILURE_EVIDENCE_CHARS - 1)}…` : line);
|
|
2270
|
+
return (` ${index + 1}. ${signature} — ${cause.tests.length} test${cause.tests.length === 1 ? '' : 's'} ` +
|
|
2271
|
+
`in ${files} file${files === 1 ? '' : 's'}: ${tests.join(', ')}${more}` +
|
|
2272
|
+
(shortened.length ? `\n ${shortened.join(' · ')}` : ''));
|
|
2273
|
+
}
|
|
2274
|
+
/**
|
|
2275
|
+
* The sentence for the causes a recap could not fit: exactly which `file:line` each one was
|
|
2276
|
+
* found at, and the ONE command that prints their bodies.
|
|
2277
|
+
*
|
|
2278
|
+
* Without it the old note said "… and 3 more causes" and left the executor to find them in a
|
|
2279
|
+
* stdout whose middle the tool-result cut had already removed — xr-molecule-flash-m102 (+12:43)
|
|
2280
|
+
* spent four `grep -A` round trips (~25 s) naming three causes the platform had withheld, and a
|
|
2281
|
+
* weaker turn reads the listed ones and skips the rest.
|
|
2282
|
+
*
|
|
2283
|
+
* @param omitted - The causes past the recap's budget.
|
|
2284
|
+
* @param run - How the run was started, and where its log is, when the caller knows.
|
|
2285
|
+
* @param excluded - The test titles the request carves out (the fallback re-run inverts them out).
|
|
2286
|
+
* @returns The sentence, or '' when nothing was omitted.
|
|
2287
|
+
*/
|
|
2288
|
+
function omittedCausesRecap(omitted, run, excluded) {
|
|
2289
|
+
if (omitted.length === 0)
|
|
2290
|
+
return '';
|
|
2291
|
+
const all = omitted
|
|
2292
|
+
.map((c) => c.tests[0]?.where || c.tests[0]?.title)
|
|
2293
|
+
.filter((l) => Boolean(l));
|
|
2294
|
+
// The one command: a grep of the run's own log when the caller knows its path (the bodies are
|
|
2295
|
+
// still there, whole), else a narrowed re-run of exactly these tests, which prints them.
|
|
2296
|
+
const print = (named) => {
|
|
2297
|
+
if (run?.log && named.length > 0)
|
|
2298
|
+
return (` Print them: \`grep -n -A 40 ` +
|
|
2299
|
+
`${named.map((l) => `-e ${shellQuote(l)}`).join(' ')} ${shellQuote(run.log)}\`.`);
|
|
2300
|
+
const rerun = run ? rerunOfRunningSuite(run, named, excluded) : null;
|
|
2301
|
+
if (rerun)
|
|
2302
|
+
return ` Print them by re-running just these: \`${rerun}\`.`;
|
|
2303
|
+
if (named.length === 0)
|
|
2304
|
+
return '';
|
|
2305
|
+
return ` Print one with \`grep -n -A 40 -e ${shellQuote(named[0])} <the run's log>\`.`;
|
|
2306
|
+
};
|
|
2307
|
+
let keep = all.length;
|
|
2308
|
+
for (;;) {
|
|
2309
|
+
const named = all.slice(0, keep);
|
|
2310
|
+
const dropped = all.length - keep;
|
|
2311
|
+
const sentence = `\n … and ${omitted.length} more cause${omitted.length === 1 ? '' : 's'}` +
|
|
2312
|
+
(named.length
|
|
2313
|
+
? `: ${named.join(', ')}${dropped > 0 ? ` (+${dropped} more, also in the log)` : ''}`
|
|
2314
|
+
: '') +
|
|
2315
|
+
` — their bodies are past the part of this result the cut keeps.` +
|
|
2316
|
+
print(named);
|
|
2317
|
+
if (sentence.length <= FAILURE_CAUSES_OMITTED_CHARS || keep <= 1)
|
|
2318
|
+
return sentence;
|
|
2319
|
+
keep = Math.max(1, Math.floor(keep / 2));
|
|
2320
|
+
}
|
|
1882
2321
|
}
|
|
1883
2322
|
/**
|
|
1884
2323
|
* The command that re-runs the tests at `locations` the way the run IN FLIGHT ran them: the same
|
|
@@ -1895,12 +2334,14 @@ export function failureCausesNote(log) {
|
|
|
1895
2334
|
* @param run - The run in flight: the command it was started with, as it was run, and the
|
|
1896
2335
|
* directory that command ran in.
|
|
1897
2336
|
* @param locations - The failing `file:line`s to re-run, as the run's log printed them.
|
|
2337
|
+
* @param excluded - The test titles the request carves out: the re-run inverts them out
|
|
2338
|
+
* (`--grep-invert`) so it cannot report them again.
|
|
1898
2339
|
* @returns The re-run command, or null when there is no suite to narrow.
|
|
1899
2340
|
*/
|
|
1900
|
-
export function rerunOfRunningSuite(run, locations) {
|
|
2341
|
+
export function rerunOfRunningSuite(run, locations, excluded = []) {
|
|
1901
2342
|
if (locations.length === 0)
|
|
1902
2343
|
return null;
|
|
1903
|
-
const scoped = scopedRerunCommand(run.command, locations);
|
|
2344
|
+
const scoped = scopedRerunCommand(run.command, locations, excluded);
|
|
1904
2345
|
if (scoped === run.command)
|
|
1905
2346
|
return null;
|
|
1906
2347
|
if (!run.cwd || commandSegments(withoutHeredocs(run.command)).some((s) => s.program === 'cd'))
|