@giamme/forge 0.11.0 → 0.12.0

package/scripts/forge_dataset/forecast.py

@@ -243,13 +243,25 @@ def _pct(v):
     return '%.0f%%' % (v * 100)
 
 
-def lines(fc: dict) -> list:
+def _names(rows: list, most: int = 8) -> str:
+    names = [r['task'] for r in rows]
+    return ', '.join(names[:most]) + (', ...' if len(names) > most else '')
+
+
+def lines(fc: dict, brief: bool = False) -> list:
+    """The forecast as text. brief (the block `run` prints) gives tasks without enough history
+    one line together instead of one row each, and no table at all when no task has enough."""
+    thin = [r for r in fc['tasks'] if not r['enough']]
+    if brief and len(thin) == len(fc['tasks']):
+        return ['no forecast yet: none of the %d task(s) has %d past tasks at its tier and model '
+                '(forge dataset forecast %s shows each count)' % (len(thin), MIN_HISTORY, fc['plan'])]
     out = ['forecast from %d past task(s) at the same tier and model (advisory; routing never reads it)'
            % fc['history_tasks']]
     for r in fc['tasks']:
         where = '%s %s' % (r['tier'] or '?', r['spec'] or '?')
         if not r['enough']:
-            out.append('  %-14s %-22s %s' % (r['task'], where, r['note']))
+            if not brief:
+                out.append('  %-14s %-22s %s' % (r['task'], where, r['note']))
             continue
         fp = r['first_pass']
         t, s = r['tokens'], r['time_s']
@@ -257,6 +269,9 @@ def lines(fc: dict) -> list:
             r['task'], where, _pct(fp['rate']), _pct(fp['ci'][0]), _pct(fp['ci'][1]), fp['k'], fp['n'],
             _tokens(t['median']), _tokens(t['p90']), _time(s['median']), _time(s['p90']),
             '  <- likely to need a retry' if r['flag'] else ''))
+    if brief and thin:
+        out.append('  %d task(s) without enough history yet (under %d past tasks): %s'
+                   % (len(thin), MIN_HISTORY, _names(thin)))
     if fc['merged']:
         out.append('  %d task(s) already merged, not forecast' % len(fc['merged']))
     tot = fc['total']
@@ -275,12 +290,17 @@ def lines(fc: dict) -> list:
     return out
 
 
+def _tmp(plan: Path) -> Path:
+    """Where save() writes forecast.json before moving it into place."""
+    return plan / ('.forecast.json.%d' % os.getpid())
+
+
 def save(plan: Path, fc: dict) -> None:
     """Keep what was predicted, so it can be checked against what happened. Fail-open."""
     body = {k: v for k, v in fc.items() if k != 'at'}
     digest = hashlib.sha256(json.dumps(body, sort_keys=True).encode()).hexdigest()[:16]
+    tmp = _tmp(plan)
     try:
-        tmp = plan / ('.forecast.json.%d' % os.getpid())
         tmp.write_text(json.dumps(fc, indent=2, sort_keys=True) + '\n')
         tmp.replace(plan / 'forecast.json')
     except OSError:
@@ -302,7 +322,7 @@ def advise(plan: Path, shown: list) -> None:
         return
     if not fc or not fc['tasks']:
         return
-    for line in lines(fc):
+    for line in lines(fc, brief=True):
         print(line)
     sys.stdout.flush()
     shown.append(True)
@@ -336,6 +356,11 @@ def main(args) -> int:
             if not shown:
                 print('forecast skipped: reading the history took over %gs' % budget())
                 sys.stdout.flush()
+            # os._exit stops save() wherever it is, so a half-written temp file would stay in the plan.
+            try:
+                _tmp(plan).unlink()
+            except OSError:
+                pass
             os._exit(0)
         return 0
     if not (plan / 'tasks.tsv').is_file():

package/scripts/forge_dataset/harvest.py

@@ -43,7 +43,7 @@ WATCHED = (
     'verification.status', 'verification.exit', 'verification.command', 'verification.log',
     'verification.retry.log', 'verification.flaky', 'verification.coverage.txt',
     'verification.jev.json', 'verification.runs.tsv', 'verification.commit', 'verification.fingerprint',
-    'results.tsv',
+    'verification.lanes.json', 'results.tsv',
 )
 
 _VERDICT = re.compile(r'^FORGE_VERDICT: (PASS|FAIL)([ \t]+(—|–|-)( .*)?)?[ \t]*$')
@@ -544,8 +544,11 @@ def verification_events(attempt: Path, out: Path, rc, prov: dict, src: str = 'li
     repo = first_line(plan / 'repo')
     command = first_line(out / 'verification.command') if made('verification.command') else None
     jev = read_json(out / 'verification.jev.json') if made('verification.jev.json') else None
+    lanes = made('verification.lanes.json')
     if not command:
         source = None
+    elif lanes:
+        source = 'lanes:plan' if (plan / 'verify.d').is_dir() else 'lanes:repo'
     elif (plan / 'verify_cmd').exists():
         source = 'flag'
     elif jev and jev.get('discovered'):
@@ -559,9 +562,12 @@ def verification_events(attempt: Path, out: Path, rc, prov: dict, src: str = 'li
         for line in (text(out / 'verification.runs.tsv') or '').splitlines():
             cells = line.split('\t')
             if len(cells) >= 5 and cells[0].isdigit():
-                runs.append(dict(n=int(cells[0]), kind=cells[1], exit=as_int(cells[2]),
-                                 tree_changed=cells[3] == '1', duration_s=as_int(cells[4]),
-                                 log=cells[5] if len(cells) > 5 else None))
+                run = dict(n=int(cells[0]), kind=cells[1], exit=as_int(cells[2]),
+                           tree_changed=cells[3] == '1', duration_s=as_int(cells[4]),
+                           log=cells[5] if len(cells) > 5 else None)
+                if len(cells) >= 8:          # a lane's run: n kind exit tree_changed duration_s log lane started
+                    run['lane'] = cells[6] or None
+                runs.append(run)
     elif command and made('verification.log'):
         runs.append(dict(n=1, kind='initial', exit=as_int(first_line(out / 'verification.exit')),
                          tree_changed=None, duration_s=None, log='verification.log'))

package/scripts/forge_verify_reports.py

@@ -0,0 +1,225 @@
+"""Test reports for forge-verify.py: JUnit XML and TAP 12-14. Stdlib only, offline.
+
+``scan(directory, since)`` reads every ``*.xml`` and ``*.tap`` under a lane's report
+directory that was written at or after ``since`` (an epoch float: the start of that run),
+recursively and without following symlinks, and returns the lane's ``tests``:
+
+    {"pass", "fail", "skip", "error",
+     "failing": [{"name", "suite", "file", "message", "excerpt", "known", "log_line"}],
+     "reports": [<paths relative to the directory>], "notes": [...], "truncated": bool}
+
+plus what a baseline verdict needs and the display does not keep: every failing name (not
+only the first 200) and whether the evidence is complete. A report that cannot be used
+(too big, unreadable, not parseable, a DOCTYPE or ENTITY declaration) is skipped with a
+note and makes the evidence incomplete; nothing here raises on a hostile file.
+"""
+from __future__ import annotations
+
+import os
+import re
+import stat
+import xml.etree.ElementTree as ET
+
+FILE_MAX = 32 << 20
+FAILING_MAX = 200
+MESSAGE_MAX = 500
+EXCERPT_MAX = 2000
+LOG_SEARCH_MAX = 64 << 20
+
+ANSI = re.compile(r'\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[@-Z\\-_]')
+# Refused before parsing: no external entities, no entity expansion bombs.
+DECLARATIONS = re.compile(rb'<!\s*(?:DOCTYPE|ENTITY)', re.I)
+TAP_LINE = re.compile(r'(not\s+)?ok\b\s*(\d+)?\s*(?:-\s*)?(.*)$')
+TAP_PLAN = re.compile(r'1\.\.(\d+)\b')
+TAP_BAIL = re.compile(r'Bail out!\s*(.*)$')
+TAP_DIRECTIVE = re.compile(r'\s*(?<!\\)#\s*(SKIP|TODO)\b', re.I)
+
+
+def clean(text: str | None, cap: int) -> str:
+    return ANSI.sub('', text or '').strip()[:cap]
+
+
+def first_line(text: str | None) -> str:
+    for line in (text or '').splitlines():
+        if line.strip():
+            return line.strip()
+    return ''
+
+
+def junit_cases(data: bytes) -> list[dict]:
+    """[{name, suite, outcome, message, excerpt}] from a JUnit XML document; ValueError when unusable."""
+    # The declaration check reads bytes, so only an ASCII-compatible encoding is let through:
+    # UTF-16 or UTF-32 (a byte-order mark, or NUL bytes) could hide a DOCTYPE from it.
+    if data[:2] in (b'\xff\xfe', b'\xfe\xff') or b'\x00' in data:
+        raise ValueError('only UTF-8 reports are read')
+    if DECLARATIONS.search(data):
+        raise ValueError('DOCTYPE or ENTITY declarations are refused')
+    try:
+        root = ET.fromstring(data)
+    except ET.ParseError as error:
+        raise ValueError('not well-formed XML (%s)' % error)
+    if root.tag not in ('testsuite', 'testsuites'):
+        raise ValueError('no testsuite or testsuites element')
+    parents = {child: parent for parent in root.iter() for child in parent}
+    cases = []
+    for case in root.iter('testcase'):
+        name, classname = case.get('name') or '?', case.get('classname')
+        suite = parents.get(case)
+        problem = case.find('failure')
+        outcome = 'fail'
+        if problem is None:
+            problem, outcome = case.find('error'), 'error'
+        if problem is None:
+            outcome = 'skip' if case.find('skipped') is not None else 'pass'
+        entry = {'name': '%s.%s' % (classname, name) if classname else name,
+                 'suite': suite.get('name') if suite is not None and suite.tag == 'testsuite' else None,
+                 'outcome': outcome, 'message': '', 'excerpt': ''}
+        if problem is not None:
+            text = problem.text or ''
+            entry['message'] = problem.get('message') or first_line(text)
+            parts = [text] + [(case.findtext(tag) or '') for tag in ('system-out', 'system-err')]
+            entry['excerpt'] = '\n'.join(p.strip('\n') for p in parts if p.strip())
+        cases.append(entry)
+    return cases
+
+
+def tap_cases(data: bytes) -> tuple[list[dict], str | None]:
+    """Top-level test points of a TAP stream, each with its YAML diagnostic block if any, and
+    why the stream is incomplete (a `Bail out!`, no plan, or fewer or more points than its
+    plan), else None. An incomplete stream's points still count, but cannot prove a baseline."""
+    try:
+        text = data.decode('utf-8')
+    except UnicodeDecodeError:
+        raise ValueError('not UTF-8 text')
+    cases: list[dict] = []
+    block: list[str] | None = None
+    planned = bail = None
+    for line in text.splitlines():
+        if block is not None:
+            if line.strip() == '...':
+                if cases:
+                    diag = '\n'.join(block)
+                    cases[-1]['excerpt'] = diag
+                    found = re.search(r'^\s*message:\s*(.*)$', diag, re.M)
+                    if found:
+                        cases[-1]['message'] = found.group(1).strip().strip('\'"')
+                block = None
+            else:
+                block.append(line)
+            continue
+        if line.strip() == '---' and line.startswith(' ') and cases:
+            block = []
+            continue
+        stop = TAP_BAIL.match(line)
+        if stop:
+            bail = stop.group(1).strip() or 'no reason given'
+            break
+        plan = TAP_PLAN.match(line)
+        if plan and planned is None:
+            planned = int(plan.group(1))
+            continue
+        match = TAP_LINE.match(line)     # top level only: subtests are indented
+        if not match:
+            continue
+        description = match.group(3)
+        directive = TAP_DIRECTIVE.search(description)
+        name = description[:directive.start()].strip() if directive else description.strip()
+        if directive:
+            outcome = 'skip'
+        else:
+            outcome = 'fail' if match.group(1) else 'pass'
+        cases.append({'name': name or 'test %s' % (match.group(2) or len(cases) + 1), 'suite': None,
+                      'outcome': outcome, 'message': '', 'excerpt': ''})
+    if not cases:
+        raise ValueError('no TAP test points')
+    if bail is not None:
+        return cases, 'it bailed out (%s)' % clean(bail, MESSAGE_MAX)
+    if planned is None:
+        return cases, 'it has no plan line (1..N)'
+    if planned != len(cases):
+        return cases, 'its plan says %d tests and it has %d' % (planned, len(cases))
+    return cases, None
+
+
+def scan(directory: str, since: float) -> tuple[dict, list[str], bool]:
+    """(tests, every failing name, whether the evidence is complete) for one lane run."""
+    tests = {'pass': 0, 'fail': 0, 'skip': 0, 'error': 0, 'failing': [], 'reports': [],
+             'notes': [], 'truncated': False}
+    names: list[str] = []
+    complete = True
+    found = []
+    for top, dirs, files in os.walk(directory, followlinks=False):
+        dirs.sort()
+        for name in sorted(files):
+            if name.endswith(('.xml', '.tap')):
+                found.append(os.path.join(top, name))
+    for path in found:
+        rel = os.path.relpath(path, directory)
+        try:
+            info = os.lstat(path)
+        except OSError:
+            continue
+        if not stat.S_ISREG(info.st_mode) or info.st_mtime < since:
+            continue          # a symlink, or a report an earlier run left
+        try:
+            if info.st_size > FILE_MAX:
+                raise ValueError('larger than %d MB' % (FILE_MAX >> 20))
+            with open(path, 'rb') as handle:
+                data = handle.read(FILE_MAX + 1)
+            if len(data) > FILE_MAX:
+                raise ValueError('larger than %d MB' % (FILE_MAX >> 20))
+            cases, problem = (junit_cases(data), None) if rel.endswith('.xml') else tap_cases(data)
+        except (OSError, ValueError, RecursionError) as error:
+            tests['notes'].append('%s skipped: %s' % (rel, error))
+            complete = False
+            continue
+        tests['reports'].append(rel)
+        if problem:
+            tests['notes'].append('%s is incomplete: %s' % (rel, problem))
+            complete = False
+        for case in cases:
+            tests[case['outcome']] += 1
+            if case['outcome'] not in ('fail', 'error'):
+                continue
+            names.append(case['name'])
+            if len(tests['failing']) >= FAILING_MAX:
+                tests['truncated'] = True
+                continue
+            tests['failing'].append({'name': clean(case['name'], MESSAGE_MAX), 'suite': case['suite'],
+                                     'file': rel, 'message': clean(case['message'], MESSAGE_MAX),
+                                     'excerpt': clean(case['excerpt'], EXCERPT_MAX),
+                                     'known': False, 'log_line': None})
+    return tests, names, complete
+
+
+def log_lines(log: str, names: list[str]) -> dict:
+    """{name: 1-based number of the first log line containing it}, within the first 64 MB."""
+    wanted = sorted({n for n in names if n}, key=len, reverse=True)
+    if not wanted:
+        return {}
+    pattern = re.compile('|'.join(re.escape(n) for n in wanted))
+    found: dict = {}
+    read = 0
+    try:
+        with open(log, 'rb') as handle:
+            for number, raw in enumerate(handle, 1):
+                read += len(raw)
+                if read > LOG_SEARCH_MAX:
+                    break
+                line = raw.decode('utf-8', 'replace')
+                if not pattern.search(line):    # one search per line; most lines name no test
+                    continue
+                for name in wanted:
+                    if name not in found and name in line:
+                        found[name] = number
+                if len(found) == len(wanted):
+                    break
+    except OSError:
+        pass
+    return found
+
+
+def load_baseline(path: str) -> set:
+    """Known failing test names: one per line, blank lines and # comments ignored."""
+    with open(path, encoding='utf-8', errors='replace') as handle:
+        return {line.strip() for line in handle if line.strip() and not line.strip().startswith('#')}

package/ui/README.md

@@ -12,9 +12,12 @@ blocks on a plate, with cables between them and lamps that light up as things ha
   dependency cables, and ends in a *finish* line of three nodes: integration, verification
   and the whole-run **final review**. *Execution* lays the same work out on wall-clock time:
   one block per attempt, in the lane it really ran in, with its dwarf and review inside, the
-  attempts that have not started projected by forge-schedule's own rules, and a strip of how
-  many slots were busy against `--max-parallel`. Switching between the two moves each task
-  from its wave to where it ran.
+  attempts that have not started projected by forge-schedule's own rules, a strip of how
+  many slots were busy against `--max-parallel`, and the verification's lanes and reruns on
+  rows of their own. Switching between the two moves each task from its wave to where it ran.
+- **Verification as it happens:** the step each lane is in, its last line, the failing tests
+  first (new or known, from the lanes' test reports) with a link into the log at each one, and
+  every lane's output a click away.
 - **Click to open.** A task opens to show its attempts, each with its dwarf, its reviewer and
   any shadow reviews, linked; an escalation is a labelled cable between attempts. The
   inspector beside the stage shows what launched each step and what came of it: the exact
@@ -178,6 +181,18 @@ keep one meaning everywhere (amber running, green merged or passed, red failed,
   with the gate decision, Clef predictions against outcomes, and its prompts; for one
   execution, its model, phases, tokens, exact prompt and final message; for the final
   review, its verdict, findings, report, prompt and history.
+- **Verification (its finish node):** while it runs, how long it has run, the step a lane
+  announced (`::forge step name=… state=start`) and for how long, the last line it printed, a
+  warning after five minutes with no output, and the live output of the lane that printed last
+  (choose a lane to follow it instead). Once it failed, the failing tests come first, grouped by
+  lane and marked *new* or *known* (from the lane's JUnit or TAP reports and its baseline), each
+  with its message, an excerpt and *open in log* at its line; a lane without reports shows the
+  end of its log, and the step it ended in is named. A table lists every lane with its state
+  (flaky and known marked), time, runs and test counts; a lane's name opens its output. A
+  changed checkout is called out. A single verify command shows the newest of its logs live.
+  In *Execution*, the verification gets rows after the tasks: one per lane, with a bar per run
+  and reruns marked, or a single command's runs back to back (dashed: only their durations are
+  recorded). The finish node counts the lanes and reruns and lights one lamp per lane.
 - **Files (the run toolbar's third mode):** a heatmap of files (grouped and collapsible by
   directory) against tasks (grouped by wave): planned files outlined, touched files filled by how
   many attempts changed them, drift hatched, merge conflicts ringed, and a dot where another task
@@ -266,7 +281,7 @@ takes effect. The routes are in [docs/api.md](docs/api.md).
   infra-retry budgets. A task has a threshold override (`Inherit`, P0–P3 or `None`, which
   exempts it from the run's threshold), its own retry budget and a **model pin**: the model
   its next full retry uses, once. A control that cannot apply (no live scheduler, a merged
-  task, a Fractal plan's pin) is disabled and says why.
+  task) is disabled and says why. A Fractal plan's pin control is hidden.
 - **Open findings (`forge issues`):** the findings a run, or every run of its repository,
   left open. The list opens from a quick offline listing and then fills in what is already
   filed; select findings to file as GitHub issues, or dismiss them (and undo that).