harness-everything 0.47.1 → 0.47.2

package/benchmarks/issue290-pilot/read_metrics.py

@@ -0,0 +1,166 @@
+"""Shared, conservative reference attribution; never execute captured commands.
+
+Filesystem access, mixed shell output and identified reference content are
+different metrics. Only paths in the episode's declared document bundle count.
+Unknown transformations remain uncertain, rather than being counted as reads.
+"""
+import hashlib
+import re
+import shlex
+
+READERS = {'cat', 'sed', 'head', 'tail', 'rg', 'grep', 'get-content', 'gc', 'type'}
+
+
+def segments(command):
+    if not isinstance(command, str):
+        return []
+    try:
+        # Preserve backslashes in PowerShell paths. Quoted shell wrappers are
+        # unwrapped before this lexer is used for their actual program.
+        command = command.replace('<HARNESS_SOURCE>', '__HARNESS_SOURCE__').replace('<TMP>', '__TMP__')
+        lexer = shlex.shlex(command, posix=True, punctuation_chars=';&|()<>')
+        lexer.escape = ''
+        lexer.whitespace_split = True
+        tokens = [token.replace('__HARNESS_SOURCE__', '<HARNESS_SOURCE>').replace('__TMP__', '<TMP>') for token in lexer]
+    except (ValueError, TypeError):
+        return []
+    if tokens and tokens[0].rsplit('/', 1)[-1] in ('sh', 'bash', 'zsh') and len(tokens) == 3 and tokens[1] in ('-c', '-lc'):
+        return segments(tokens[2])
+    result, current = [], []
+    for token in tokens:
+        if token and all(c in ';&|()<>' for c in token):
+            if current:
+                result.append(current)
+            current = []
+        else:
+            current.append(token)
+    if current:
+        result.append(current)
+    return result
+
+
+def is_content_command(command):
+    return any(part and part[0].lower().rsplit('/', 1)[-1] in READERS
+               and not (part[0].lower() == 'rg' and '--files' in part)
+               for part in segments(command))
+
+
+def reference_path(value, documents, source=''):
+    value = str(value).replace('\\', '/').removeprefix('./')
+    for prefix in ('.harness/', '<HARNESS_SOURCE>/', source.rstrip('/') + '/' if source else ''):
+        if prefix and value.startswith(prefix):
+            relative = value[len(prefix):]
+            return relative if relative in documents else None
+    # Native Read can supply an absolute workspace/.harness path.
+    if '/.harness/' in value:
+        relative = value.split('/.harness/', 1)[1]
+        return relative if relative in documents else None
+    return value if value in documents else None
+
+
+def observed_read(command, output, documents, source='', native_path=None):
+    paths = []
+    if native_path is not None:
+        relative = reference_path(native_path, documents, source)
+        if relative:
+            paths.append(relative)
+    else:
+        for part in segments(command):
+            if not part or not is_content_command(shlex.join(part)):
+                continue
+            for token in part[1:]:
+                relative = reference_path(token, documents, source)
+                if relative and relative not in paths:
+                    paths.append(relative)
+    if not paths:
+        return None
+    remaining = output or ''
+    matched = []
+    # Match complete line runs or complete files. A numeric wc result, a
+    # filename listing, or prose mentioning a path does not identify content.
+    # Very short isolated fragments stay uncertain to avoid incidental matches.
+    for relative in paths:
+        content = documents[relative]
+        if not content:
+            continue
+        if content in remaining:
+            remaining = remaining.replace(content, '', 1)
+            matched.append({'path': relative, 'bytes': len(content.encode()), 'kind': 'full-file'})
+            continue
+        lines = remaining.splitlines(keepends=True)
+        index = 0
+        while index < len(lines):
+            end = index + 1
+            best = None
+            while end <= len(lines):
+                fragment = ''.join(lines[index:end])
+                if fragment not in content:
+                    break
+                if len(fragment.strip()) >= 32:
+                    best = (end, fragment)
+                end += 1
+            if best:
+                end, fragment = best
+                matched.append({'path': relative, 'bytes': len(fragment.encode()), 'kind': 'partial-content'})
+                remaining = remaining.replace(fragment, '', 1)
+                index = end
+            else:
+                index += 1
+    attributable = sum(item['bytes'] for item in matched)
+    total = len((output or '').encode())
+    return {'command': command, 'paths': paths, 'filesystem_access_candidates': len(paths),
+            'shell_output_bytes': total, 'returned_bytes': attributable,
+            'exact_reference_bytes': attributable,
+            'uncertain_or_mixed_output_bytes': total - attributable,
+            'matched_content': matched,
+            'content_sha256': hashlib.sha256((output or '').encode()).hexdigest(),
+            'evidence': 'content match against frozen declared bundle; access alone is not returned content'}
+
+
+def collect_reads(events, documents, source=''):
+    reads, commands, errors = [], [], []
+    tools = {tool['id']: tool for event in events
+             for tool in event.get('message', {}).get('content', [])
+             if isinstance(tool, dict) and tool.get('type') == 'tool_use' and tool.get('id')}
+    active = 'uncomposed'
+    for event in events:
+        outputs = []
+        item = event.get('item', {})
+        if event.get('type') == 'item.completed' and item.get('type') == 'command_execution':
+            outputs.append((item.get('command', ''), item.get('aggregated_output', '')))
+            if item.get('exit_code') not in (0, None):
+                errors.append({'command': item.get('command'), 'exit': item.get('exit_code')})
+        result = event.get('tool_use_result')
+        if isinstance(result, dict):
+            if result.get('type') == 'text' and isinstance(result.get('file'), dict):
+                file = result['file']
+                read = observed_read('', file.get('content', ''), documents, source, file.get('filePath', ''))
+                if read:
+                    reads.append({'active_step': active, **read})
+            if 'stdout' in result:
+                command = ''
+                for tool in event.get('message', {}).get('content', []):
+                    if isinstance(tool, dict) and tool.get('type') == 'tool_result':
+                        command = tools.get(tool.get('tool_use_id'), {}).get('input', {}).get('command', '')
+                outputs.append((command, result.get('stdout', '')))
+                if result.get('interrupted'):
+                    errors.append({'command': command, 'interrupted': True})
+        elif isinstance(result, str) and result.startswith('Error:'):
+            errors.append({'tool_error': result})
+        for command, output in outputs:
+            commands.append(command)
+            if 'workflow-disposition.js' in command:
+                import json
+                try:
+                    data, _ = json.JSONDecoder().raw_decode(output.lstrip())
+                    if isinstance(data, dict):
+                        if data.get('activeStep'):
+                            active = data['activeStep']['id']
+                        elif data.get('activeStages'):
+                            active = ','.join(stage.get('stageId', '?') for stage in data['activeStages'])
+                except (ValueError, KeyError, TypeError):
+                    pass
+            read = observed_read(command, output, documents, source)
+            if read:
+                reads.append({'active_step': active, **read})
+    return reads, commands, errors

package/benchmarks/issue290-pilot/replay.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""Rebuild pilot evidence offline, from hash-checked committed inputs only."""
+import argparse
+import hashlib
+import importlib
+import json
+from pathlib import Path, PurePosixPath
+import tarfile
+import tempfile
+
+from analyze import inspect_episode
+from reporting import paired_rows, report_tables, summarize_groups
+
+HERE = Path(__file__).resolve().parent
+exporter = importlib.import_module('export')
+
+
+def key(row):
+    return (row['series'], row['model']['engine'], row['workload'], row['arm'], row['repeat'])
+
+
+def extract_verified(target):
+    inventory = json.loads((HERE/'evidence/inventory.json').read_text())
+    expected = {item['path']: item for item in inventory}
+    found = set()
+    with tarfile.open(HERE/'evidence.tar.gz') as archive:
+        for member in archive.getmembers():
+            parts = PurePosixPath(member.name).parts
+            if not parts or parts[0] != 'evidence' or '..' in parts or member.issym() or member.islnk():
+                raise ValueError('unsafe archive member')
+            if member.isdir():
+                continue
+            relative = '/'.join(parts[1:])
+            if relative == 'inventory.json':
+                continue
+            if not member.isfile() or relative not in expected or relative in found:
+                raise ValueError('unexpected/duplicate evidence file')
+            raw = archive.extractfile(member).read()
+            if hashlib.sha256(raw).hexdigest() != expected[relative]['exported_sha256']:
+                raise ValueError('exported evidence hash mismatch')
+            exporter.validate_no_secrets(raw.decode())
+            path = target/relative
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(raw)
+            found.add(relative)
+    if found != set(expected):
+        raise ValueError('missing exported evidence files')
+    return len(found)
+
+
+def regenerate():
+    with tempfile.TemporaryDirectory(prefix='harness-pilot-replay-') as temporary:
+        root = Path(temporary)
+        count = extract_verified(root)
+        frozen_rows = json.loads((root/'issue290-final-audit/audited-rows.json').read_text())
+        frozen = {key(row): row for row in frozen_rows}
+        if len(frozen) != 40:
+            raise ValueError('frozen audit count/identity mismatch')
+        bundle = json.loads((HERE/'frozen-documents.json').read_text())
+        rows = []
+        for manifest_path in sorted(root.glob('*/*/manifest.json')):
+            episode = manifest_path.parent
+            manifest = json.loads(manifest_path.read_text())
+            identity = {**manifest, 'series': episode.parent.name}
+            documents = {}
+            for name, digest in manifest['documents'].items():
+                document = bundle['documents'][name]
+                if document['source_sha256'] != digest or hashlib.sha256(document['content'].encode()).hexdigest() != digest:
+                    raise ValueError('document revision differs from episode manifest')
+                documents[name] = document['content']
+            # The live JS fixture declared ESM, but the original episode
+            # exporter retained only the implementation. Restore that pinned
+            # launch prerequisite instead of relying on host module detection.
+            if manifest['workload'] == 'polyglot-simple-linked-list-staged':
+                package = '{"type":"module"}\n'
+                if manifest['fixture'].get('package.json') != hashlib.sha256(package.encode()).hexdigest():
+                    raise ValueError('unexpected module fixture identity')
+                (episode/'package.json').write_text(package)
+            row = inspect_episode(episode, offline=True, frozen=frozen[key(identity)], documents=documents)
+            row['series'] = episode.parent.name
+            # No host reasoning, assistant prose, dispatch arguments, or child
+            # transcripts are republished by the derived ledger.
+            keep = ('series', 'model', 'workload', 'arm', 'repeat', 'preflight', 'classification',
+                    'wall_seconds', 'exit', 'stop_reason', 'budget_failure', 'cost_observability',
+                    'api_equivalent_cost_usd', 'api_equivalent_input_cost_usd', 'parent_api_equivalent_cost_usd',
+                    'reference_returned_bytes', 'exact_reference_bytes_lower_bound',
+                    'reference_filesystem_access_candidates', 'reference_shell_output_bytes',
+                    'uncertain_or_mixed_output_bytes', 'observed_reference_reads',
+                    'effective_effort_reported', 'input_usage_reconciles', 'usage',
+                    'child_usage_observability', 'strategies', 'tdd_quality_gate_name_matched_invocation',
+                    'independent_grader', 'telemetry_provenance')
+            rows.append({name: row.get(name) for name in keep})
+        if len(rows) != 40 or {key(row) for row in rows} != set(frozen):
+            raise ValueError('replayed episode identities differ')
+        rows.sort(key=key)
+        pairs = paired_rows(rows)
+        output = {'ledger.json': rows, 'group-summary.json': summarize_groups(rows), 'pairs.json': pairs,
+                  'provenance.json': {'verified_exported_files': count,
+                    'archive_sha256': hashlib.sha256((HERE/'evidence.tar.gz').read_bytes()).hexdigest(),
+                    'pricing_assumptions_sha256': hashlib.sha256((HERE/'pricing-assumptions.json').read_bytes()).hexdigest(),
+                    'frozen_documents_sha256': hashlib.sha256((HERE/'frozen-documents.json').read_bytes()).hexdigest(),
+                    'recomputed': ['parent-stream content attribution', 'artifact checks', 'gate-name invocation', 'metric denominators', 'paired deltas'],
+                    'frozen_not_independently_recovered': ['request usage', 'child dispatch metadata', 'effort', 'usage aggregates'],
+                    'irretrievable': ['unobserved worker usage/model identities', 'interrupted final usage'],
+                    'unknown_policy': 'non-reconstructable whole-run costs remain null; no host-home reads or model calls'}}
+        result = {}
+        for name, value in output.items():
+            text = json.dumps(value, indent=2, ensure_ascii=False, sort_keys=True)+'\n'
+            text = text.replace(str(root), '<REPLAY_ROOT>').replace(str(HERE.parent.parent), '<HARNESS_SOURCE>')
+            exporter.validate_no_secrets(text)
+            result[name] = text
+        result['tables.md'] = report_tables(rows, pairs)
+        return result
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--check', action='store_true', help='compare regenerated data with committed expected values')
+    args = parser.parse_args()
+    result = regenerate()
+    def destination(name):
+        return HERE/'evidence'/name if name in ('ledger.json', 'group-summary.json') else HERE/'evidence/replay'/name
+    if args.check:
+        for name, text in result.items():
+            if not destination(name).is_file() or destination(name).read_text() != text:
+                if name.endswith('.json') and destination(name).is_file():
+                    def differences(expected, actual, path='$'):
+                        if type(expected) is not type(actual):
+                            yield path+' (type)'
+                        elif isinstance(expected, dict):
+                            for field in sorted(set(expected) | set(actual)):
+                                if field not in expected or field not in actual:
+                                    yield path+'.'+field+' (missing)'
+                                else:
+                                    yield from differences(expected[field], actual[field], path+'.'+field)
+                        elif isinstance(expected, list):
+                            if len(expected) != len(actual):
+                                yield path+' (length)'
+                            for index, (left, right) in enumerate(zip(expected, actual)):
+                                yield from differences(left, right, f'{path}[{index}]')
+                        elif expected != actual:
+                            yield path
+                    import itertools
+                    print('different fields:', ', '.join(itertools.islice(differences(
+                        json.loads(destination(name).read_text()), json.loads(text)), 12)))
+                raise SystemExit('offline replay differs: '+name)
+        readme = (HERE/'README.md').read_text()
+        if '<!-- BEGIN REPLAY TABLES -->\n'+result['tables.md']+'<!-- END REPLAY TABLES -->' not in readme:
+            raise SystemExit('offline replay differs: README tables')
+        print('offline replay PASS: 40 episodes; 621 hashes; ledger, groups, pairs, tables and provenance match')
+    else:
+        for name, text in result.items():
+            path = destination(name)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(text)
+        readme = HERE/'README.md'
+        content = readme.read_text()
+        if '<!-- BEGIN REPLAY TABLES -->' in content:
+            import re
+            content = re.sub(r'<!-- BEGIN REPLAY TABLES -->.*?<!-- END REPLAY TABLES -->',
+                             lambda _: '<!-- BEGIN REPLAY TABLES -->\n'+result['tables.md']+'<!-- END REPLAY TABLES -->', content, flags=re.S)
+            readme.write_text(content)
+        print('regenerated offline evidence:', len(result), 'files')
+
+
+if __name__ == '__main__':
+    main()

package/benchmarks/issue290-pilot/reporting.py

@@ -0,0 +1,82 @@
+"""Metric-specific denominators and deterministic descriptive pairs."""
+from collections import defaultdict
+
+
+def quality_included(row):
+    return not row.get('preflight') and (row.get('classification') != 'inconclusive' or row.get('budget_failure'))
+
+
+def summarize_groups(rows):
+    groups = defaultdict(list)
+    for row in rows:
+        groups[(row['series'], row['model']['engine'], row['workload'], row['arm'])].append(row)
+    result = []
+    for key, group in sorted(groups.items()):
+        included = [row for row in group if quality_included(row)]
+        record = dict(zip(('series', 'engine', 'workload', 'arm'), key))
+        record.update(n_attempted=len(group), n_quality_included=len(included),
+                      n_quality_excluded=len(group)-len(included),
+                      artifact_passes=sum(bool(r['independent_grader'].get('pass')) and not r.get('budget_failure') for r in included),
+                      budget_failures=sum(bool(r.get('budget_failure')) for r in included))
+        for output, field in [('reference_bytes', 'reference_returned_bytes'),
+                              ('exact_reference_bytes_lower_bound', 'exact_reference_bytes_lower_bound'),
+                              ('mean_wall_seconds', 'wall_seconds'),
+                              ('api_equivalent_cost_usd', 'api_equivalent_cost_usd'),
+                              ('api_equivalent_input_cost_usd', 'api_equivalent_input_cost_usd')]:
+            known = [r[field] for r in included if r.get(field) is not None]
+            record['n_'+output+'_included'] = len(known)
+            record['n_'+output+'_excluded'] = len(group)-len(known)
+            # Missing data never becomes zero. A known subtotal is explicitly
+            # different from an unknown group total.
+            record[output+'_known_subtotal'] = sum(known) if known else None
+            record[output] = (sum(known)/len(known) if output.startswith('mean_') else sum(known)) if known and len(known) == len(included) else None
+        result.append(record)
+    return result
+
+
+def paired_rows(rows):
+    cells = defaultdict(dict)
+    for row in rows:
+        if quality_included(row):
+            key = (row['series'], row['model']['engine'], row['workload'], row['repeat'])
+            if row['arm'] in cells[key]:
+                raise ValueError('duplicate arm in paired cell')
+            cells[key][row['arm']] = row
+    pairs = []
+    for key, arms in sorted(cells.items()):
+        pair = dict(zip(('series', 'engine', 'workload', 'repeat'), key))
+        for field in ('reference_returned_bytes', 'api_equivalent_input_cost_usd', 'api_equivalent_cost_usd', 'wall_seconds'):
+            a, b = (arms.get(arm, {}).get(field) for arm in ('A', 'B'))
+            known = a is not None and b is not None
+            pair[field] = {'included': known, 'A': a, 'B': b,
+                           'delta_percent': (b-a)/a*100 if known and a else None}
+        pairs.append(pair)
+    return pairs
+
+
+def report_tables(rows, pairs):
+    lines = ['<!-- Generated by replay.py; parent-stream content attribution only. -->',
+             '| Family | Model | Pass / quality n | Excluded attempts |',
+             '|---|---|---:|---:|']
+    families = defaultdict(list)
+    for row in rows:
+        if not row.get('preflight'):
+            name = 'True Fable' if 'fable' in row['series'] else row['workload']
+            families[(name, row['model']['engine'])].append(row)
+    for (name, engine), group in sorted(families.items()):
+        included = [r for r in group if quality_included(r)]
+        passes = sum(bool(r['independent_grader'].get('pass')) and not r.get('budget_failure') for r in included)
+        lines.append(f'| {name} | {engine} | {passes}/{len(included)} | {len(group)-len(included)} |')
+    for field, title, precision in [('reference_returned_bytes', 'Attributable reference bytes (lower bound)', 0),
+                                    ('api_equivalent_input_cost_usd', 'List-price input cost (frozen telemetry)', 6)]:
+        lines.extend(['', '### '+title, '', '| Family | Model | Included / candidate pairs | A | B | Delta |', '|---|---|---:|---:|---:|---:|'])
+        grouped = defaultdict(list)
+        for pair in pairs:
+            name = 'True Fable' if 'fable' in pair['series'] else pair['workload']
+            grouped[(name, pair['engine'])].append(pair[field])
+        for (name, engine), group in sorted(grouped.items()):
+            known = [p for p in group if p['included']]
+            a, b = (sum(p[arm] for p in known) for arm in ('A', 'B'))
+            values = [f'{a:.{precision}f}', f'{b:.{precision}f}', f'{(b-a)/a*100:+.1f}%' if a else 'unknown'] if known else ['unknown']*3
+            lines.append(f'| {name} | {engine} | {len(known)}/{len(group)} | '+ ' | '.join(values)+' |')
+    return '\n'.join(lines)+'\n'