@andespindola/crow-skills 0.40.0 → 0.41.0

package/skills/crow-skill-maker/scripts/creator.py

@@ -0,0 +1,839 @@
+#!/usr/bin/env python3
+"""Explicit isolated native-Claude Skill Creator runtime; never a Codex equivalence claim."""
+import argparse
+import contextlib
+import hashlib
+import fcntl
+import json
+import math
+import os
+from pathlib import Path, PurePosixPath
+import re
+import runpy
+from datetime import datetime, timezone
+import selectors
+import shutil
+import signal
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+import threading
+import uuid
+
+SCHEMA = 'crow-maker-creator/1'
+MANIFEST_SHA256 = '002bd425fe66fba00460e4b853cf2ce9955877ca309528f98cac00973195a023'
+COMMIT = '2c5958ecac7cee2b583967a2c4981d0f4a75c579'
+DEPENDENCIES = ['PyYAML==6.0.3']
+SCRIPTS = {name: 'scripts/' + name + '.py' for name in (
+    'run_eval', 'run_loop', 'improve_description', 'quick_validate',
+    'package_skill', 'aggregate_benchmark', 'generate_report')}
+SCRIPTS['generate_review'] = 'eval-viewer/generate_review.py'
+VENDOR = Path(__file__).absolute().parent.parent / 'vendor'
+
+class CreatorError(ValueError):
+    pass
+
+def _hash(raw):
+    return hashlib.sha256(raw).hexdigest()
+
+def _path(path, missing=False):
+    p = Path(os.path.abspath(os.fspath(path)))
+    for part in [*reversed(p.parents), p]:
+        try:
+            info = part.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise CreatorError('path-unavailable') from None
+        if stat.S_ISLNK(info.st_mode):
+            raise CreatorError('symlink-path-rejected')
+    return p
+
+def _read(path, limit=16*1024*1024):
+    p = _path(path)
+    fd = os.open(p.anchor, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        for i, part in enumerate(p.parts[1:]):
+            flags = os.O_RDONLY | os.O_NOFOLLOW | getattr(os, 'O_NONBLOCK', 0)
+            if i < len(p.parts[1:])-1:
+                flags |= os.O_DIRECTORY
+            child = os.open(part, flags, dir_fd=fd)
+            os.close(fd)
+            fd = child
+        if not stat.S_ISREG(os.fstat(fd).st_mode):
+            raise CreatorError('nonregular-file')
+        chunks, remaining = [], limit+1
+        while remaining:
+            chunk = os.read(fd, min(65536, remaining))
+            if not chunk:
+                break
+            chunks.append(chunk)
+            remaining -= len(chunk)
+        raw = b''.join(chunks)
+        if len(raw) > limit:
+            raise CreatorError('file-too-large')
+        return raw
+    finally:
+        os.close(fd)
+
+def _private_directory(path, create=False):
+    p = _path(path, missing=create)
+    if create:
+        # Only caller-provided final directory: parents must already be safe.
+        _path(p.parent)
+        p.mkdir(mode=0o700, exist_ok=True)
+    info = p.stat()
+    if not p.is_dir() or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) & 0o077:
+        raise CreatorError('directory-must-be-owned-and-private')
+    return p
+
+def _write(path, raw, mode=0o600):
+    p = _path(path, missing=True)
+    fd = os.open(p.anchor,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
+    try:
+        for part in p.parts[1:-1]:
+            child=os.open(part,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=fd)
+            os.close(fd);fd=child
+        out=os.open(p.name,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode,dir_fd=fd)
+        with os.fdopen(out,'wb') as stream:
+            stream.write(raw)
+    finally:
+        os.close(fd)
+
+
+def _donor_files():
+    raw = _read(VENDOR / 'vendor-manifest.json')
+    if _hash(raw) != MANIFEST_SHA256:
+        raise CreatorError('manifest-hash-mismatch')
+    manifest = json.loads(raw)
+    prefix = 'skills/crow-skill-maker/vendor/skill-creator/'
+    files = {}
+    for entry in manifest['files']:
+        copied = entry['copiedAs']
+        if not copied.startswith(prefix):
+            continue
+        relative = copied[len(prefix):]
+        parts = PurePosixPath(relative).parts
+        if not parts or '..' in parts or PurePosixPath(relative).is_absolute() or '\\' in relative:
+            raise CreatorError('manifest-path-escape')
+        if entry['repository'] != 'anthropics/skills' or entry['commit'] != COMMIT or relative in files:
+            raise CreatorError('unexpected-source-identity')
+        raw = _read(VENDOR / 'skill-creator' / relative)
+        if _hash(raw) != entry['upstreamSha256']:
+            raise CreatorError('donor-hash-mismatch')
+        files[relative] = raw
+    expected = set(files)
+    actual = set()
+    for directory, dirs, names in os.walk(_path(VENDOR / 'skill-creator'), followlinks=False):
+        for name in dirs + names:
+            path = Path(directory) / name
+            if path.is_symlink():
+                raise CreatorError('donor-symlink')
+        for name in names:
+            actual.add((Path(directory)/name).relative_to(VENDOR/'skill-creator').as_posix())
+    if actual != expected or not all(v in files for v in SCRIPTS.values()) or 'SKILL.md.source' not in files:
+        raise CreatorError('donor-file-set-mismatch')
+    return files
+
+def _kill_group_impl(process):
+    try:
+        os.killpg(process.pid, signal.SIGTERM)
+    except ProcessLookupError:
+        pass
+    try:
+        process.wait(timeout=.5)
+    except subprocess.TimeoutExpired:
+        pass
+    # Leader may have exited while descendants remain; still signal its group.
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass
+    try:
+        process.wait(timeout=2)
+    except subprocess.TimeoutExpired:
+        pass
+
+class _Cancellation(KeyboardInterrupt):
+    def __init__(self,signum):
+        self.signum=signum
+
+class _SignalGuard:
+    def __init__(self):
+        self.depth=0;self.pending=None
+    def handler(self,signum,frame):
+        if self.depth:
+            if self.pending is None:self.pending=signum
+        else:raise _Cancellation(signum)
+    @contextlib.contextmanager
+    def deferred(self):
+        self.depth+=1
+        try:yield
+        finally:
+            self.depth-=1
+            if not self.depth and self.pending is not None:
+                signum,self.pending=self.pending,None
+                raise _Cancellation(signum)
+
+@contextlib.contextmanager
+def _cancellation_scope():
+    guard=_SignalGuard()
+    installed=threading.current_thread() is threading.main_thread()
+    previous={sig:signal.getsignal(sig) for sig in (signal.SIGTERM,signal.SIGINT)} if installed else {}
+    if installed:
+        for sig in previous:signal.signal(sig,guard.handler)
+    try:yield guard
+    finally:
+        for sig,handler in previous.items():signal.signal(sig,handler)
+
+def _kill_group(process):
+    # Parent-only handler deferral cannot leak a blocked mask into exec. A
+    # second cancellation is delivered only AFTER TERM -> grace -> KILL ends.
+    with _cancellation_scope() as guard:
+        with guard.deferred():
+            _kill_group_impl(process)
+
+def _spawn_owned(command,**kwargs):
+    process=None
+    try:
+        with _cancellation_scope() as guard:
+            with guard.deferred():
+                process=subprocess.Popen(command,start_new_session=True,**kwargs)
+        return process
+    except BaseException:
+        if process is not None:
+            try:_kill_group(process)
+            except _Cancellation:pass  # Preserve original cancellation after cleanup.
+        raise
+
+
+def _bounded_setup(command, timeout=300):
+    with _cancellation_scope():
+        process=None
+        try:
+            process=_spawn_owned(command,stdin=subprocess.DEVNULL,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+            code = process.wait(timeout=timeout)
+            if code:raise CreatorError('dependency-setup-failed')
+        except _Cancellation:
+            raise
+        except KeyboardInterrupt:
+            raise _Cancellation(signal.SIGINT) from None
+        except subprocess.TimeoutExpired:
+            raise CreatorError('dependency-setup-interrupted') from None
+        finally:
+            if process is not None:_kill_group(process)
+
+
+def _outside_native_host(path):
+    path = _path(path,missing=True)
+    if path == Path.home():
+        raise CreatorError('native-home-runtime-forbidden')
+    for forbidden in (Path.home()/'.claude',Path.home()/'.codex',Path.home()/'.agents',Path.home()/'.config/devin'):
+        if path == forbidden or forbidden in path.parents:
+            raise CreatorError('native-host-runtime-forbidden')
+    return path
+
+def setup(runtime_root, yes=False):
+    if not yes:
+        raise CreatorError('explicit-setup-yes-required')
+    if sys.version_info < (3,10):
+        raise CreatorError('native-donor-requires-python-3.10-or-later')
+    files = _donor_files()  # Verify everything BEFORE any write.
+    root = _private_directory(_outside_native_host(runtime_root), create=True)
+    if any(root.iterdir()):
+        raise CreatorError('runtime-root-must-be-empty-no-overwrite')
+    # A failed setup remains visibly partial, never ready or implicitly repaired.
+    _write(root/'setup-state.json', json.dumps({'status':'partial','dependencies':DEPENDENCIES}).encode())
+    creator = root/'skill-creator'
+    creator.mkdir(mode=0o700)
+    for relative, raw in files.items():
+        destination = creator / ('SKILL.md' if relative == 'SKILL.md.source' else relative)
+        destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        _write(destination, raw)
+    _bounded_setup([sys.executable, '-m', 'venv', str(root/'venv')])
+    python = root/'venv/bin/python'
+    _bounded_setup([str(python), '-m', 'pip', 'install', '--disable-pip-version-check', *DEPENDENCIES])
+    _write(root/'ready.json', json.dumps({'schema':SCHEMA,'commit':COMMIT,'manifestSha256':MANIFEST_SHA256,
+        'dependencies':DEPENDENCIES,'pythonVersion':sys.version.split()[0],
+        'files':{('SKILL.md' if k=='SKILL.md.source' else k):_hash(v) for k,v in files.items()}}).encode())
+    return status(root)
+
+def status(runtime_root):
+    root = _private_directory(_outside_native_host(runtime_root))
+    ready = root/'ready.json'
+    if not ready.exists():
+        return {'schema':SCHEMA,'status':'partial' if any(root.iterdir()) else 'not-installed','exitCode':1}
+    meta = json.loads(_read(ready, 1024*1024))
+    files = _donor_files()
+    expected = {('SKILL.md' if k=='SKILL.md.source' else k):_hash(v) for k,v in files.items()}
+    if meta.get('manifestSha256') != MANIFEST_SHA256 or meta.get('commit') != COMMIT or meta.get('files') != expected or meta.get('dependencies') != DEPENDENCIES:
+        raise CreatorError('runtime-metadata-mismatch')
+    for relative, digest in expected.items():
+        if _hash(_read(root/'skill-creator'/relative)) != digest:
+            raise CreatorError('runtime-source-hash-mismatch')
+    # venv's interpreter symlink is created by venv itself, not donor-controlled.
+    if not (root/'venv/bin/python').is_file():
+        raise CreatorError('runtime-python-missing')
+    return {'schema':SCHEMA,'status':'ready','exitCode':0,'dependencies':DEPENDENCIES,
+            'nativeHost':'Claude Code CLI','scripts':sorted(SCRIPTS),
+            'limitations':['Native Claude trigger evaluation, not Codex host equivalence.','Runtime and workspace are isolation conventions, not a sandbox.']}
+
+def _option(args, name):
+    for i, arg in enumerate(args):
+        if arg == name:
+            if i+1 >= len(args):
+                raise CreatorError('missing-option-value')
+            return args[i+1]
+        if arg.startswith(name+'='):
+            return arg.split('=',1)[1]
+    return None
+
+def _arguments(script, arguments):
+    args = list(arguments)
+    if script in ('run_eval','run_loop') and _option(args,'--num-workers') is None:
+        args += ['--num-workers','1']
+    if script == 'run_loop' and _option(args,'--report') is None:
+        args += ['--report','none']
+    return args
+
+def _guard_skill(arguments, workspace):
+    value = _option(arguments,'--skill-path')
+    if value is None:
+        return
+    path = Path(value)
+    if not path.is_absolute():
+        path = workspace/path
+    raw = _read(_path(path)/'SKILL.md').decode('utf-8')
+    lines = raw.split('\n')
+    if not lines or lines[0].strip() != '---':
+        raise CreatorError('unsafe-native-skill-name')
+    end = next((i for i in range(1,len(lines)) if lines[i].strip()=='---'),None)
+    names = [line[len('name:'):].strip().strip('"').strip("'")
+             for line in lines[1:end] if line.startswith('name:')] if end is not None else []
+    if len(names) != 1 or not re.fullmatch(r'[a-z0-9][a-z0-9-]*',names[0]):
+        raise CreatorError('unsafe-native-skill-name')
+
+
+def _guard_io(script,args,work):
+    def checked(value):
+        p=Path(value)
+        return _path(p if p.is_absolute() else work/p,missing=True)
+    for option in ('--eval-set','--skill-path','--eval-results','--history','--results-dir',
+                   '--output','-o','--previous-workspace','--benchmark','--static','-s','--report'):
+        value=_option(args,option)
+        if value and not (option=='--report' and value in ('none','auto')):
+            checked(value)
+    if not args or args[0].startswith('-'):
+        return
+    if script in ('package_skill','quick_validate','aggregate_benchmark','generate_report','generate_review') and args[0]!='-':
+        source=checked(args[0])
+        if script=='package_skill':
+            output=checked(args[1]) if len(args)>1 else work
+            _path(output/(source.name+'.skill'),missing=True)
+            if source.is_dir():
+                for directory,dirs,files in os.walk(source,followlinks=False):
+                    for name in dirs+files:
+                        if (Path(directory)/name).is_symlink():
+                            raise CreatorError('package-source-symlink-rejected')
+        if script=='aggregate_benchmark':
+            output=_option(args,'--output') or _option(args,'-o')
+            target=checked(output) if output else source/'benchmark.json'
+            _path(target,missing=True);_path(target.with_suffix('.md'),missing=True)
+        if script=='generate_review':
+            _path(source/'feedback.json',missing=True)
+
+def _observed(value, integer=False):
+    try:
+        if type(value) not in (int,float) or not math.isfinite(value) or value < 0:
+            return None
+    except OverflowError:
+        return None
+    return value if not integer or type(value) is int else None
+
+def _benchmark_stats(values):
+    # A partial mean would silently drop missing runs. Keep the whole statistic
+    # unknown; coverage separately records exactly how many metrics were observed.
+    if not values or any(value is None for value in values):
+        return None
+    try:
+        mean=sum(values)/len(values)
+        variance=sum((x-mean)**2 for x in values)/(len(values)-1) if len(values)>1 else 0
+        result={'mean':round(mean,4),'stddev':round(math.sqrt(variance),4),
+                'min':min(values),'max':max(values),'n':len(values)}
+        return result if all(math.isfinite(value) for value in result.values()) else None
+    except (OverflowError,ValueError):
+        return None
+
+def corrected_benchmark(directory,skill_name='',skill_path=''):
+    """Creator-owned adapter. Retains donor artifact shape, not its invented metrics."""
+    directory=Path(directory)
+    search=directory/'runs' if (directory/'runs').is_dir() else directory
+    rows=[];issues=[]
+    def data(path,required=False):
+        if not path.exists():
+            if required:issues.append('missing-grading-file')
+            return {}
+        try:
+            item=json.loads(_read(path))
+            if not isinstance(item,dict):raise ValueError()
+            return item
+        except (CreatorError,OSError,ValueError):
+            issues.append('unreadable-or-invalid-artifact')
+            return {}
+    for index,eval_dir in enumerate(sorted(search.glob('eval-*'))):
+        if not eval_dir.is_dir():continue
+        metadata=data(eval_dir/'eval_metadata.json')
+        fallback=int(eval_dir.name[5:]) if eval_dir.name[5:].isdigit() else index
+        eval_id=metadata.get('eval_id',fallback)
+        if type(eval_id) is not int:eval_id=fallback
+        for config in sorted(eval_dir.iterdir()):
+            if not config.is_dir():continue
+            for run_dir in sorted(config.glob('run-*')):
+                if not run_dir.is_dir():continue
+                grading=data(run_dir/'grading.json',True)
+                timing=data(run_dir/'timing.json')
+                grade_timing=grading.get('timing',{})
+                grade_timing=grade_timing if isinstance(grade_timing,dict) else {}
+                metrics=grading.get('execution_metrics',{})
+                metrics=metrics if isinstance(metrics,dict) else {}
+                summary=grading.get('summary',{})
+                summary=summary if isinstance(summary,dict) else {}
+                elapsed=_observed(grade_timing.get('total_duration_seconds'))
+                if elapsed is None:elapsed=_observed(timing.get('total_duration_seconds'))
+                tokens=_observed(timing.get('total_tokens'),True)
+                if tokens is None:tokens=_observed(metrics.get('total_tokens'),True)
+                rate=_observed(summary.get('pass_rate'))
+                if rate is not None and rate>1:rate=None
+                result={'pass_rate':rate,'passed':_observed(summary.get('passed'),True),
+                    'failed':_observed(summary.get('failed'),True),'total':_observed(summary.get('total'),True),
+                    'time_seconds':elapsed,'tokens':tokens,
+                    'output_chars':_observed(metrics.get('output_chars'),True),
+                    'tool_calls':_observed(metrics.get('total_tool_calls'),True),
+                    'errors':_observed(metrics.get('errors_encountered'),True)}
+                number=run_dir.name[4:]
+                rows.append({'eval_id':eval_id,'configuration':config.name,
+                    'run_number':int(number) if number.isdigit() else run_dir.name,'result':result,
+                    'expectations':grading.get('expectations',[]),'notes':[]})
+    configs=sorted({row['configuration'] for row in rows})
+    evals=sorted({row['eval_id'] for row in rows})
+    counts={config:sum(row['configuration']==config for row in rows) for config in configs}
+    cells={str(e):{config:sum(row['eval_id']==e and row['configuration']==config for row in rows)
+                  for config in configs} for e in evals}
+    cell_values=[count for item in cells.values() for count in item.values()]
+    uniform=cell_values[0] if cell_values and len(set(cell_values))==1 else None
+    summary={};coverage={}
+    for config in configs:
+        selected=[row for row in rows if row['configuration']==config]
+        summary[config]={};coverage[config]={}
+        for metric in ('pass_rate','time_seconds','tokens','output_chars'):
+            values=[row['result'][metric] for row in selected]
+            summary[config][metric]=_benchmark_stats(values)
+            observed=sum(value is not None for value in values)
+            coverage[config][metric]={'observed':observed,'missing':len(values)-observed}
+    delta={}
+    for metric,digits in (('pass_rate',2),('time_seconds',1),('tokens',0),('output_chars',0)):
+        a=summary[configs[0]][metric] if configs else None
+        b=summary[configs[1]][metric] if len(configs)>1 else None
+        delta[metric]=format(a['mean']-b['mean'],'+.'+str(digits)+'f') if a and b else 'unknown'
+    summary['delta']=delta
+    complete=bool(rows) and not issues and all(counts['missing']==0 for item in coverage.values()
+                for metric,counts in item.items() if metric in ('pass_rate','time_seconds','tokens')) and all(
+                summary[config][metric] is not None for config in configs for metric in ('pass_rate','time_seconds','tokens'))
+    return {'metadata':{'skill_name':skill_name or '<skill-name>','skill_path':skill_path or '<path/to/skill>',
+        'executor_model':None,'analyzer_model':None,'timestamp':datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
+        'evals_run':evals,'runs_per_configuration':uniform,'observed_run_counts':counts,
+        'observed_counts_by_eval':cells,'adapter':'crow-creator-truthful-benchmark/1'},
+        'runs':rows,'run_summary':summary,'coverage':{'complete':complete,'metrics':coverage,'issues':issues},
+        'notes':['Creator-owned corrected aggregation; donor source remains unchanged.',
+            'Unknown metrics are null, never zero. Output characters are not tokens.',
+            'Supplied grading and token fields are input evidence, not independent behavior/provider attestations.']}
+
+def corrected_benchmark_markdown(benchmark):
+    metadata=benchmark['metadata'];summary=benchmark['run_summary']
+    configs=[key for key in summary if key!='delta']
+    lines=['# Skill Benchmark: '+metadata['skill_name'],'',
+        '**Model**: unknown (not inferred from artifacts)',
+        '**Date**: '+metadata['timestamp'],
+        '**Observed runs**: '+('; '.join(key+': '+str(value) for key,value in metadata['observed_run_counts'].items()) or 'none'),
+        '**Coverage**: '+('complete for reported metrics' if benchmark['coverage']['complete'] else 'incomplete; missing metrics remain unknown'),'',
+        '| Metric | '+' | '.join(key.replace('_',' ').title() for key in configs)+' | Delta |',
+        '|---|'+'---|'*(len(configs)+1)]
+    for metric,label in (('pass_rate','Pass Rate'),('time_seconds','Time (s)'),('tokens','Tokens'),('output_chars','Output Characters')):
+        values=[]
+        for config in configs:
+            item=summary[config][metric]
+            if item is None:values.append('unknown')
+            elif metric=='pass_rate':values.append(format(item['mean']*100,'.0f')+'% (n='+str(item['n'])+')')
+            else:values.append(format(item['mean'],'.4g')+' (n='+str(item['n'])+')')
+        lines.append('| '+label+' | '+' | '.join(values)+' | '+summary['delta'][metric]+' |')
+    lines += ['','## Notes','']+['- '+note for note in benchmark['notes']]
+    return '\n'.join(lines)
+
+def _aggregate_entry(target,arguments):
+    namespace=runpy.run_path(target)
+    namespace['main'].__globals__.update(generate_benchmark=corrected_benchmark,
+                                         generate_markdown=corrected_benchmark_markdown)
+    sys.argv=[target,*arguments]
+    namespace['main']()
+    return 0
+
+def _truthful_viewer_html(html):
+    """Patch only reviewed pinned renderer snippets, leaving all viewer features."""
+    replacements = [
+        ('    function renderBenchmark() {', '''    function renderBenchmark() {
+      // Crow adapter: missing observations stay unknown through rendering.
+      const crowKnown = v => typeof v === "number" && Number.isFinite(v) && v >= 0;
+      const crowMean = values => values.length && values.every(crowKnown)
+        ? values.reduce((a,b) => a+b, 0) / values.length : null;
+      const crowMeanText = (values, percent=false) => {
+        const mean = crowMean(values);
+        return mean == null || !Number.isFinite(mean) ? "unknown (incomplete)"
+          : (mean * (percent ? 100 : 1)).toFixed(percent ? 0 : 1) + (percent ? "%" : "");
+      };
+'''),
+        ('if (!stat) return "—";', 'if (!stat || !crowKnown(stat.mean) || !crowKnown(stat.stddev)) return "unknown";'),
+        ('(metadata.runs_per_configuration || "?") + " runs per configuration"',
+         '(metadata.observed_run_counts ? escapeHtml(Object.entries(metadata.observed_run_counts).map(([k,v]) => k + ": " + v).join("; ")) + " observed runs" : (metadata.runs_per_configuration ?? "unknown") + " runs per configuration")'),
+        ('if (a.time_seconds || b.time_seconds) {', 'if ("time_seconds" in a || "time_seconds" in b) {'),
+        ('if (a.tokens || b.tokens) {', 'if ("tokens" in a || "tokens" in b) {'),
+        ('(delta.time_seconds ? delta.time_seconds + "s" : "—")',
+         '(delta.time_seconds != null && Number.isFinite(Number(delta.time_seconds)) ? delta.time_seconds + "s" : "unknown")'),
+        ('const hasTime = runs.some(r => r.result && r.result.time_seconds != null);',
+         'const hasTime = runs.some(r => r.result && "time_seconds" in r.result);'),
+        ('const hasErrors = runs.some(r => r.result && r.result.errors > 0);',
+         'const hasErrors = runs.some(r => r.result && "errors" in r.result);'),
+        ('const prClass = r.pass_rate >= 0.8 ? "benchmark-delta-positive" : r.pass_rate < 0.5 ? "benchmark-delta-negative" : "";',
+         'const prClass = crowKnown(r.pass_rate) ? (r.pass_rate >= 0.8 ? "benchmark-delta-positive" : r.pass_rate < 0.5 ? "benchmark-delta-negative" : "") : "";'),
+        ('((r.pass_rate || 0) * 100).toFixed(0) + "% (" + (r.passed || 0) + "/" + (r.total || 0) + ")',
+         '(crowKnown(r.pass_rate) ? (r.pass_rate * 100).toFixed(0) + "%" : "unknown") + " (" + (crowKnown(r.passed) ? r.passed : "unknown") + "/" + (crowKnown(r.total) ? r.total : "unknown") + ")'),
+        ('(r.time_seconds != null ? r.time_seconds.toFixed(1) : "—")',
+         '(crowKnown(r.time_seconds) ? r.time_seconds.toFixed(1) : "unknown")'),
+        ('(r.errors || 0)', '(crowKnown(r.errors) ? r.errors : "unknown")'),
+        ('const rates = configRuns.map(r => (r.result || {}).pass_rate || 0);',
+         'const rates = configRuns.map(r => (r.result || {}).pass_rate);'),
+        ('const avgRate = rates.reduce((a, b) => a + b, 0) / rates.length;', 'const avgRate = crowMean(rates);'),
+        ('const avgPrClass = avgRate >= 0.8 ? "benchmark-delta-positive" : avgRate < 0.5 ? "benchmark-delta-negative" : "";',
+         'const avgPrClass = avgRate == null ? "" : (avgRate >= 0.8 ? "benchmark-delta-positive" : avgRate < 0.5 ? "benchmark-delta-negative" : "");'),
+        ('(avgRate * 100).toFixed(0) + "%', 'crowMeanText(rates, true) + "'),
+        ('const times = configRuns.map(r => (r.result || {}).time_seconds).filter(t => t != null);',
+         'const times = configRuns.map(r => (r.result || {}).time_seconds);'),
+        ('(times.length ? (times.reduce((a, b) => a + b, 0) / times.length).toFixed(1) : "—")',
+         'crowMeanText(times)'),
+        ('const cls = exp.passed ? "benchmark-delta-positive" : "benchmark-delta-negative";',
+         'const cls = exp.passed === true ? "benchmark-delta-positive" : exp.passed === false ? "benchmark-delta-negative" : "";'),
+        ('const icon = exp.passed ? "\\u2713" : "\\u2717";',
+         'const icon = exp.passed === true ? "\\u2713" : exp.passed === false ? "\\u2717" : "unknown";'),
+    ]
+    for original, replacement in replacements:
+        if html.count(original) != 1:
+            raise CreatorError('pinned-viewer-adapter-target-changed')
+        html=html.replace(original,replacement,1)
+    return html
+
+def _viewer_namespace(target):
+    namespace=runpy.run_path(target)
+    globals_=namespace['main'].__globals__
+    original=globals_['generate_html']
+    def render(*args,**kwargs):
+        return _truthful_viewer_html(original(*args,**kwargs))
+    # Static main and served ReviewHandler share these module globals.
+    globals_.update(generate_html=render,_kill_port=lambda port:None)
+    return namespace
+
+def _viewer_entry(target,arguments):
+    namespace=_viewer_namespace(target)
+    sys.argv=[target,*arguments]
+    namespace['main']()
+    return 0
+
+def _event(path, value):
+    # Readers only glob final event suffixes; publish fully written bytes once.
+    temp=path.with_name('.event-'+uuid.uuid4().hex)
+    try:
+        _write(temp,json.dumps(value).encode())
+        os.replace(temp,path)
+    finally:
+        if temp.exists():temp.unlink()
+
+def _traces(directory):
+    starts = list(directory.glob('*.start'))
+    done, failures, early, invalid, protocol_failures, protocol_incomplete = 0, 0, 0, 0, 0, 0
+    for start in starts:
+        path = start.with_suffix('.done')
+        if path.exists():
+            try:
+                item = json.loads(_read(path,4096))
+                if not isinstance(item,dict) or not (type(item.get('returncode')) is int or
+                        (item.get('returncode') is None and isinstance(item.get('error'),str) and item['error'])):
+                    raise CreatorError('invalid-final-trace')
+            except (OSError,CreatorError,ValueError):
+                invalid += 1
+                continue
+            done += 1
+            failures += item.get('returncode') != 0 or bool(item.get('error'))
+            early += bool(item.get('consumerClosedEarly'))
+            protocol=item.get('protocol',{})
+            if isinstance(protocol,dict) and protocol.get('status')=='error':protocol_failures+=1
+            elif not isinstance(protocol,dict) or protocol.get('status')!='success':protocol_incomplete+=1
+    return {'started':len(starts),'completed':done,'cliFailures':failures,'pending':len(starts)-done,
+            'actualStarted':len(list(directory.glob('*.running'))),'earlyConsumerClosed':early,'invalidFinalEvents':invalid,'protocolFailures':protocol_failures,'protocolIncomplete':protocol_incomplete}
+
+def _acquire_native_slot(directory, workers):
+    # Locks are scoped to this run and held until the REAL CLI exits, not until
+    # its disposable donor-facing proxy is deliberately killed on early output.
+    while True:
+        for index in range(workers):
+            fd=os.open(Path(directory)/('slot-'+str(index)),os.O_RDWR|os.O_CREAT|os.O_NOFOLLOW,0o600)
+            try:
+                fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
+                return fd
+            except BlockingIOError:
+                os.close(fd)
+        time.sleep(.025)
+
+class _ProtocolEvidence:
+    """Bounded stream evidence, not an OS-exit-code surrogate or behavior grader."""
+    def __init__(self,format_):
+        self.format=format_;self.buffer=b'';self.discarding=False
+        self.malformed=0;self.errors=0;self.successes=0;self.nonempty=False
+        self.max_line=4*1024*1024
+    def _line(self,line):
+        if not line.strip():return
+        if len(line)>self.max_line:
+            self.malformed+=1;return
+        try:
+            def invalid_constant(_):raise ValueError('non-json-number')
+            event=json.loads(line,parse_constant=invalid_constant)
+        except (ValueError,UnicodeError,RecursionError):
+            self.malformed+=1;return
+        if not isinstance(event,dict) or not isinstance(event.get('type'),str):
+            self.malformed+=1;return
+        if event['type']=='error':self.errors+=1
+        if event['type']=='result':
+            if event.get('is_error') is True or str(event.get('subtype','')).startswith('error') or event.get('errors'):
+                self.errors+=1
+            elif event.get('subtype')=='success' and event.get('is_error') is False:
+                self.successes+=1
+    def feed(self,raw):
+        self.nonempty=self.nonempty or bool(raw.strip())
+        if self.format!='stream-json':return
+        if self.discarding:
+            boundary=raw.find(b'\n')
+            if boundary<0:return
+            raw=raw[boundary+1:];self.discarding=False
+        self.buffer+=raw
+        while b'\n' in self.buffer:
+            line,_,self.buffer=self.buffer.partition(b'\n');self._line(line)
+        if len(self.buffer)>self.max_line:
+            self.malformed+=1;self.buffer=b'';self.discarding=True
+    def finish(self):
+        if self.format=='stream-json':
+            if self.buffer.strip():self._line(self.buffer)
+            self.buffer=b''
+            state='error' if self.errors or self.malformed or self.successes>1 else 'success' if self.successes==1 else 'incomplete'
+        elif self.format=='text':state='success' if self.nonempty else 'incomplete'
+        else:state='incomplete'
+        return {'format':self.format,'status':state,'terminalSuccesses':self.successes,
+                'errorEvents':self.errors,'malformedEvents':self.malformed,'nonemptyText':self.nonempty}
+
+def _supervise(real, trace):
+    token = uuid.uuid4().hex
+    start, done = Path(trace)/(token+'.start'), Path(trace)/(token+'.done')
+    _event(start,{})
+    result = {'returncode':None,'consumerClosedEarly':False}
+    protocol=_ProtocolEvidence(_option(sys.argv[4:],'--output-format') or 'text')
+    slot = None
+    try:
+        slot = _acquire_native_slot(os.environ['CROW_CREATOR_SLOTS'],int(os.environ['CROW_CREATOR_WORKERS']))
+        _event(Path(trace)/(token+'.running'),{})
+        process = subprocess.Popen([real,*sys.argv[4:]], stdout=subprocess.PIPE)
+        forwarding = True
+        while True:
+            raw = os.read(process.stdout.fileno(),65536)
+            if not raw:
+                break
+            protocol.feed(raw)
+            if forwarding:
+                try:
+                    offset = 0
+                    while offset < len(raw):
+                        offset += os.write(1,raw[offset:])
+                except BrokenPipeError:
+                    forwarding = False  # Native donor intentionally stops early.
+                    result['consumerClosedEarly'] = True
+        result['returncode'] = process.wait()
+    except (OSError, KeyboardInterrupt):
+        result['error'] = 'cli-start-or-supervisor-failure'
+    finally:
+        result['protocol']=protocol.finish()
+        _event(done,result)
+        if slot is not None:
+            os.close(slot)
+    return result['returncode'] if result['returncode'] is not None else 125
+
+def _proxy():
+    real, trace = os.environ['CROW_CREATOR_REAL_CLAUDE'], os.environ['CROW_CREATOR_TRACE']
+    # Supervisor survives donor's deliberate kill of this proxy; caller owns group.
+    process = subprocess.Popen([sys.executable, str(Path(__file__).absolute()), '__supervise',real,trace,*sys.argv[2:]])
+    return process.wait()
+
+def run(script, runtime_root, workspace, timeout, arguments=(), stdio=False, interactive=False):
+    if script not in SCRIPTS or type(timeout) not in (int,float) or not math.isfinite(timeout) or timeout <= 0:
+        raise CreatorError('invalid-script-or-timeout')
+    root = _private_directory(runtime_root)
+    if status(root)['status'] != 'ready':
+        raise CreatorError('runtime-not-ready-explicit-setup-required')
+    work = _private_directory(workspace, create=True)
+    if work == Path.home() or work == root or root in work.parents or work in root.parents:
+        raise CreatorError('workspace-must-be-separate')
+    for forbidden in (Path.home()/'.claude',Path.home()/'.codex',Path.home()/'.agents'):
+        if work == forbidden or forbidden in work.parents:
+            raise CreatorError('native-host-workspace-forbidden')
+    local = work/'.claude'
+    _private_directory(local, create=True)
+    commands=_private_directory(local/'commands', create=True)
+    if any(p.name!='commands' for p in local.iterdir()) or any(commands.iterdir()):
+        raise CreatorError('workspace-claude-must-be-unconfigured-and-empty')
+    arguments = _arguments(script,arguments)
+    _guard_io(script,arguments,work)
+    if script in ('run_eval','run_loop','improve_description'):
+        _guard_skill(arguments,work)
+    if script == 'generate_review' and not any(a in ('--help','-h') for a in arguments):
+        static_viewer = _option(arguments,'--static') or _option(arguments,'-s')
+        if not static_viewer and not interactive:
+            raise CreatorError('viewer-requires-static-or-explicit-interactive')
+    run_dir = Path(tempfile.mkdtemp(prefix='.creator-run-',dir=work))
+    run_dir.chmod(0o700)
+    trace = run_dir/'trace'; trace.mkdir(mode=0o700)
+    slots = run_dir/'slots'; slots.mkdir(mode=0o700)
+    try:
+        workers = int(_option(arguments,'--num-workers') or '1')
+    except ValueError:
+        raise CreatorError('num-workers-must-be-positive-integer') from None
+    if workers < 1:
+        raise CreatorError('num-workers-must-be-positive-integer')
+    bin_dir = run_dir/'bin'; bin_dir.mkdir(mode=0o700)
+    env = os.environ.copy()
+    env.pop('PYTHONHOME',None)
+    env['PYTHONPATH'] = str(root/'skill-creator')
+    env['PYTHONDONTWRITEBYTECODE'] = '1'
+    is_native = script in ('run_eval','run_loop','improve_description') and '--help' not in arguments and '-h' not in arguments
+    if is_native:
+        real = shutil.which('claude')
+        if not real:
+            raise CreatorError('claude-cli-not-found')
+        real = str(Path(real).resolve())
+        if env.get('CROW_CREATOR_PROXY') == '1' or real == str(bin_dir/'claude'):
+            raise CreatorError('recursive-claude-proxy')
+        shim = '#!'+str(root/'venv/bin/python')+'\nimport runpy,sys\nsys.argv=['+repr(str(Path(__file__).absolute()))+",'__proxy',*sys.argv[1:]]\nrunpy.run_path(sys.argv[0],run_name='__main__')\n"
+        _write(bin_dir/'claude',shim.encode(),0o700)
+        env.update(PATH=str(bin_dir)+os.pathsep+env.get('PATH',''), CROW_CREATOR_PROXY='1',
+                   CROW_CREATOR_REAL_CLAUDE=real,CROW_CREATOR_TRACE=str(trace),
+                   CROW_CREATOR_SLOTS=str(slots),CROW_CREATOR_WORKERS=str(workers))
+    target = root/'skill-creator'/SCRIPTS[script]
+    command = [str(root/'venv/bin/python'),str(target),*arguments]
+    if script == 'aggregate_benchmark':
+        command=[str(root/'venv/bin/python'),str(Path(__file__).absolute()),'__aggregate',str(target),*arguments]
+    if script == 'generate_review':
+        command=[str(root/'venv/bin/python'),str(Path(__file__).absolute()),'__viewer',str(target),*arguments]
+    stdout_path, stderr_path = run_dir/'stdout.log', run_dir/'stderr.log'
+    started = time.monotonic(); state = 'completed'; process = None
+    cancellation=None;selector=None
+    old_umask = os.umask(0o077)
+    try:
+        with _cancellation_scope():
+            try:
+                with os.fdopen(os.open(stdout_path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600),'wb') as out, \
+                     os.fdopen(os.open(stderr_path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600),'wb') as err:
+                    process=_spawn_owned(command,cwd=work,env=env,stdin=None if (stdio or interactive) else subprocess.DEVNULL,
+                        stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+                    selector = selectors.DefaultSelector()
+                    for pipe,stream,fd in ((process.stdout,out,1),(process.stderr,err,2)):
+                        selector.register(pipe,selectors.EVENT_READ,(stream,fd))
+                    while True:
+                        if time.monotonic()-started >= timeout:
+                            state = 'timeout'; break
+                        for key,_ in selector.select(.05):
+                            raw = os.read(key.fileobj.fileno(),65536)
+                            if not raw:
+                                selector.unregister(key.fileobj);continue
+                            stream,fd = key.data;stream.write(raw);stream.flush()
+                            if stdio or interactive:
+                                try:os.write(fd,raw)
+                                except BrokenPipeError:pass
+                        if process.poll() is not None and not _traces(trace)['pending'] and not selector.get_map():
+                            break
+            except KeyboardInterrupt as interruption:
+                state='interrupted'
+                cancellation=interruption.signum if isinstance(interruption,_Cancellation) else signal.SIGINT
+            finally:
+                if process is not None:
+                    try:_kill_group(process)
+                    except _Cancellation as late:
+                        state='interrupted';cancellation=cancellation or late.signum
+                    finally:
+                        process.stdout.close();process.stderr.close()
+                if selector is not None:selector.close()
+    finally:
+        os.umask(old_umask)
+    traces = _traces(trace)
+    code = process.returncode if process is not None else None
+    if state == 'completed':
+        if code or traces['cliFailures'] or traces.get('protocolFailures',0):state='operational-failure'
+        elif traces['pending'] or traces.get('protocolIncomplete',0) or (is_native and traces['started']==0):state='incomplete-cli-evidence'
+    result = {'schema':SCHEMA,'status':state,'exitCode':128+cancellation if cancellation else 0 if state=='completed' else 1,
+        'timedOut':state=='timeout','cancellationSignal':signal.Signals(cancellation).name if cancellation else None,
+        'executionKind':'help-only' if '--help' in arguments or '-h' in arguments else ('native-Claude' if is_native else 'offline-utility'),
+        'upstreamExitCode':code,'nativeHost':'Claude Code CLI' if is_native else 'offline-upstream-utility',
+        'integrationAdapter':'truthful-benchmark/1' if script=='aggregate_benchmark' else None,
+        'effectiveNativeWorkers':workers,'script':script,'stdoutPath':str(stdout_path),'stderrPath':str(stderr_path),'trace':traces,
+        'elapsedSeconds':round(time.monotonic()-started,6),
+        'limitations':['Donor output is not certified when CLI evidence fails or is incomplete.',
+            'Native Claude trigger behavior is not Codex host equivalence.','Not a sandbox; explicit upstream arguments retain upstream effects.',
+            'Viewer never kills an existing port occupant; process group is cleaned on completion/timeout/interrupt.']}
+    _write(run_dir/'result.json',json.dumps(result).encode())
+    return result
+
+def main(argv=None):
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if argv and argv[0] == '__viewer':
+        return _viewer_entry(argv[1],argv[2:])
+    if argv and argv[0] == '__aggregate':
+        return _aggregate_entry(argv[1],argv[2:])
+    if argv and argv[0] == '__proxy':
+        return _proxy()
+    if argv and argv[0] == '__supervise':
+        return _supervise(argv[1],argv[2])
+    donor = []
+    if '--' in argv:
+        index = argv.index('--');donor=argv[index+1:];argv=argv[:index]
+    parser = argparse.ArgumentParser(description=__doc__)
+    subs = parser.add_subparsers(dest='action',required=True)
+    for name in ('setup','status','run'):
+        sub = subs.add_parser(name)
+        sub.add_argument('--runtime-root',required=True)
+        if name=='setup':sub.add_argument('--yes',action='store_true')
+        if name=='run':
+            sub.add_argument('script',choices=sorted(SCRIPTS))
+            sub.add_argument('--workspace',required=True)
+            sub.add_argument('--timeout',type=float,required=True)
+            sub.add_argument('--stdio',action='store_true',help='Forward stdin and tee stdout/stderr; logs remain private.')
+            sub.add_argument('--interactive',action='store_true',help='Explicit viewer/browser/server operation within the caller timeout.')
+    args = parser.parse_args(argv)
+    try:
+        with _cancellation_scope():
+            if args.action=='setup':result=setup(args.runtime_root,args.yes)
+            elif args.action=='status':result=status(args.runtime_root)
+            else:result=run(args.script,args.runtime_root,args.workspace,args.timeout,donor,args.stdio,args.interactive)
+    except _Cancellation as cancellation:
+        result={'schema':SCHEMA,'status':'interrupted','exitCode':128+cancellation.signum,
+                'cancellationSignal':signal.Signals(cancellation.signum).name,'timedOut':False}
+    except (CreatorError,OSError,ValueError,TypeError,KeyError):
+        result={'schema':SCHEMA,'status':'operational-error','exitCode':2,'error':'unsafe-unavailable-or-incomplete-runtime-input'}
+    print(json.dumps(result,sort_keys=True,ensure_ascii=True))
+    return result['exitCode']
+
+if __name__=='__main__':
+    sys.exit(main())