@tangle-network/starter-foundry 0.7.2 → 0.13.0

package/registry/families/agent-research-harness-py/files/src/research/runner.py

@@ -0,0 +1,230 @@
+"""Runner — orchestrates the propose / screen / validate loop.
+
+Reads `hypotheses/queue.json`, dispatches sandbox runs, persists
+results to `research-results/<hypothesis-id>/<run-id>.json`, and
+returns structured verdicts.
+
+The runner is intentionally I/O-thin: most logic lives in the
+purpose-specific modules (`screener`, `validator`, `sandbox_runner`,
+`proposer`). The runner wires them together and handles persistence.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import hashlib
+import json
+import os
+import platform
+import sys
+import uuid
+from collections import defaultdict
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from . import __version__
+from .sandbox_runner import SandboxRunConfig, run_hypothesis_rep
+from .screener import ScreeningRanking, run_screen
+from .types import Hypothesis, HypothesisResult, ValidationVerdict
+from .validator import run_validate
+
+DEFAULT_QUEUE_PATH = Path("hypotheses/queue.json")
+DEFAULT_RESULTS_DIR = Path("research-results")
+DEFAULT_TOP_K = 3
+DEFAULT_VALIDATE_REPS = 5
+
+
+def _scipy_version() -> str:
+    try:
+        import scipy  # noqa: PLC0415
+
+        return scipy.__version__
+    except Exception:
+        return "unknown"
+
+
+def load_queue(path: Path = DEFAULT_QUEUE_PATH) -> list[Hypothesis]:
+    """Load + validate the hypothesis queue.
+
+    Malformed entries cause the whole load to fail — we'd rather
+    raise loudly than silently drop a hypothesis the operator
+    expected to run.
+    """
+    if not path.exists():
+        return []
+    raw = json.loads(path.read_text())
+    if not isinstance(raw, list):
+        raise ValueError(f"{path}: queue root must be a JSON array")
+    out: list[Hypothesis] = []
+    for i, entry in enumerate(raw):
+        try:
+            out.append(Hypothesis.model_validate(entry))
+        except ValidationError as e:
+            raise ValueError(f"{path} entry [{i}] invalid: {e}") from e
+    return out
+
+
+def save_queue(queue: list[Hypothesis], path: Path = DEFAULT_QUEUE_PATH) -> None:
+    """Atomic-replace write of the queue."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(
+        json.dumps(
+            [h.model_dump(by_alias=True) for h in queue],
+            indent=2,
+            sort_keys=False,
+        )
+        + "\n"
+    )
+    os.replace(tmp, path)
+
+
+def _hypothesis_hash(h: Hypothesis) -> str:
+    payload = json.dumps(h.model_dump(by_alias=True), sort_keys=True).encode()
+    return hashlib.sha256(payload).hexdigest()[:16]
+
+
+def _provenance(h: Hypothesis) -> dict[str, str]:
+    return {
+        "harness_version": __version__,
+        "scipy_version": _scipy_version(),
+        "python_version": sys.version.split()[0],
+        "platform": platform.platform(),
+        "sandbox_image": h.sandbox_image,
+        "hypothesis_hash": _hypothesis_hash(h),
+        "timestamp": dt.datetime.now(dt.timezone.utc).isoformat(),
+    }
+
+
+def _persist(
+    results_dir: Path,
+    hypothesis: Hypothesis,
+    phase: str,
+    payload: dict[str, object],
+) -> Path:
+    out_dir = results_dir / hypothesis.id
+    out_dir.mkdir(parents=True, exist_ok=True)
+    run_id = f"{phase}-{uuid.uuid4().hex[:12]}"
+    path = out_dir / f"{run_id}.json"
+    body = {"provenance": _provenance(hypothesis), **payload}
+    tmp = path.with_suffix(".json.tmp")
+    tmp.write_text(json.dumps(body, indent=2) + "\n")
+    os.replace(tmp, path)
+    return path
+
+
+def _collect_reps(
+    hypotheses: list[Hypothesis],
+    n_reps: int,
+    config: SandboxRunConfig | None,
+    *,
+    client_factory: object | None = None,
+) -> dict[str, list[HypothesisResult]]:
+    """Run `n_reps` reps of every hypothesis. One sandbox per rep."""
+    out: dict[str, list[HypothesisResult]] = defaultdict(list)
+    for h in hypotheses:
+        for rep in range(n_reps):
+            results = run_hypothesis_rep(
+                h, rep, config, client_factory=client_factory  # type: ignore[arg-type]
+            )
+            out[h.id].extend(results)
+    return out
+
+
+def screen(
+    queue_path: Path = DEFAULT_QUEUE_PATH,
+    results_dir: Path = DEFAULT_RESULTS_DIR,
+    config: SandboxRunConfig | None = None,
+    *,
+    client_factory: object | None = None,
+) -> ScreeningRanking:
+    """1 rep per hypothesis. Persist per-hypothesis run + ranking."""
+    queue = load_queue(queue_path)
+    if not queue:
+        return ScreeningRanking(rows=[])
+    results = _collect_reps(queue, 1, config, client_factory=client_factory)
+    ranking = run_screen(results)
+    for h in queue:
+        _persist(
+            results_dir,
+            h,
+            "screen",
+            {
+                "phase": "screen",
+                "results": [r.model_dump() for r in results.get(h.id, [])],
+            },
+        )
+    (results_dir / "screen-ranking.json").write_text(
+        json.dumps(ranking.model_dump(), indent=2) + "\n"
+    )
+    return ranking
+
+
+def validate(
+    queue_path: Path = DEFAULT_QUEUE_PATH,
+    results_dir: Path = DEFAULT_RESULTS_DIR,
+    n_reps: int = DEFAULT_VALIDATE_REPS,
+    top_k: int = DEFAULT_TOP_K,
+    config: SandboxRunConfig | None = None,
+    *,
+    client_factory: object | None = None,
+    winners: list[str] | None = None,
+) -> list[ValidationVerdict]:
+    """Multi-rep validation on screening winners.
+
+    If `winners` is provided, validate exactly that subset. Otherwise
+    re-run a fresh screen and take the top_k.
+    """
+    queue = load_queue(queue_path)
+    if not queue:
+        return []
+    if winners is None:
+        ranking = screen(queue_path, results_dir, config, client_factory=client_factory)
+        winners = ranking.top(top_k)
+    short = [h for h in queue if h.id in winners]
+    if not short:
+        return []
+    results = _collect_reps(short, n_reps, config, client_factory=client_factory)
+    verdicts = run_validate(results)
+    by_id = {v.hypothesis_id: v for v in verdicts}
+    for h in short:
+        v = by_id.get(h.id)
+        _persist(
+            results_dir,
+            h,
+            "validate",
+            {
+                "phase": "validate",
+                "n_reps": n_reps,
+                "results": [r.model_dump() for r in results.get(h.id, [])],
+                "verdict": v.model_dump() if v else None,
+            },
+        )
+    (results_dir / "validate-verdicts.json").write_text(
+        json.dumps([v.model_dump() for v in verdicts], indent=2) + "\n"
+    )
+    return verdicts
+
+
+def sweep(
+    queue_path: Path = DEFAULT_QUEUE_PATH,
+    results_dir: Path = DEFAULT_RESULTS_DIR,
+    top_k: int = DEFAULT_TOP_K,
+    n_reps: int = DEFAULT_VALIDATE_REPS,
+    config: SandboxRunConfig | None = None,
+    *,
+    client_factory: object | None = None,
+) -> list[ValidationVerdict]:
+    """Convenience: screen then validate the top_k."""
+    ranking = screen(queue_path, results_dir, config, client_factory=client_factory)
+    winners = ranking.top(top_k)
+    return validate(
+        queue_path,
+        results_dir,
+        n_reps=n_reps,
+        top_k=top_k,
+        config=config,
+        client_factory=client_factory,
+        winners=winners,
+    )

package/registry/families/agent-research-harness-py/files/src/research/sandbox_runner.py

@@ -0,0 +1,180 @@
+"""Sandbox runner — wraps `tangle_sandbox` to execute one (hypothesis,
+arm) pair in an isolated sandbox.
+
+Lifecycle for each rep:
+
+    1. create sandbox from `hypothesis.sandbox_image`
+    2. wait_for("running")
+    3. write treatment (or baseline) files via box.write
+    4. apply env vars
+    5. for each scenario: exec `scenario.command`, parse the metric
+       value out of stdout's last JSON line
+    6. delete sandbox
+
+If `tangle-sandbox` is unreachable, this raises — we never silently
+fall back to host-side execution. That's the harness's whole point.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from .types import Hypothesis, HypothesisResult, Scenario
+
+
+class SandboxClientLike(Protocol):
+    """Subset of `tangle_sandbox.Sandbox` we depend on.
+
+    Defined as a Protocol so tests can mock it without importing the
+    real client (which requires an API key and network).
+    """
+
+    def create(self, **kwargs: Any) -> Any: ...
+
+
+@dataclass
+class SandboxRunConfig:
+    """Per-run knobs."""
+
+    api_key: str
+    base_url: str = "https://api.tangle.tools"
+    timeout_seconds: int = 300
+
+
+def _extract_metric(stdout: str, metric: str) -> float | None:
+    """Pull a numeric metric out of the scenario's stdout.
+
+    Convention: the scenario command prints a single JSON line as its
+    last output line. We search backwards until we find one. If the
+    JSON has the named metric and it's numeric, return float(value).
+    Otherwise None.
+    """
+    for line in reversed(stdout.strip().splitlines()):
+        line = line.strip()
+        if not line.startswith("{") or not line.endswith("}"):
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if metric in obj and isinstance(obj[metric], (int, float)):
+            return float(obj[metric])
+    return None
+
+
+def _build_client(config: SandboxRunConfig) -> SandboxClientLike:
+    """Lazy import so the module can be imported in tests without the
+    SDK installed."""
+    from tangle_sandbox import Sandbox  # type: ignore[import-not-found]
+
+    return Sandbox(api_key=config.api_key, base_url=config.base_url)
+
+
+def _apply_files(box: Any, files: dict[str, str]) -> None:
+    for path, content in files.items():
+        box.write(path, content)
+
+
+def _run_one_arm(
+    box: Any,
+    arm_files: dict[str, str],
+    arm_env: dict[str, str],
+    scenarios: list[Scenario],
+    metrics: list[str],
+) -> dict[tuple[str, str], float | None]:
+    """Apply files + env, run every scenario, collect every metric.
+
+    Returns a {(scenario_id, metric): value-or-None} map.
+    """
+    _apply_files(box, arm_files)
+    out: dict[tuple[str, str], float | None] = {}
+    for sc in scenarios:
+        cmd = " ".join(sc.command)
+        result = box.exec(cmd, env=arm_env) if arm_env else box.exec(cmd)
+        if getattr(result, "exit_code", 0) != 0:
+            for metric in metrics:
+                out[(sc.id, metric)] = None
+            continue
+        for metric in metrics:
+            out[(sc.id, metric)] = _extract_metric(result.stdout, metric)
+    return out
+
+
+def run_hypothesis_rep(
+    hypothesis: Hypothesis,
+    rep_index: int,
+    config: SandboxRunConfig | None = None,
+    *,
+    client_factory: Any = None,
+) -> list[HypothesisResult]:
+    """Execute one rep of a single hypothesis (treatment + baseline).
+
+    Each rep stands up exactly one sandbox, applies the treatment,
+    runs all scenarios, applies the baseline (clobbering treatment
+    files), runs all scenarios, and tears down. This keeps env-cost
+    constant in n_scenarios while still isolating reps.
+
+    `client_factory` is a test seam — if provided, it must return a
+    sandbox-like client matching `tangle_sandbox.Sandbox`'s contract.
+    In production, omit it and the real client is constructed.
+    """
+    cfg = config or SandboxRunConfig(
+        api_key=os.environ.get("TANGLE_SANDBOX_KEY", ""),
+        base_url=os.environ.get("TANGLE_SANDBOX_URL", "https://api.tangle.tools"),
+    )
+    if not cfg.api_key and client_factory is None:
+        raise RuntimeError(
+            "TANGLE_SANDBOX_KEY is required (or pass client_factory in tests)"
+        )
+
+    client = client_factory(cfg) if client_factory else _build_client(cfg)
+    box = client.create(image=hypothesis.sandbox_image)
+    try:
+        box.wait_for("running")
+        treatment = _run_one_arm(
+            box,
+            hypothesis.treatment.files,
+            hypothesis.treatment.env,
+            hypothesis.scenarios,
+            hypothesis.metrics,
+        )
+        baseline = _run_one_arm(
+            box,
+            hypothesis.baseline.files,
+            hypothesis.baseline.env,
+            hypothesis.scenarios,
+            hypothesis.metrics,
+        )
+    finally:
+        try:
+            box.delete()
+        except Exception:
+            # Surface delete failures via stderr but don't shadow a
+            # real exception that's already in flight. The sandbox
+            # service has its own GC; a leaked sandbox won't poison
+            # other reps.
+            pass
+
+    out: list[HypothesisResult] = []
+    for sc in hypothesis.scenarios:
+        for metric in hypothesis.metrics:
+            t_val = treatment.get((sc.id, metric))
+            b_val = baseline.get((sc.id, metric))
+            err: str | None = None
+            if t_val is None and b_val is None:
+                err = "both-arms-missing-metric"
+            out.append(
+                HypothesisResult(
+                    hypothesis_id=hypothesis.id,
+                    scenario_id=sc.id,
+                    metric=metric,
+                    treatment_value=t_val,
+                    baseline_value=b_val,
+                    error=err,
+                    rep_index=rep_index,
+                )
+            )
+    return out

package/registry/families/agent-research-harness-py/files/src/research/screener.py

@@ -0,0 +1,55 @@
+"""Screener — 1 rep per hypothesis, rank by mean delta.
+
+Cheap-pass filter that decides which hypotheses are worth full
+multi-rep validation. Output is a deterministic ranking sorted by
+mean-delta descending. Errors don't drop a hypothesis from the
+ranking — they're surfaced via `n_errors` so the operator can see
+which entries were partially observed.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from .types import HypothesisResult, ScreeningRow
+
+
+class ScreeningRanking(BaseModel):
+    rows: list[ScreeningRow] = Field(default_factory=list)
+
+    def top(self, k: int) -> list[str]:
+        """IDs of the top-k hypotheses by mean delta (descending)."""
+        return [r.hypothesis_id for r in self.rows[:k] if r.n_errors < r.n_scenarios]
+
+
+def _row_for(hid: str, results: list[HypothesisResult]) -> ScreeningRow:
+    deltas: list[float] = []
+    n_errors = 0
+    for r in results:
+        if r.error is not None or r.treatment_value is None or r.baseline_value is None:
+            n_errors += 1
+            continue
+        deltas.append(r.treatment_value - r.baseline_value)
+    n_scenarios = len(results)
+    mean_delta = sum(deltas) / len(deltas) if deltas else float("-inf")
+    return ScreeningRow(
+        hypothesis_id=hid,
+        mean_delta=mean_delta,
+        n_scenarios=n_scenarios,
+        n_errors=n_errors,
+    )
+
+
+def run_screen(
+    results_by_hypothesis: dict[str, list[HypothesisResult]],
+) -> ScreeningRanking:
+    """Rank hypotheses by mean delta across their scenarios.
+
+    Hypotheses where every scenario errored are sorted to the bottom
+    (mean_delta = -inf) so they don't accidentally win a tie-break.
+    """
+    rows = [_row_for(hid, results) for hid, results in results_by_hypothesis.items()]
+    rows.sort(key=lambda r: (r.mean_delta, r.hypothesis_id), reverse=True)
+    # Stable secondary sort by id so the ranking is deterministic.
+    rows.sort(key=lambda r: r.mean_delta, reverse=True)
+    return ScreeningRanking(rows=rows)

package/registry/families/agent-research-harness-py/files/src/research/types.py

@@ -0,0 +1,128 @@
+"""Pydantic v2 models for the research-harness.
+
+These mirror the schema in the TS sibling family
+(`agent-research-harness-ts`). Keep them in sync — the JSON files in
+`hypotheses/` are consumed by both languages.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Scenario(BaseModel):
+    """A single scenario the hypothesis is evaluated against.
+
+    The harness is metric-agnostic — `metric` names a numeric field in
+    the scenario's emitted result JSON. Higher-is-better is assumed
+    unless `direction` is explicitly `lower-is-better`.
+    """
+
+    id: str = Field(min_length=1)
+    description: str = Field(min_length=1)
+    command: list[str] = Field(min_length=1)
+    direction: Literal["higher-is-better", "lower-is-better"] = "higher-is-better"
+
+
+class Treatment(BaseModel):
+    """The change under test."""
+
+    description: str = Field(min_length=1)
+    files: dict[str, str] = Field(default_factory=dict)
+    env: dict[str, str] = Field(default_factory=dict)
+
+
+class Baseline(BaseModel):
+    """The control."""
+
+    description: str = Field(min_length=1)
+    files: dict[str, str] = Field(default_factory=dict)
+    env: dict[str, str] = Field(default_factory=dict)
+
+
+class Hypothesis(BaseModel):
+    """A single hypothesis in the research queue.
+
+    Cross-language portability contract: this model's JSON shape is
+    identical to the Zod schema in `agent-research-harness-ts`.
+    """
+
+    id: str = Field(min_length=1, pattern=r"^[a-z][a-z0-9-]*$")
+    description: str = Field(min_length=20)
+    sandbox_image: str = Field(default="ubuntu", alias="sandboxImage")
+    treatment: Treatment
+    baseline: Baseline
+    scenarios: list[Scenario] = Field(min_length=1)
+    metrics: list[str] = Field(min_length=1)
+    tags: list[str] = Field(default_factory=list)
+
+    model_config = {"populate_by_name": True}
+
+    @field_validator("metrics")
+    @classmethod
+    def _metrics_unique(cls, v: list[str]) -> list[str]:
+        if len(set(v)) != len(v):
+            raise ValueError("metrics must be unique")
+        return v
+
+
+class HypothesisResult(BaseModel):
+    """Per-rep run result for a single (hypothesis, scenario) pair.
+
+    `treatment_value` and `baseline_value` are the numeric metric
+    readings extracted from the sandbox run. `error` is set if the rep
+    failed; downstream code must check `error` before averaging.
+    """
+
+    hypothesis_id: str
+    scenario_id: str
+    metric: str
+    treatment_value: float | None = None
+    baseline_value: float | None = None
+    error: str | None = None
+    rep_index: int = 0
+
+
+class ScreeningRow(BaseModel):
+    """One row of the screener's ranking table."""
+
+    hypothesis_id: str
+    mean_delta: float
+    n_scenarios: int
+    n_errors: int
+
+
+class ValidationVerdict(BaseModel):
+    """Per-hypothesis validator output.
+
+    `p_value` is the raw two-sided Welch p; `q_value` is the
+    Benjamini–Hochberg-adjusted q across the hypothesis family in this
+    run. Verdicts use `q_value` so the family-wise false-promote rate is
+    bounded at the configured FDR. Both fields ship so consumers can
+    audit raw vs corrected significance.
+    """
+
+    hypothesis_id: str
+    n_reps: int
+    mean_treatment: float
+    mean_baseline: float
+    mean_diff: float
+    ci_lower: float
+    ci_upper: float
+    cohens_d: float
+    p_value: float | None = None
+    q_value: float | None = None
+    n_errors: int
+    verdict: Literal["winner", "neutral", "loser", "insufficient-data"]
+
+
+class ProposerConfig(BaseModel):
+    """Configuration for the LLM-driven proposer."""
+
+    base_url: str = "https://router.tangle.tools/v1"
+    model: str = "anthropic/claude-sonnet-4"
+    n_proposals: int = 3
+    temperature: float = 0.7
+    max_tokens: int = 4096