@tangle-network/starter-foundry 0.7.2 → 0.13.0

package/registry/families/agent-research-harness-py/files/tests/test_validator.py

@@ -0,0 +1,250 @@
+"""Validator math — bootstrap CI + Cohen's d against known values.
+
+Catches:
+    - Cohen's d formula drift (sample vs. population variance)
+    - bootstrap seed not pinned (non-deterministic CIs across runs)
+    - verdict ladder thresholds drifting from the documented contract
+"""
+
+from __future__ import annotations
+
+import math
+
+import pytest
+
+from research.types import HypothesisResult
+from research.validator import (
+    benjamini_hochberg,
+    bootstrap_mean_diff_ci,
+    cohens_d,
+    run_validate,
+    summarize,
+    welch_p_value,
+)
+
+
+# --- Cohen's d -----------------------------------------------------
+
+
+def test_cohens_d_pooled_variance_known_value() -> None:
+    """t=[10,12,14,11,13] (mean 12, var 2.5), b=[8,9,10,11,12] (mean 10, var 2.5)
+    pooled var = 2.5 -> s_pooled = sqrt(2.5)
+    d = (12 - 10) / sqrt(2.5) = 1.2649110640673518
+    """
+    d = cohens_d([10, 12, 14, 11, 13], [8, 9, 10, 11, 12])
+    assert math.isclose(d, 2.0 / math.sqrt(2.5), rel_tol=1e-12)
+    assert math.isclose(d, 1.2649110640673518, rel_tol=1e-12)
+
+
+def test_cohens_d_zero_for_identical_samples() -> None:
+    d = cohens_d([5, 6, 7, 8, 9], [5, 6, 7, 8, 9])
+    assert d == 0.0
+
+
+def test_cohens_d_negative_when_baseline_wins() -> None:
+    d = cohens_d([8, 9, 10, 11, 12], [10, 12, 14, 11, 13])
+    assert d < 0
+
+
+def test_cohens_d_zero_for_constant_samples_avoids_div_by_zero() -> None:
+    d = cohens_d([5, 5, 5, 5, 5], [5, 5, 5, 5, 5])
+    assert d == 0.0
+
+
+def test_cohens_d_requires_n_at_least_2() -> None:
+    with pytest.raises(ValueError):
+        cohens_d([5.0], [3.0, 4.0])
+
+
+# --- bootstrap CI --------------------------------------------------
+
+
+def test_bootstrap_ci_is_deterministic_with_pinned_seed() -> None:
+    t = [10.0, 12.0, 14.0, 11.0, 13.0, 12.5, 11.5]
+    b = [8.0, 9.0, 10.0, 11.0, 12.0, 10.5, 9.5]
+    a = bootstrap_mean_diff_ci(t, b)
+    c = bootstrap_mean_diff_ci(t, b)
+    assert a == c, f"bootstrap is not deterministic: {a} vs {c}"
+
+
+def test_bootstrap_ci_brackets_observed_mean_diff_for_clear_signal() -> None:
+    """Strong positive signal — the CI must be entirely above zero."""
+    t = [10.0, 12.0, 14.0, 11.0, 13.0, 12.5, 11.5]
+    b = [2.0, 3.0, 4.0, 1.0, 2.0, 3.5, 2.5]
+    lo, hi = bootstrap_mean_diff_ci(t, b)
+    assert lo > 0, f"expected lo>0 for clear winner, got ({lo}, {hi})"
+    observed = sum(t) / len(t) - sum(b) / len(b)
+    assert lo <= observed <= hi
+
+
+def test_bootstrap_ci_straddles_zero_for_no_signal() -> None:
+    """Same distribution -> CI must include zero."""
+    same = [10.0, 11.0, 9.0, 10.5, 9.5, 10.0, 11.0]
+    lo, hi = bootstrap_mean_diff_ci(same, same)
+    assert lo <= 0 <= hi
+
+
+# --- Verdict ladder ------------------------------------------------
+
+
+def _result(hid: str, t: float | None, b: float | None, rep: int) -> HypothesisResult:
+    return HypothesisResult(
+        hypothesis_id=hid,
+        scenario_id="s1",
+        metric="qps",
+        treatment_value=t,
+        baseline_value=b,
+        rep_index=rep,
+    )
+
+
+def test_verdict_winner_for_strong_positive_signal() -> None:
+    results = [
+        _result("h", 20.0 + i * 0.1, 5.0 + i * 0.1, i)
+        for i in range(5)
+    ]
+    v = summarize("h", results)
+    assert v.verdict == "winner"
+    assert v.cohens_d > 0.2
+    assert v.ci_lower > 0
+
+
+def test_verdict_loser_for_strong_negative_signal() -> None:
+    results = [
+        _result("h", 5.0 + i * 0.1, 20.0 + i * 0.1, i)
+        for i in range(5)
+    ]
+    v = summarize("h", results)
+    assert v.verdict == "loser"
+    assert v.cohens_d < -0.2
+    assert v.ci_upper < 0
+
+
+def test_verdict_neutral_for_no_signal() -> None:
+    same = [10.0, 11.0, 9.0, 10.5, 9.5]
+    results = [
+        HypothesisResult(
+            hypothesis_id="h",
+            scenario_id="s1",
+            metric="qps",
+            treatment_value=same[i],
+            baseline_value=same[i],
+            rep_index=i,
+        )
+        for i in range(5)
+    ]
+    v = summarize("h", results)
+    assert v.verdict == "neutral"
+
+
+def test_benjamini_hochberg_known_sequence() -> None:
+    """Hand-validated BH: p=[0.01, 0.04, 0.03, 0.20, 0.50] sorted [0.01,0.03,0.04,0.20,0.50]
+    raw q sorted = [0.05, 0.075, 0.0667, 0.25, 0.50]
+    step-up min trailing = [0.05, 0.0667, 0.0667, 0.25, 0.50]
+    Re-scattered to original order: [0.05, 0.0667, 0.0667, 0.25, 0.50]
+    """
+    p = [0.01, 0.04, 0.03, 0.20, 0.50]
+    q = benjamini_hochberg(p)
+    assert math.isclose(q[0], 0.05, abs_tol=1e-9)
+    assert math.isclose(q[1], 1 / 15.0, abs_tol=1e-9)  # 0.06666...
+    assert math.isclose(q[2], 1 / 15.0, abs_tol=1e-9)
+    assert math.isclose(q[3], 0.25, abs_tol=1e-9)
+    assert math.isclose(q[4], 0.50, abs_tol=1e-9)
+    # Monotone after sort: q must be non-decreasing in p-rank.
+    sorted_q = sorted(q)
+    for a, b in zip(sorted_q, sorted_q[1:], strict=False):
+        assert a <= b
+
+
+def test_benjamini_hochberg_caps_false_promotions_under_null() -> None:
+    """20 hypotheses uniformly under H0 → expected ≤ 1 q < 0.05."""
+    rng = __import__("random").Random(42)
+    p_values = [rng.random() for _ in range(20)]
+    q = benjamini_hochberg(p_values, fdr=0.05)
+    promoted_raw = sum(1 for x in p_values if x < 0.05)
+    promoted_bh = sum(1 for x in q if x < 0.05)
+    assert promoted_bh <= promoted_raw
+    # Under uniform-null with N=20, raw catches ~1 false positive and BH
+    # tightens or matches it. Exact counts are RNG-dependent; the
+    # invariant is "BH never promotes more than raw."
+
+
+def test_benjamini_hochberg_empty_input() -> None:
+    assert benjamini_hochberg([]) == []
+
+
+def test_run_validate_applies_bh_family_wise() -> None:
+    """Multi-hypothesis run: a single weak winner that passed raw-CI but
+    has a borderline p must be downgraded by BH when the family includes
+    other tests.
+    """
+    # Strong winner: t mean way above b. Should still be 'winner' post-BH.
+    strong = [
+        HypothesisResult(
+            hypothesis_id="strong",
+            scenario_id="s1",
+            metric="qps",
+            treatment_value=20.0 + i * 0.1,
+            baseline_value=5.0 + i * 0.1,
+            rep_index=i,
+        )
+        for i in range(5)
+    ]
+    # Null hypothesis 1: identical samples → p ≈ 1, q ≈ 1 → neutral.
+    same1 = [
+        HypothesisResult(
+            hypothesis_id="null1",
+            scenario_id="s1",
+            metric="qps",
+            treatment_value=10.0,
+            baseline_value=10.0,
+            rep_index=i,
+        )
+        for i in range(5)
+    ]
+    # Null hypothesis 2.
+    same2 = [
+        HypothesisResult(
+            hypothesis_id="null2",
+            scenario_id="s1",
+            metric="qps",
+            treatment_value=12.0,
+            baseline_value=12.0,
+            rep_index=i,
+        )
+        for i in range(5)
+    ]
+    verdicts = run_validate({"strong": strong, "null1": same1, "null2": same2})
+    by_id = {v.hypothesis_id: v for v in verdicts}
+    assert by_id["strong"].verdict == "winner"
+    assert by_id["strong"].q_value is not None
+    assert by_id["strong"].q_value < 0.05
+    assert by_id["null1"].verdict == "neutral"
+    assert by_id["null2"].verdict == "neutral"
+    # p_value must be carried through alongside q_value.
+    for v in verdicts:
+        assert v.p_value is not None
+
+
+def test_welch_p_value_returns_one_for_insufficient_data() -> None:
+    assert welch_p_value([1.0], [2.0, 3.0]) == 1.0
+    assert welch_p_value([], [1.0, 2.0]) == 1.0
+
+
+def test_verdict_insufficient_data_when_all_reps_error() -> None:
+    results = [
+        HypothesisResult(
+            hypothesis_id="h",
+            scenario_id="s1",
+            metric="qps",
+            treatment_value=None,
+            baseline_value=None,
+            error="boom",
+            rep_index=i,
+        )
+        for i in range(5)
+    ]
+    v = summarize("h", results)
+    assert v.verdict == "insufficient-data"
+    assert v.n_errors == 5
+    assert v.n_reps == 0

package/registry/families/agent-research-harness-py/manifest.json

@@ -0,0 +1,199 @@
+{
+  "id": "agent-research-harness-py",
+  "description": "Python research harness — propose hypotheses, screen cheaply (1 rep), validate winners (5 rep) with bootstrap CI + Cohen's d. Each hypothesis runs in an isolated tangle-sandbox so treatments cannot contaminate each other. Statistical math (scipy.stats) mirrors the TS sibling family byte-for-byte; verdicts are stable across languages.",
+  "tags": [
+    "agent-runtime",
+    "research-harness",
+    "experimentation",
+    "tangle-sandbox",
+    "scipy",
+    "python"
+  ],
+  "taxonomy": {
+    "language": "python",
+    "runtime": "python",
+    "surface": "research-runner"
+  },
+  "defaults": {
+    "projectType": "agent-research-harness-py",
+    "agentName": "research-harness",
+    "outboundDomains": [
+      "router.tangle.tools",
+      "api.tangle.tools"
+    ],
+    "secrets": [
+      "TANGLE_API_KEY",
+      "TANGLE_SANDBOX_KEY"
+    ]
+  },
+  "files": [
+    {
+      "source": "files/agent.json",
+      "target": "agent.json"
+    },
+    {
+      "source": "files/AGENTS.md",
+      "target": "AGENTS.md"
+    },
+    {
+      "source": "files/pyproject.toml",
+      "target": "pyproject.toml"
+    },
+    {
+      "source": "files/requirements.txt",
+      "target": "requirements.txt"
+    },
+    {
+      "source": "files/README.md",
+      "target": "README.md"
+    },
+    {
+      "source": "files/src/research/__init__.py",
+      "target": "src/research/__init__.py"
+    },
+    {
+      "source": "files/src/research/__main__.py",
+      "target": "src/research/__main__.py"
+    },
+    {
+      "source": "files/src/research/cli.py",
+      "target": "src/research/cli.py"
+    },
+    {
+      "source": "files/src/research/runner.py",
+      "target": "src/research/runner.py"
+    },
+    {
+      "source": "files/src/research/proposer.py",
+      "target": "src/research/proposer.py"
+    },
+    {
+      "source": "files/src/research/screener.py",
+      "target": "src/research/screener.py"
+    },
+    {
+      "source": "files/src/research/validator.py",
+      "target": "src/research/validator.py"
+    },
+    {
+      "source": "files/src/research/sandbox_runner.py",
+      "target": "src/research/sandbox_runner.py"
+    },
+    {
+      "source": "files/hypotheses/queue.json",
+      "target": "hypotheses/queue.json"
+    },
+    {
+      "source": "files/hypotheses/example.json",
+      "target": "hypotheses/example.json"
+    },
+    {
+      "source": "files/hypotheses/README.md",
+      "target": "hypotheses/README.md"
+    },
+    {
+      "source": "files/tests/__init__.py",
+      "target": "tests/__init__.py"
+    },
+    {
+      "source": "files/tests/test_screener.py",
+      "target": "tests/test_screener.py"
+    },
+    {
+      "source": "files/tests/test_validator.py",
+      "target": "tests/test_validator.py"
+    },
+    {
+      "source": "files/tests/test_sandbox.py",
+      "target": "tests/test_sandbox.py"
+    },
+    {
+      "source": "files/.github/workflows/research.yml",
+      "target": ".github/workflows/research.yml"
+    }
+  ],
+  "validationChecks": [
+    {
+      "type": "file-exists",
+      "path": "AGENTS.md"
+    },
+    {
+      "type": "file-exists",
+      "path": "pyproject.toml"
+    },
+    {
+      "type": "file-exists",
+      "path": "src/research/__main__.py"
+    },
+    {
+      "type": "agents-md-valid",
+      "path": "AGENTS.md"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/research/runner.py"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/research/validator.py"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/research/screener.py"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/research/sandbox_runner.py"
+    }
+  ],
+  "contextHints": {
+    "commands": [
+      "python -m research propose",
+      "python -m research screen",
+      "python -m research validate",
+      "python -m research sweep"
+    ],
+    "entrypoints": [
+      "src/research/__main__.py",
+      "AGENTS.md"
+    ],
+    "extensionPoints": [
+      "hypotheses/queue.json",
+      "research-results/"
+    ]
+  },
+  "keywords": [
+    "research-harness",
+    "experimentation",
+    "hypothesis-testing",
+    "ab-testing",
+    "tangle-sandbox",
+    "scipy"
+  ],
+  "tieredKeywords": {
+    "tier1": [
+      "research-harness-py",
+      "python-research-loop",
+      "auto-research-python",
+      "python-experiment-runner",
+      "hypothesis-screener-python",
+      "scipy-research-harness"
+    ],
+    "tier2": [
+      "bootstrap-ci",
+      "cohen-d",
+      "hypothesis-queue",
+      "screen-validate-loop",
+      "isolated-experiment-sandbox"
+    ],
+    "tier3": [
+      "experimentation",
+      "ab-testing",
+      "research"
+    ],
+    "archetypes": [
+      "research-runner",
+      "experiment-loop"
+    ]
+  }
+}

package/registry/families/agent-research-harness-ts/files/.github/workflows/research.yml

@@ -0,0 +1,65 @@
+name: research-harness
+
+on:
+  schedule:
+    # Nightly screener pass over the queue. UTC.
+    - cron: '0 7 * * *'
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: 'screen | validate | sweep'
+        required: true
+        default: 'screen'
+        type: choice
+        options:
+          - screen
+          - validate
+          - sweep
+      candidates:
+        description: 'Comma-separated hypothesis ids (validate only)'
+        required: false
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  research:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v3
+        with:
+          version: 9
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run research-harness
+        env:
+          TANGLE_API_KEY: ${{ secrets.TANGLE_API_KEY }}
+          RESEARCH_RESULTS_DIR: research-results
+        run: |
+          MODE="${{ github.event.inputs.mode || 'screen' }}"
+          case "$MODE" in
+            screen)   pnpm research:screen ;;
+            validate) pnpm research:validate --candidates "${{ github.event.inputs.candidates }}" ;;
+            sweep)    pnpm research:sweep ;;
+            *) echo "unknown mode: $MODE" >&2; exit 1 ;;
+          esac
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: research-results-${{ github.run_id }}
+          path: research-results/
+          retention-days: 30

package/registry/families/agent-research-harness-ts/files/AGENTS.md

@@ -0,0 +1,121 @@
+---
+name: research-harness
+role: Hypothesis-driven research harness — runs screener + validator passes on a hypothesis queue, promotes winners through bootstrap-CI gates, never overfits to specific test cases
+domain: agent-evaluation
+allowedDomains:
+  - api.tangle.tools
+  - router.tangle.tools
+allowedEnv:
+  - TANGLE_API_KEY
+  - RESEARCH_RESULTS_DIR
+version: 0.1.0
+---
+
+## Role
+
+You operate the research harness for this agent / model / pipeline.
+Your job is not to "make the number go up". Your job is to discover
+durable improvements through hypothesis-driven experimentation, gate
+them with statistical rigor, and write down everything you learn.
+
+This bundle implements `/research`-style hypothesis-driven optimization
+on top of `@tangle-network/agent-eval@^0.77.0`. The eval-harness layers
+(scenarios, judge-rubric, regression) supply the measurement
+substrate; the auto-research layer wraps `PairwiseSteeringOptimizer` +
+`runOptimization` + `runImprovementLoop` + `runProposeReview` +
+`paretoFrontier`.
+
+Use `runOptimization` as the default optimization path for agent behavior.
+Use `runImprovementLoop` when you have a real held-out split and need the
+promotion gate decision. Analyze only captured `RunRecord[]`; do not invent
+records from aggregate scores.
+
+## Workflow
+
+1. **Add hypotheses** to `hypotheses/queue.json`. One JSON array of
+   `Hypothesis` objects. See `hypotheses/example.json` and
+   `hypotheses/README.md` for the schema.
+
+2. **Screen** — `pnpm research:screen`. Cheap pass: 1 rep of every
+   queued hypothesis against the baseline scenarios. Ranks by metric
+   delta. Output: `research-results/<runId>/screen.json`.
+
+3. **Validate** — `pnpm research:validate`. Winners-only pass: 5 reps
+   of each hypothesis whose screener delta cleared the floor. Applies
+   bootstrap-CI gates from agent-eval (`bootstrapCi`, `cohensD`).
+   Output: `research-results/<runId>/validate.json`.
+
+4. **Sweep** — `pnpm research:sweep`. Runs screen → validate end to
+   end and writes a single scorecard.
+
+5. **Propose** — `pnpm research:propose`. LLM-assisted: given prior
+   run traces, drafts new hypotheses via `runProposeReview`. Append
+   the proposals to the queue (you, not the harness — keeps human in
+   the loop on what gets tested).
+
+## Anti-overfitting (non-negotiable)
+
+These rules ship with the harness because the failure mode is
+silent: a hypothesis that "works" on the calibration scenarios but
+hurts the broader system is actively bad.
+
+- **Never tune to specific test cases.** If a treatment only helps
+  case X, it is memorisation, not improvement. Reject.
+- **Validate on held-out scenarios.** The validator must run a
+  superset of what the screener used — otherwise the gate is a
+  rubber stamp.
+- **Separate reach from reliability.** Reaching a new scenario is a
+  bug fix; getting cheaper on already-passing ones is optimisation.
+  Don't let one mask the other in the scorecard.
+- **Architectural over parameter-tuning.** A 30s → 60s timeout knob
+  is parameter tuning. A new tool-batching primitive is
+  architectural. Architectural wins are durable; tuning wins rot.
+- **Goodhart watch.** If the metric improves but the agent feels
+  worse, the metric is wrong. Fix the metric, not the agent.
+
+## Hypothesis categories (priority order)
+
+1. **Bug fixes** — failures that should be passes. Always first.
+2. **Architectural** — new capabilities, better abstractions.
+3. **Efficiency** — same quality, less cost / latency / tokens.
+4. **Parameter tuning** — config knob adjustments. Lowest priority.
+
+## What the harness does
+
+- Loads `hypotheses/queue.json`.
+- For each: applies `treatment` to the runtime, runs the scenarios,
+  collects `RunScore` from agent-eval, captures cost + duration.
+- Screener: 1 rep / hypothesis. Ranks by score delta. Cheap.
+- Validator: 5 reps / hypothesis above the screener floor. Applies
+  `bootstrapCi` + `cohensD` for statistical sign-off.
+- Pareto-filters the validated set across (quality, cost, latency)
+  via the auto-research layer's `frontier`. Surfaces non-dominated
+  variants — the operator decides which point to ship.
+
+## What the harness will NOT do
+
+- Promote a hypothesis automatically. Promotion is a human call.
+  The harness emits a recommendation; the operator merges the change.
+- Run experiments without a queued hypothesis. No "explore" mode by
+  design — every run traces back to a written rationale.
+- Mix screener and validator scenario sets. Held-out is held out.
+- Tune to a single scenario. The screener requires ≥3 scenarios to
+  rank.
+
+## Output blocks
+
+Wrap structured deliverables for downstream tooling:
+
+- `:::artifact research-result` — final scorecard (per-hypothesis
+  promote / reject / candidate / inconclusive).
+- `:::artifact pareto-frontier` — the non-dominated variant set.
+
+## Files
+
+- `hypotheses/queue.json` — pending hypotheses
+- `hypotheses/example.json` — schema reference
+- `src/research/runner.ts` — drives the loop
+- `src/research/proposer.ts` — LLM-assisted hypothesis drafting
+- `src/research/screener.ts` — cheap 1-rep ranker
+- `src/research/validator.ts` — 5-rep statistical gate
+- `src/research/cli.ts` — `pnpm research <subcommand>` entrypoint