@tangle-network/starter-foundry 0.7.2 → 0.13.0

package/registry/families/agent-research-harness-ts/files/README.md

@@ -0,0 +1,155 @@
+# research-harness
+
+Hypothesis-driven research harness on top of `@tangle-network/agent-eval@^0.77.0`.
+
+## What this bundle is
+
+A Node-only TypeScript bundle that implements `/research`-style
+hypothesis-driven optimization. The eval-harness layers (scenarios,
+judge-rubric, regression) supply the measurement substrate; this
+bundle drives the optimizer that runs variants against it.
+
+It composes the upstream agent-eval primitives:
+
+- `runOptimization` — default path for optimizing full variable-length
+  agent trajectories through the campaign driver API.
+- `runImprovementLoop` — promotion shell that re-scores against holdout
+  scenarios and returns the release gate decision.
+- `PairwiseSteeringOptimizer` — N scored steering variants → ranked
+  winner.
+- `runProposeReview` — propose / verify / review inner loop for the
+  hypothesis proposer.
+- `paretoFrontier` + `paretoFrontierWithCrowding` — multi-objective
+  filtering across (quality, cost, latency).
+- `bootstrapCi` + `cohensD` — statistical sign-off in the validator
+  pass.
+- **`analyzeRuns`** (agent-eval contract bridge, exposed here as
+  `analyzeSweep`) — converts captured `RunRecord[]` into the decision
+  report: score distributions, paired lift, cost/quality frontier, and
+  ranked recommendations. Call after the optimizer emits real records;
+  idempotent and read-only with respect to the sweep result.
+
+## Workflow
+
+```
+hypotheses/queue.json
+        │
+        ▼
+   pnpm research:screen        (1 rep / hypothesis, ranks by delta)
+        │
+        ▼  passedFloor[]
+   pnpm research:validate      (5 reps / candidate, bootstrap-CI gate)
+        │
+        ▼
+research-results/<runId>/scorecard.json
+```
+
+Or all at once: `pnpm research:sweep`.
+
+## Subcommands
+
+| Command                | What it does                                              |
+| ---------------------- | --------------------------------------------------------- |
+| `pnpm research:propose`| LLM-assisted hypothesis drafting (requires LLM hook)      |
+| `pnpm research:screen` | 1-rep cheap pass over every hypothesis in `queue.json`    |
+| `pnpm research:validate` | 5-rep statistical gate over candidates                  |
+| `pnpm research:sweep`  | Screen → validate end-to-end, write scorecard             |
+
+## Anti-overfitting (built in)
+
+These are not suggestions:
+
+- Screener requires ≥ 3 scenarios. Single-scenario "wins" are
+  memorisation, not improvement.
+- Validator runs the FULL scenario set, not the screener subset. The
+  held-out check is the gate.
+- Pareto frontier surfaces non-dominated variants on (quality, cost,
+  latency). The harness never collapses three axes into one without
+  the operator opting in.
+- Cost + wall-time tracked per hypothesis. A 2pp pass-rate win that
+  doubles cost is surfaced, not hidden.
+- Promotion is human, not automatic. The harness emits a
+  recommendation; the operator merges the change.
+
+## Wiring a real `ScenarioRunner`
+
+The CLI ships a `noOpRunner` that throws on first call — by design.
+Wire your own runner (one that talks to your eval harness) and call
+`runScreen` / `runValidate` / `runSweep` programmatically:
+
+```ts
+import { runSweep } from './src/research/runner.js'
+import type { ScenarioRunner } from './src/research/types.js'
+
+const runner: ScenarioRunner = {
+  scenarioIds: ['math-1', 'math-2', 'math-3', 'tools-a', 'tools-b'],
+  runTrial: async ({ hypothesis, scenarioId, rep }) => {
+    // Apply hypothesis.treatment to the agent, run the scenario via your
+    // eval-harness, return the score. For agents, prefer wrapping this
+    // same execution as a MultiShotRun so the optimizer can consume the
+    // full trajectory and ASI.
+    const out = await myEvalHarness.run({ scenarioId, treatment: hypothesis?.treatment })
+    return {
+      scenarioId,
+      score: out.judgeMean,
+      costUsd: out.costUsd,
+      wallSeconds: out.wallSeconds,
+      ok: out.pass,
+    }
+  },
+}
+
+await runSweep({
+  queuePath: 'hypotheses/queue.json',
+  resultsDir: 'research-results',
+  runner,
+})
+```
+
+## Closing the loop into run analysis
+
+After a `runMultiShotTrajectoryOptimization` / `runEvolution` sweep emits
+real `RunRecord[]`, call `analyzeSweep` to emit the canonical decision packet:
+
+```ts
+import { runMultiShotTrajectoryOptimization } from './src/eval/auto-research/loop.js'
+import { analyzeSweep } from './src/research/runner.js'
+
+const optimized = await runMultiShotTrajectoryOptimization(config)
+
+const rl = await analyzeSweep({
+  runs,
+  baselineCandidateId: 'baseline',
+  candidateCandidateId: optimized.winnerSurfaceHash,
+  split: 'holdout',
+})
+
+// rl.recommendations → ranked launch / hold / investigate guidance
+```
+
+The report is the evidence-backed handoff from search to launch decisions.
+Do not synthesize `RunRecord[]` from aggregate scores; analysis must start
+from real captured runs.
+
+## Tests
+
+`pnpm test` runs the harness's own regression tests against synthetic
+runners. The tests cover: queue load, proposer shape verification,
+screener ranking + floor, validator promote / reject / inconclusive
+verdicts, sweep persistence.
+
+## CI
+
+`.github/workflows/research.yml` runs the screener nightly. Manual
+dispatch supports `screen`, `validate`, or `sweep` modes. Results
+upload as a workflow artifact retained 30 days.
+
+## Where this fits
+
+- `agent-eval-harness-ts` measures.
+- `agent-research-harness-ts` (this bundle) optimizes.
+- `auto-research` (capability layer) wraps the agent-eval primitives.
+
+Keep the boundary clean: don't add the loop INSIDE the eval-harness.
+Measurement and optimization rot at different rates and live in
+different families on purpose.

package/registry/families/agent-research-harness-ts/files/agent.json

@@ -0,0 +1,31 @@
+{
+  "name": "research-harness",
+  "description": "Hypothesis-driven research harness — reads hypotheses/queue.json, runs screener + validator passes on top of @tangle-network/agent-eval optimization primitives, writes results to research-results/.",
+  "version": "0.1.0",
+  "tags": [
+    "research-harness",
+    "agent-eval",
+    "optimization-loop",
+    "hypothesis-loop"
+  ],
+  "prompt": {
+    "systemPromptFile": "AGENTS.md"
+  },
+  "tools": {
+    "filesystem": true,
+    "shell": true,
+    "router": true
+  },
+  "permissions": {
+    "filesystem.write": "ask",
+    "shell.exec": "ask",
+    "network": "deny"
+  },
+  "resources": {
+    "files": [
+      { "source": "hypotheses/" },
+      { "source": "README.md" },
+      { "source": "src/research/" }
+    ]
+  }
+}

package/registry/families/agent-research-harness-ts/files/hypotheses/README.md

@@ -0,0 +1,79 @@
+# Hypothesis Schema
+
+Each entry in `queue.json` is a `Hypothesis`:
+
+```ts
+interface Hypothesis {
+  id: string                  // kebab-case, unique
+  name: string                // short human label
+  rationale: string           // WHY + supporting evidence
+  category:
+    | 'bug-fix'
+    | 'architectural'
+    | 'efficiency'
+    | 'parameter-tuning'
+  expected_impact: string     // metric: direction (e.g. "pass rate: +5-10pp")
+  risk?: string               // what could regress
+  priority: 1 | 2 | 3         // 1 = ship-blocker, 3 = nice-to-have
+  treatment: Record<string, unknown>
+                              // free-form payload — your runner interprets it
+}
+```
+
+Mirrors the contract documented in `~/.claude/skills/research/SKILL.md`.
+
+## Categories (priority order)
+
+1. **`bug-fix`** — failures that should be passes. Highest ROI, always
+   tested first.
+2. **`architectural`** — new capabilities, better abstractions, smarter
+   strategies. Higher durability than tuning.
+3. **`efficiency`** — same quality, less cost / latency / tokens.
+4. **`parameter-tuning`** — config knob adjustments. Lowest priority,
+   most likely to overfit.
+
+## Anti-patterns
+
+The runner does not enforce these directly, but the operator should
+reject queue entries that violate them:
+
+- Hypothesis tied to a single scenario id ("only fixes case X") →
+  memorisation, not improvement.
+- Hypothesis without a `rationale` linking to evidence → vibes-driven
+  experimentation, no learning loop.
+- Hypothesis with `expected_impact: "make it better"` → not measurable,
+  not gateable.
+- `parameter-tuning` priority 1 → almost always wrong; bug fixes and
+  architectural changes outrank tuning.
+
+## Workflow
+
+1. Append entries to `queue.json`.
+2. Run `pnpm research:screen` (cheap 1-rep pass, ranks by delta).
+3. Run `pnpm research:validate` on the screener winners (5-rep
+   bootstrap-CI gate).
+4. Operator reads `research-results/<runId>/scorecard.json`, decides
+   what to promote. The harness emits a recommendation; promotion is a
+   human call.
+
+## Treatment payload
+
+The runner is treatment-agnostic. `treatment` is forwarded verbatim to
+your `ScenarioRunner.runTrial`. Common shapes:
+
+```jsonc
+// Prompt delta
+{ "kind": "system-prompt-delta", "append": "...", "promptVersion": "v2" }
+
+// Steering bundle id (consumer-side resolved)
+{ "kind": "steering-bundle", "bundleId": "v3-aggressive" }
+
+// Config knob
+{ "kind": "config", "delta": { "temperature": 0.2, "maxTokens": 2048 } }
+
+// Code patch reference
+{ "kind": "patch", "branch": "feat/parallel-tools", "sha": "abc123" }
+```
+
+Whatever shape you use, document it once in this file so the proposer
+emits compatible drafts.

package/registry/families/agent-research-harness-ts/files/hypotheses/example.json

@@ -0,0 +1,18 @@
+{
+  "hypotheses": [
+    {
+      "id": "tool-batching-architectural",
+      "name": "Batch parallel tool calls in a single turn",
+      "rationale": "Trace analysis shows ~30% of multi-step scenarios issue 3+ independent tool calls serially. Batching them into one turn cuts wall-time without changing the agent's reasoning surface. Anthropic-published evidence: parallel tool calls reduce latency 2-3x with no quality regression on independent calls.",
+      "category": "architectural",
+      "expected_impact": "wallSeconds: -25-40%, costUsd: ~unchanged, quality: ~unchanged or +0.5pp",
+      "risk": "If the agent batches calls with hidden dependencies (search-then-fetch where the URL comes from the search) the batch fails and the retry costs more than serial. Validator must include scenarios with dependent tool sequences.",
+      "priority": 1,
+      "treatment": {
+        "kind": "system-prompt-delta",
+        "append": "When you need to call multiple INDEPENDENT tools in a turn, issue them in parallel. Only serialize when the next call depends on the previous result.",
+        "promptVersion": "v2-parallel-tools"
+      }
+    }
+  ]
+}

package/registry/families/agent-research-harness-ts/files/hypotheses/queue.json

@@ -0,0 +1,3 @@
+{
+  "hypotheses": []
+}

package/registry/families/agent-research-harness-ts/files/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "my-research-harness",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "description": "Hypothesis-driven research harness on @tangle-network/agent-eval",
+  "bin": {
+    "research-harness": "./dist/research/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "research": "tsx src/research/cli.ts",
+    "research:propose": "tsx src/research/cli.ts propose",
+    "research:screen": "tsx src/research/cli.ts screen",
+    "research:validate": "tsx src/research/cli.ts validate",
+    "research:sweep": "tsx src/research/cli.ts sweep",
+    "test": "node --test --test-concurrency=1 tests/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@tangle-network/agent-eval": "^0.23.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.4.0"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "pnpm": {
+    "minimumReleaseAge": 4320,
+    "minimumReleaseAgeExclude": []
+  }
+}

package/registry/families/agent-research-harness-ts/files/src/research/cli.ts

@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+/**
+ * research-harness CLI — `pnpm research <subcommand>`.
+ *
+ * Subcommands:
+ *   propose   — draft hypotheses via runProposeReview (reads --goal, prints JSON)
+ *   screen    — 1-rep screener pass over hypotheses/queue.json
+ *   validate  — multi-rep validator over passed-floor candidates
+ *   sweep     — screen + validate end-to-end, write scorecard
+ *
+ * The CLI is intentionally thin. All real work lives in runner.ts +
+ * screener.ts + validator.ts + proposer.ts so callers can compose them
+ * directly without going through a shell.
+ *
+ * The harness ships with a NoOpScenarioRunner that throws with a clear
+ * message — a real consumer must inject a real runner. We refuse to ship
+ * a "default" runner that fakes success (muffled-gate pattern).
+ */
+
+import { resolve } from 'node:path'
+import process from 'node:process'
+
+import { runScreen, runSweep, runValidate } from './runner.js'
+import { propose } from './proposer.js'
+import type { ScenarioRunner } from './types.js'
+
+interface Argv {
+  command: string
+  flags: Map<string, string>
+  positional: string[]
+}
+
+function parseArgv(args: readonly string[]): Argv {
+  const command = args[0] ?? ''
+  const flags = new Map<string, string>()
+  const positional: string[] = []
+  for (let i = 1; i < args.length; i += 1) {
+    const arg = args[i]
+    if (arg === undefined) continue
+    if (arg.startsWith('--')) {
+      const eqIdx = arg.indexOf('=')
+      if (eqIdx >= 0) {
+        flags.set(arg.slice(2, eqIdx), arg.slice(eqIdx + 1))
+      } else {
+        const next = args[i + 1]
+        if (next !== undefined && !next.startsWith('--')) {
+          flags.set(arg.slice(2), next)
+          i += 1
+        } else {
+          flags.set(arg.slice(2), 'true')
+        }
+      }
+    } else {
+      positional.push(arg)
+    }
+  }
+  return { command, flags, positional }
+}
+
+function fail(message: string): never {
+  process.stderr.write(`research-harness: ${message}\n`)
+  process.exit(1)
+}
+
+function noOpRunner(): ScenarioRunner {
+  return {
+    scenarioIds: [],
+    runTrial: async () => {
+      throw new Error(
+        'research-harness CLI invoked without an injected ScenarioRunner. ' +
+          'Either compose the runner programmatically (import runScreen / runSweep) or wire ' +
+          'a custom CLI that supplies a real runner. The default refuses to fake results.',
+      )
+    },
+  }
+}
+
+async function main(): Promise<void> {
+  const argv = parseArgv(process.argv.slice(2))
+  const queuePath = resolve(argv.flags.get('queue') ?? 'hypotheses/queue.json')
+  const resultsDir = resolve(
+    argv.flags.get('results') ?? process.env.RESEARCH_RESULTS_DIR ?? 'research-results',
+  )
+
+  switch (argv.command) {
+    case 'propose': {
+      const goal = argv.flags.get('goal')
+      if (!goal) fail('propose requires --goal "<description>"')
+      const count = Number.parseInt(argv.flags.get('count') ?? '5', 10)
+      // Default: refuse to run without a wired LLM. This keeps the bundle
+      // honest — the consumer has to inject `callJson` for real proposals.
+      const out = await propose({
+        goal,
+        count,
+        callJson: async () => {
+          throw new Error(
+            'propose: no LLM hook wired. Compose `propose()` programmatically with a callJson ' +
+              'that hits router.tangle.tools (or any compatible JSON endpoint).',
+          )
+        },
+      })
+      process.stdout.write(`${JSON.stringify(out.hypotheses, null, 2)}\n`)
+      return
+    }
+    case 'screen': {
+      const { report, path } = await runScreen({
+        queuePath,
+        resultsDir,
+        runner: noOpRunner(),
+      })
+      process.stdout.write(`screened ${report.ranked.length} → ${path}\n`)
+      return
+    }
+    case 'validate': {
+      const candidatesArg = argv.flags.get('candidates')
+      if (!candidatesArg) fail('validate requires --candidates "id1,id2,..."')
+      const candidateIds = candidatesArg.split(',').map((s) => s.trim()).filter(Boolean)
+      const { report, path } = await runValidate({
+        queuePath,
+        resultsDir,
+        runner: noOpRunner(),
+        candidateIds,
+      })
+      process.stdout.write(`validated ${report.results.length} → ${path}\n`)
+      return
+    }
+    case 'sweep': {
+      const out = await runSweep({
+        queuePath,
+        resultsDir,
+        runner: noOpRunner(),
+      })
+      process.stdout.write(
+        `sweep ${out.runId}: queue=${out.queueSize} screened=${out.screen.ranked.length} ` +
+          `validated=${out.validate?.results.length ?? 0} scorecard=${out.scorecardPath}\n`,
+      )
+      return
+    }
+    case '':
+    case 'help':
+    case '--help':
+    case '-h': {
+      process.stdout.write(
+        'usage: research-harness <propose|screen|validate|sweep> [flags]\n' +
+          '  propose  --goal "<text>" [--count N]\n' +
+          '  screen   [--queue PATH] [--results DIR]\n' +
+          '  validate --candidates id1,id2,...  [--queue PATH] [--results DIR]\n' +
+          '  sweep    [--queue PATH] [--results DIR]\n',
+      )
+      return
+    }
+    default:
+      fail(`unknown command "${argv.command}". Try --help.`)
+  }
+}
+
+main().catch((err) => {
+  process.stderr.write(`${err instanceof Error ? err.stack ?? err.message : String(err)}\n`)
+  process.exit(1)
+})