@tangle-network/agent-bench 0.3.7 → 0.4.0

package/src/hev-improve.mts

@@ -1,8 +1,6 @@
 /**
- * SELF-IMPROVEMENT on HumanEval — the prompt-sensitive, VISIBLE-ORACLE counterpart
- * to the SWE-bench run. Same machinery (improve(surface:'prompt') + gepaProposer +
- * held-out gate), but the worker is a single chat completion and the judge is the
- * deterministic Docker checker (run the function against its own hidden unit tests).
+ * Official GEPA prompt optimization on HumanEval. The worker is a single chat
+ * completion and the judge is the deterministic Docker checker.
  *
  * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test
  * is withheld — the worker cannot verify, so prompt wording cannot move resolve.
@@ -13,11 +11,22 @@
  * Worker + reflect models call the zai coding endpoint directly (no tangle router,
  * no WAF, no 503):  TANGLE_API_KEY=$ZAI_API_KEY  ROUTER_BASE=https://api.z.ai/api/coding/paas/v4
  */
-import { improve } from '@tangle-network/agent-runtime'
-import type { AgentProfile } from '@tangle-network/agent-interface'
+import {
+  improve,
+  officialGepa,
+  type ReadonlyAgentProfile,
+} from '@tangle-network/agent-runtime'
+import {
+  canonicalCandidateDigest,
+  type AgentProfile,
+} from '@tangle-network/agent-interface'
 import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
-import { gepaProposer } from '@tangle-network/agent-eval/campaign'
 import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
+import {
+  assertCompleteCost,
+  officialOptimizerModel,
+  requiredTokenPricing,
+} from './official-optimizer-config.mjs'
 
 // The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's
 // solveInstruction so the baseline arm reproduces the plain-prompt denominator.
@@ -26,7 +35,6 @@ const SEED_INSTRUCTION =
 
 interface Completion {
   text: string
-  usd: number
   tokIn: number
   tokOut: number
 }
@@ -45,10 +53,7 @@ async function complete(base: string, key: string, model: string, prompt: string
   const text = d.choices?.[0]?.message?.content ?? ''
   const tokIn = d.usage?.prompt_tokens ?? 0
   const tokOut = d.usage?.completion_tokens ?? 0
-  // zai glm pricing is ~ $0.6/M in, $2.2/M out (coding plan); a rough cost tag so the
-  // stub-guard sees a real backend. Exact cost is not the metric (pass-rate is).
-  const usd = (tokIn * 0.6 + tokOut * 2.2) / 1_000_000
-  return { text, usd, tokIn, tokOut }
+  return { text, tokIn, tokOut }
 }
 
 async function main(): Promise<void> {
@@ -63,51 +68,81 @@ async function main(): Promise<void> {
   const reflectBase = process.env.REFLECT_BASE ?? base
   const reflectKey = process.env.REFLECT_KEY ?? key
   const trainN = Number(process.env.TRAIN_N ?? 12)
-  const holdoutN = Number(process.env.HOLDOUT_N ?? 12)
+  const selectionN = Number(process.env.SELECTION_N ?? 12)
+  const testN = Number(process.env.TEST_N ?? 12)
   const offset = Number(process.env.OFFSET ?? 80)
-  // generations=1 never exercises the GEPA Pareto/combine path (the frontier
-  // needs >=1 completed generation before combine can fire) — default to a
-  // multi-generation budget so the default run measures the full loop.
-  const generations = Number(process.env.GENERATIONS ?? 6)
-  const population = Number(process.env.POPULATION ?? 4)
+  const maxEvaluations = Number(process.env.MAX_EVALUATIONS ?? 24)
+  const maxProposerCostUsd = Number(process.env.MAX_PROPOSER_COST_USD ?? 5)
   const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000)
   const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000)
   const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4)
+  const runDir = process.env.RUN_DIR ?? '.runs/humaneval-official-gepa'
+  if (process.env.DRYRUN) {
+    console.log(
+      `DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa})`,
+    )
+    return
+  }
+  const workerPricing = requiredTokenPricing(process.env, 'WORKER')
+  const optimizer = officialOptimizerModel({
+    env: process.env,
+    model: reflectModel,
+    baseUrl: reflectBase,
+    apiKey: reflectKey,
+    maxCostUsd: maxProposerCostUsd,
+    maxOutputTokensPerRequest: reflectMaxTokens,
+  })
 
-  // TRAIN and HOLDOUT are DISJOINT slices of the harder middle band (offset).
+  // All three partitions are disjoint slices of the harder middle band.
   const train = await loadHumanEval(trainN, offset)
-  const holdout = await loadHumanEval(holdoutN, offset + trainN)
-  const byId = new Map<string, HumanEvalTask>([...train, ...holdout].map((t) => [t.taskId, t]))
-  const allIds = [...byId.keys()]
+  const selection = await loadHumanEval(selectionN, offset + trainN)
+  const testCases = await loadHumanEval(testN, offset + trainN + selectionN)
+  const byId = new Map<string, HumanEvalTask>(
+    [...train, ...selection, ...testCases].map((t) => [t.taskId, t]),
+  )
 
-  console.log('═══ HumanEval self-improvement — VISIBLE oracle (deterministic Docker checker) ═══')
-  console.log(`worker=${workerModel}  reflect=${reflectModel}  base=${base}`)
+  console.log('=== HumanEval prompt optimization with official GEPA ===')
+  console.log(`worker=${workerModel} reflect=${reflectModel} base=${base}`)
   console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`)
-  console.log(`holdout=[${holdout.map((t) => t.taskId).join(', ')}]`)
-  console.log(`generations=${generations} population=${population} offset=${offset} maxTokens=${workerMaxTokens}`)
-  console.log(`≈ ${trainN * (1 + generations * population) + 2 * holdoutN} cells (each = 1 completion + 1 Docker check)\n`)
+  console.log(`selection=[${selection.map((t) => t.taskId).join(', ')}]`)
+  console.log(`test=[${testCases.map((t) => t.taskId).join(', ')}]`)
+  console.log(`maxEvaluations=${maxEvaluations} maxProposerCostUsd=${maxProposerCostUsd} offset=${offset} maxTokens=${workerMaxTokens}`)
+  console.log(`runDir=${runDir}\n`)
 
   const stats = { n: 0 }
-  const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
-    const instr = String(surface)
+  const agent = async (candidate: ReadonlyAgentProfile, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
+    const instr = candidate.prompt?.systemPrompt
+    if (instr === undefined) throw new Error('agent: candidate profile has no system prompt')
     const t = byId.get(scenario.id)
     if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`)
     const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\``
     const t0 = Date.now()
-    const r = await complete(base, key, workerModel, prompt, workerMaxTokens)
-    const zeroUsage = r.tokIn === 0 && r.tokOut === 0
+    const paid = await ctx.cost.runPaidCall({
+      channel: 'agent',
+      actor: 'humaneval-worker',
+      model: workerModel,
+      execute: () => complete(base, key, workerModel, prompt, workerMaxTokens),
+      receipt: (result) => {
+        const usageUnknown = result.tokIn === 0 && result.tokOut === 0
+        return {
+          model: workerModel,
+          inputTokens: result.tokIn,
+          outputTokens: result.tokOut,
+          customTokenPricing: workerPricing,
+          ...(usageUnknown ? { usageUnknown: true } : {}),
+        }
+      },
+    })
+    if (!paid.succeeded) throw paid.error
+    const r = paid.value
     const hasText = r.text.trim().length > 0
-    ctx.cost.observe(zeroUsage && hasText ? Math.max(r.usd, 0.0001) : r.usd, workerModel)
-    ctx.cost.observeTokens(
-      zeroUsage && hasText ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut },
-    )
     stats.n += 1
     const codeLen = extractCode(r.text).length
     console.log(`  [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`)
     return hasText ? r.text : null
   }
 
-  const judge: JudgeConfig<string, Scenario> = {
+  const judge: JudgeConfig<string | null, Scenario> = {
     name: 'humaneval-docker',
     dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }],
     async score({ artifact, scenario }) {
@@ -137,49 +172,59 @@ async function main(): Promise<void> {
   }
 
   const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } }
-  const proposer = gepaProposer({
-    llm: { baseUrl: reflectBase, apiKey: reflectKey },
-    model: reflectModel,
-    target:
-      'the instruction/system prompt strategy for a SMALL model completing Python functions to pass hidden unit tests. ' +
-      'Propose SUBSTANTIALLY different strategies, not wording tweaks: e.g. require the model to first reason step-by-step ' +
-      'about the algorithm and edge cases (empty inputs, off-by-one, boundary values, types) in a brief plan or comments ' +
-      'BEFORE writing the code; provide a short worked example; or add an explicit self-check against the docstring. ' +
-      'Bold rewrites that change model BEHAVIOR beat cosmetic edits.',
-    maxTokens: reflectMaxTokens,
-    temperature: 0.7,
-  })
-
-  const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'humaneval' }))
-  const holdoutScenarios: Scenario[] = holdout.map((t) => ({ id: t.taskId, kind: 'humaneval' }))
+  const scenario = (task: HumanEvalTask): Scenario => ({ id: task.taskId, kind: 'humaneval' })
 
-  const out = await improve(profile, [], {
+  const out = await improve(profile, {
     surface: 'prompt',
-    gate: 'holdout',
-    generator: proposer,
-    scenarios,
-    judge,
+    executionRef: canonicalCandidateDigest({
+      callback: 'bench/hev-improve',
+      model: workerModel,
+      endpoint: new URL(base).origin,
+      maxTokens: workerMaxTokens,
+      checker: 'local-python',
+    }),
+    method: officialGepa<Scenario, string | null>({
+      objective:
+        'Improve the complete instruction for a small model that writes Python functions which pass hidden unit tests.',
+      background:
+        'Prefer behavioral strategies over wording changes. Address algorithm choice, edge cases, boundary values, type behavior, and self-checking. Return only the complete instruction.',
+      recipe: {
+        kind: 'engine',
+        run: {
+          engine: 'gepa',
+          maxEvaluations,
+          maxProposerCostUsd,
+        },
+      },
+      optimizer,
+      resume: 'if-compatible',
+      trustResumeState: true,
+      describeScenario: (item) => ({ prompt: byId.get(item.id)?.prompt ?? item.id }),
+    }),
+    trainScenarios: train.map(scenario),
+    selectionScenarios: selection.map(scenario),
+    testScenarios: testCases.map(scenario),
+    judges: [judge],
     agent,
     expectUsage: 'warn',
-    // rawTraceContext stays OFF deliberately: it swaps the distilled findings for
-    // filesystem paths + grep/cat instructions (rawTraceDistiller), which only a
-    // coding harness can execute. This run's proposer is prompt-tier (gepaProposer
-    // — a single LLM call that cannot run grep), so the trace evidence arrives via
-    // the judge's traceback notes + the breakdown's `emitted` excerpt instead.
-    budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 },
-    llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel },
+    maxConcurrency,
+    reps: 1,
+    runDir,
+    optimizationRunOptions: {
+      expectUsage: 'warn',
+      maxConcurrency,
+      reps: 1,
+    },
   })
 
-  console.log('\n═══ RESULT ═══')
-  console.log(`gateDecision=${out.gateDecision}  shipped=${out.shipped}  lift=${out.lift}`)
-  console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`)
-  console.log(`winner   holdout pass-rate = ${out.raw.winner.compositeMean}`)
-  console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
-  console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
-  if (out.raw.winner.label) console.log(`winner label   : ${out.raw.winner.label}`)
-  if ((out.raw.winner as { surface?: unknown }).surface) {
-    console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`)
-  }
+  assertCompleteCost('humaneval official GEPA run', out.cost)
+  console.log('\n=== RESULT ===')
+  console.log(`decision=${out.decision} lift=${out.lift} interval=[${out.liftInterval.low}, ${out.liftInterval.high}]`)
+  console.log(`baseline test pass-rate=${out.raw.best.baselineComposite}`)
+  console.log(`winner test pass-rate=${out.raw.best.winnerComposite}`)
+  console.log(`test scenarios=${JSON.stringify(out.raw.best.scenarioScores)}`)
+  console.log(`cost=${JSON.stringify(out.cost)}`)
+  console.log(`winner instruction:\n${String(out.candidate.value).slice(0, 2000)}`)
 }
 
 main().catch((e) => {

package/src/official-optimizer-config.mts

@@ -0,0 +1,89 @@
+function requiredNonNegativeNumber(
+  env: NodeJS.ProcessEnv,
+  name: string,
+): number {
+  const raw = env[name]
+  if (raw === undefined || raw.trim() === '') {
+    throw new Error(`env ${name} is required`)
+  }
+  const value = Number(raw)
+  if (!Number.isFinite(value) || value < 0) {
+    throw new Error(`env ${name} must be a finite non-negative number`)
+  }
+  return value
+}
+
+function positiveInteger(
+  env: NodeJS.ProcessEnv,
+  name: string,
+  fallback: number,
+): number {
+  const value = Number(env[name] ?? fallback)
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`env ${name} must be a positive integer`)
+  }
+  return value
+}
+
+export function requiredTokenPricing(
+  env: NodeJS.ProcessEnv,
+  prefix: string,
+) {
+  return {
+    inputUsdPerMillion: requiredNonNegativeNumber(
+      env,
+      `${prefix}_INPUT_USD_PER_MILLION`,
+    ),
+    cachedInputUsdPerMillion: requiredNonNegativeNumber(
+      env,
+      `${prefix}_CACHED_INPUT_USD_PER_MILLION`,
+    ),
+    cacheWriteUsdPerMillion: requiredNonNegativeNumber(
+      env,
+      `${prefix}_CACHE_WRITE_USD_PER_MILLION`,
+    ),
+    outputUsdPerMillion: requiredNonNegativeNumber(
+      env,
+      `${prefix}_OUTPUT_USD_PER_MILLION`,
+    ),
+  }
+}
+
+export function officialOptimizerModel(options: {
+  env: NodeJS.ProcessEnv
+  model: string
+  baseUrl: string
+  apiKey: string
+  maxCostUsd: number
+  maxOutputTokensPerRequest: number
+  envPrefix?: string
+}) {
+  const { env } = options
+  const envPrefix = options.envPrefix ?? 'REFLECT'
+  return {
+    model: options.model,
+    baseUrl: options.baseUrl,
+    apiKey: options.apiKey,
+    budget: {
+      maxCostUsd: options.maxCostUsd,
+      maxRequests: positiveInteger(env, `${envPrefix}_MAX_REQUESTS`, 100),
+      maxRequestBytes: positiveInteger(env, `${envPrefix}_MAX_REQUEST_BYTES`, 2_000_000),
+      maxResponseBytes: positiveInteger(env, `${envPrefix}_MAX_RESPONSE_BYTES`, 2_000_000),
+      maxOutputTokensPerRequest: options.maxOutputTokensPerRequest,
+      requestTimeoutMs: positiveInteger(env, `${envPrefix}_REQUEST_TIMEOUT_MS`, 300_000),
+      pricing: requiredTokenPricing(env, envPrefix),
+    },
+  }
+}
+
+export function assertCompleteCost(
+  label: string,
+  cost: { accountingComplete: boolean; incompleteReasons: readonly string[] },
+): void {
+  if (cost.accountingComplete) return
+  const reasons =
+    cost.incompleteReasons.length > 0
+      ? cost.incompleteReasons.join('; ')
+      : 'no incomplete reason was recorded'
+  throw new Error(`${label}: cost accounting is incomplete: ${reasons}`)
+}

package/src/official-optimizer-config.test.mts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from 'vitest'
+import {
+  assertCompleteCost,
+  officialOptimizerModel,
+  requiredTokenPricing,
+} from './official-optimizer-config.mts'
+
+const pricingEnv = {
+  OPT_INPUT_USD_PER_MILLION: '1',
+  OPT_CACHED_INPUT_USD_PER_MILLION: '0.1',
+  OPT_CACHE_WRITE_USD_PER_MILLION: '1.25',
+  OPT_OUTPUT_USD_PER_MILLION: '5',
+}
+
+describe('official optimizer configuration', () => {
+  it('builds a bounded model configuration from an arbitrary environment prefix', () => {
+    const model = officialOptimizerModel({
+      env: {
+        ...pricingEnv,
+        OPT_MAX_REQUESTS: '7',
+        OPT_MAX_REQUEST_BYTES: '1000',
+        OPT_MAX_RESPONSE_BYTES: '2000',
+        OPT_REQUEST_TIMEOUT_MS: '3000',
+      },
+      envPrefix: 'OPT',
+      model: 'test-model',
+      baseUrl: 'http://127.0.0.1:8080/v1',
+      apiKey: 'test-key',
+      maxCostUsd: 2,
+      maxOutputTokensPerRequest: 4000,
+    })
+
+    expect(model).toEqual({
+      model: 'test-model',
+      baseUrl: 'http://127.0.0.1:8080/v1',
+      apiKey: 'test-key',
+      budget: {
+        maxCostUsd: 2,
+        maxRequests: 7,
+        maxRequestBytes: 1000,
+        maxResponseBytes: 2000,
+        maxOutputTokensPerRequest: 4000,
+        requestTimeoutMs: 3000,
+        pricing: {
+          inputUsdPerMillion: 1,
+          cachedInputUsdPerMillion: 0.1,
+          cacheWriteUsdPerMillion: 1.25,
+          outputUsdPerMillion: 5,
+        },
+      },
+    })
+  })
+
+  it('requires every token price instead of inventing cost data', () => {
+    expect(() =>
+      requiredTokenPricing(
+        {
+          ...pricingEnv,
+          OPT_OUTPUT_USD_PER_MILLION: undefined,
+        },
+        'OPT',
+      ),
+    ).toThrow('env OPT_OUTPUT_USD_PER_MILLION is required')
+  })
+
+  it('rejects invalid request limits before constructing the optimizer', () => {
+    expect(() =>
+      officialOptimizerModel({
+        env: { ...pricingEnv, OPT_MAX_REQUESTS: '0' },
+        envPrefix: 'OPT',
+        model: 'test-model',
+        baseUrl: 'http://127.0.0.1:8080/v1',
+        apiKey: 'test-key',
+        maxCostUsd: 2,
+        maxOutputTokensPerRequest: 4000,
+      }),
+    ).toThrow('env OPT_MAX_REQUESTS must be a positive integer')
+  })
+
+  it('rejects incomplete cost records', () => {
+    expect(() =>
+      assertCompleteCost('official optimizer', {
+        accountingComplete: false,
+        incompleteReasons: ['provider omitted usage'],
+      }),
+    ).toThrow('official optimizer: cost accounting is incomplete: provider omitted usage')
+  })
+})

package/src/profiles.ts

@@ -5,8 +5,8 @@
  * "operator type" or "analyst type" — there are profiles + the operator toolbox they use to manage
  * each other (in-process via the Scope, in a sandbox via the same verbs exposed as MCP tools).
  *
- * The directory is OPTIMIZED over time by the agent-eval RSI loop (`runImprovementLoop`): `gepaProposer`
- * evolves the prompts, a `skillOptProposer` evolves `skills`, Pareto-tracked + holdout-gated.
+ * Complete agent-eval methods optimize these profiles over explicit train,
+ * selection, and final-test partitions.
  * The trace-analyst's findings are the optimizer's input — the loop improves the profile that the
  * findings say is weak.
  */

package/src/quant-arena/README.md

@@ -0,0 +1,144 @@
+# QUANT-ARENA — a self-improving trading-strategy lab
+
+A small, fully auditable research loop: an AI strategy author writes candidate strategies, every candidate is screened for look-ahead bias, backtested walk-forward on overlapping in-sample windows against pinned benchmarks, and judged by an acceptance rule whose bar **rises with every strategy tried**.
+Every attempt — accepted, rejected, or killed for leaking — becomes a permanent row in a lab notebook.
+The final two years of data are a locked out-of-sample set that only a separate certification command may touch, once.
+
+Everything is plain TypeScript you can read in an afternoon: the backtester is one file with zero dependencies, the acceptance math is one file, the data is committed CSV.
+
+## 1. Quickstart
+
+```bash
+# from bench/ (needs node >= 20, the `claude` CLI logged in, and `uv` on PATH
+# — scoring runs in a pinned python environment, see "Two engines" below)
+npx tsx src/quant-arena/quant-loop.mts --out /tmp/quant-demo --candidates 2
+```
+
+That runs a full research campaign: 2 strategy authors x 2 candidates each — the committed capture of exactly this command cost $1.65 of model spend across 8 metered calls; backtests are free.
+No API? `--skip-llm-audit` keeps everything but the adversarial code review; the mechanical look-ahead check still runs.
+
+Run the unit tests (backtester hand-computed cases, look-ahead detection, acceptance math, window reproducibility):
+
+```bash
+npx vitest run src/quant-arena
+```
+
+## 2. What happens when you run it
+
+1. **Data loads.** ~8 years of daily bars for 11 tickers (an index `IDX` plus `S01`-`S10`) from `fixtures/data/insample/`. The final 2 years live in `fixtures/data/holdout/` and are **not** loaded — see step 8. The series are synthetic (regime-switching factor model, seeded, regenerable) because the free real-data source we checked licenses personal use only; `fixtures/data/PROVENANCE.md` has the details and how to drop in your own CSVs.
+2. **Evaluation windows are drawn.** 8 overlapping 504-day (~2-year) blocks, block-bootstrap sampled from the in-sample years with a fixed seed — every candidate in the campaign is scored on the same windows, and reruns reproduce bit-identically.
+3. **Benchmarks run.** Three pinned incumbents: buy-and-hold the index, equal-weight monthly rebalance, and a 20/100 moving-average crossover. Their per-window Sharpe ratios define the bar: "best benchmark" is the per-window maximum.
+4. **Strategy authors write code.** Each author is a Claude call with a pinned identity (one plain, one with a quant-researcher system prompt). It gets the strategy contract, the universe summary, the benchmarks' per-window Sharpes, and the current acceptance bar — and must reply with one self-contained TypeScript module exporting `onBar(ctx)`: the harness calls it once per trading day with the bars **up to that day only** (the arrays are physically sliced, so reading the future is structurally impossible), plus the strategy's current holdings and equity, and it answers with target portfolio weights or "hold". Strategies never place orders — a single shared rebalancer (`oms.ts`) turns everyone's target weights into orders under the same sizing rule, LEAN-style `(targetWeight x equity - currentPosition) / price`, long-only. Model spend is metered into a durable cost log (`cost-ledger.jsonl`) with per-call receipts.
+5. **Look-ahead screening, stage 1 (mechanical).** The candidate is re-run on data truncated at several cutoff days. Signals up to each cutoff must be bit-identical to the full-data run — any divergence proves the code read the future, and the candidate is killed with the divergence quoted.
+6. **Look-ahead screening, stage 2 (adversarial).** A second, cheap model reads the source with one job: find look-ahead — indexing past `t`, whole-series statistics feeding per-day decisions, hardcoded dates that smell like memorization. Verdict is JSON; anything but a clean verdict kills the candidate, and an unparseable reply kills it too (the rule fails closed).
+7. **Backtest and verdict.** Survivors are backtested over the whole in-sample period (next-day-open fills, 15 bps one-way costs, no shorting, no leverage) and scored per window. Two engines run: the one-file TypeScript reference engine first, as a fail-closed contract check, then the industry-standard **vectorbt** engine (a persistent python worker in a version-locked environment) produces the official numbers. A parity test suite holds the two engines to agreement on golden fixtures — exact on a no-trade book, within machine precision whenever the book holds cash, and within a documented 0.5% on fully-invested books (the engines differ only in whether fees may be financed by a slightly negative cash balance). The acceptance rule (section 3) decides. Accepted or not, the try is appended to `notebook.jsonl` with its window scores, audit evidence, code hash, and authoring cost.
+8. **Certification, later and by hand.** When you believe a winner, run it once against the untouched final 2 years:
+
+   ```bash
+   npx tsx src/quant-arena/holdout-certify.mts --strategy <path>/strategy.ts --out /tmp/quant-demo
+   ```
+
+   It backtests in-sample + out-of-sample on one axis (so lookbacks are warm), scores only the out-of-sample days against the same three benchmarks, and appends the in-sample vs out-of-sample comparison to the notebook. A second run for the same strategy hash refuses without `--force` — an out-of-sample set answers once; re-rolling it until it agrees turns it into another in-sample set.
+
+## 3. Why the acceptance rule is strict
+
+If you test enough random strategies against the same data, the best one looks brilliant by luck alone.
+Under the assumption of zero skill, the expected best Sharpe among N independent tries grows roughly like sqrt(2 ln N) — try 50 strategies and luck alone buys the winner a substantial edge.
+So the bar a candidate must clear is not fixed: it is `0.10 + 0.15 * sqrt(2 ln N)` of mean excess Sharpe, where N counts **every** candidate ever tried in the campaign, including ones killed for leaking (this is a simplified, auditable version of the Deflated Sharpe Ratio of Bailey & Lopez de Prado, Journal of Portfolio Management, 2014).
+A candidate must ALSO beat the best benchmark in at least 6 of the 8 windows, because one lucky two-year stretch should never carry a decision.
+Every try is a permanent notebook row, so N can never be quietly reset — the price of another shot at the data is a higher bar for everyone after it.
+
+## 4. Reading the notebook
+
+`notebook.jsonl` is append-only JSON lines. Three row types:
+
+- `quant-arena.baselines.v1` — the campaign header: seed, cost assumptions, the 8 windows with dates, and each benchmark's per-window Sharpe.
+- `quant-arena.candidate.v1` — one per try. The fields that matter:
+  - `nTried` — this try's position in the campaign; sets its acceptance bar.
+  - `leakAudit.truncation` / `leakAudit.llm` — both screening verdicts with evidence.
+  - `eval.perWindow` — Sharpe vs best-benchmark Sharpe for each window, with dates.
+  - `verdict` — `accepted`, `rejected-no-edge` (failed the acceptance rule), `rejected-leak`, `rejected-contract` (didn't satisfy the module contract), or `rejected-error`.
+  - `reasons` — the decision spelled out, numbers included.
+- `quant-arena.certification.v1` — the one-shot out-of-sample result, in-sample stats side by side.
+
+A real excerpt from the committed demo campaign (`fixtures/demo-campaign/`): try #4 cleared both look-ahead screens, then lost to the benchmarks in 7 of 8 windows — and after four tries the bar it would have needed had already risen to 0.35 (condensed for width):
+
+```
+{
+  "candidateId": "cand-004-quant-researcher",
+  "proposer": "quant-researcher",
+  "nTried": 4,
+  "leakAudit": {
+    "truncation": {
+      "clean": true
+    },
+    "llm": {
+      "verdict": "clean"
+    }
+  },
+  "eval": {
+    "wins": 1,
+    "requiredWins": 6,
+    "meanExcessSharpe": -0.085,
+    "threshold": 0.35,
+    "perWindow": [
+      {
+        "startDate": "2017-04-06",
+        "endDate": "2019-03-12",
+        "sharpe": 0.94,
+        "bestBaselineSharpe": 1.21,
+        "excess": -0.27
+      },
+      {
+        "startDate": "2018-04-18",
+        "endDate": "2020-03-23",
+        "sharpe": 0.56,
+        "bestBaselineSharpe": 0.61,
+        "excess": -0.05
+      },
+      {
+        "startDate": "2018-08-06",
+        "endDate": "2020-07-09",
+        "sharpe": 0.2,
+        "bestBaselineSharpe": 0.29,
+        "excess": -0.08
+      },
+      "... 5 more windows"
+    ]
+  },
+  "verdict": "rejected-no-edge",
+  "reasons": [
+    "consistency: beat the best baseline in only 1/8 windows (need 6)",
+    "multiplicity: mean excess Sharpe -0.085 < required 0.350 (bar after 4 tried candidates)"
+  ]
+}
+```
+
+## 5. Plugging in your own backtester and data
+
+Scoring goes through one narrow seam: a worker process that takes `{open prices, close prices, target-weight rows, costs, windows}` as JSON lines on stdin and answers `{per-window total return / max drawdown / Sharpe / trade count, full equity curve}` on stdout — see the protocol comment at the top of `python/vbt-worker.py` and the client in `vbt-client.ts`.
+The shipped worker is vectorbt (`Portfolio.from_orders`, target-percent sizing, shared cash, sells before buys), version-locked by `python/pyproject.toml` + `python/uv.lock`; to swap in your own engine, speak the same protocol and keep the fill model (decide at close, fill at next open, bps fees on traded dollars) or re-derive the parity fixtures in `vbt-parity.test.mts` for your model.
+Point the data loader at your own Stooq-format CSVs (one file per ticker, `IDX.csv` as the benchmark asset, an `insample/` and a `holdout/` directory).
+Keep the physical in-sample/out-of-sample split and the once-only certification rule — they are the point, not an implementation detail.
+A third engine is planned but not built: event-driven certification of a winner's order stream through Nautilus Trader (`nautilus-certify.ts` is the named stub).
+
+## Files
+
+| file | what it is |
+| --- | --- |
+| `types.ts` | the strategy contract (v2 `onBar` + the order types), including the no-look-ahead rule |
+| `driver.ts` | the incremental harness: feeds `onBar` day by day with physically truncated history; wraps old batch strategies unchanged |
+| `oms.ts` | the one shared rebalancer: target weights -> orders (strategies never place orders) |
+| `backtest.ts` | the TypeScript reference engine: next-open fills, bps costs, no shorting — zero dependencies; contract prefilter |
+| `vbt-client.ts` + `python/vbt-worker.py` | the official scorer: persistent vectorbt worker, pinned env (`python/uv.lock`), crash-safe request handling |
+| `vbt-parity.test.mts` | the two engines held to agreement on golden fixtures (prints both curves on any disagreement) |
+| `nautilus-certify.ts` | named stub for the planned event-driven certification engine (not implemented) |
+| `windows.ts` | seeded block-bootstrap evaluation windows |
+| `multiplicity.ts` | the rising acceptance bar (documented formula + citation) |
+| `leak-audit.ts` | the mechanical truncation-invariance check |
+| `quant-loop.mts` | the campaign: author -> screen -> backtest -> verdict -> notebook |
+| `holdout-certify.mts` | the once-only out-of-sample certification |
+| `strategies/` | the three pinned benchmarks |
+| `fixtures/data/` | committed daily bars + provenance; `holdout/` is the locked final 2 years |
+| `fixtures/demo-campaign/` | a real captured campaign against the v1 batch contract: notebook, authored strategies, cost receipts |
+| `fixtures/demo-campaign-v2/` | a real captured campaign against the v2 `onBar` contract (1 candidate, honestly rejected: 0/8 windows) |