@tangle-network/agent-bench 0.3.7 → 0.4.0

package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts

@@ -0,0 +1,102 @@
+type Bar = { date: string; open: number; high: number; low: number; close: number; volume: number };
+type Signal = { t: number; weights: number[] };
+
+/**
+ * ECONOMIC RATIONALE
+ * -------------------
+ * One effect, harvested two ways: volatility is persistent and heterogeneous across names and
+ * across time, while expected returns are not reliably predictable day to day. Two robust,
+ * well-documented consequences of that fact:
+ *
+ *  1. Cross-sectional: weighting stocks inversely to their own trailing volatility (risk parity)
+ *     extracts more diversification benefit than equal-weighting, because it does not let the
+ *     noisiest names dominate portfolio variance. This should hold in any regime where the 10
+ *     names have unequal, imperfectly-correlated volatility — i.e. most of the time.
+ *  2. Time-series: realized volatility clusters, so scaling total equity exposure down when
+ *     trailing portfolio vol rises above a fixed target de-risks ahead of stress regimes without
+ *     forecasting direction (the "volatility-managed portfolio" effect). This should specifically
+ *     help in the calm-to-stress transition windows where buy-and-hold and SMA-crossover get hurt.
+ *
+ * Both are single, few-parameter, regime-agnostic effects (no return forecasting, no curve
+ * fitting to dates). Rebalancing only every ~21 trading days keeps turnover — and the 15bps
+ * one-way cost — well below the vol-timing edge, which shows up at monthly-or-slower horizons.
+ * No leverage is ever taken (scale is capped at 1); the model only ever de-risks toward cash.
+ */
+export function generateSignals(bars: Bar[][]): Signal[] {
+  const numTickers = bars.length;       // 0 = IDX (untraded), 1..10 = S01..S10
+  const numStocks = numTickers - 1;
+  const numDays = bars[0].length;
+
+  const VOL_LOOKBACK = 63;              // ~1 trading quarter: per-asset & portfolio realized-vol window
+  const REBAL_INTERVAL = 21;            // ~monthly cadence: keeps turnover cheap relative to the vol-timing edge
+  const TARGET_VOL_ANN = 0.15;          // target annualized portfolio vol; only de-risk when realized vol exceeds it
+  const ANNUALIZATION = Math.sqrt(252);
+  const MIN_HISTORY = VOL_LOOKBACK + 1;
+
+  const signals: Signal[] = [];
+
+  function logReturn(k: number, t: number): number {
+    const prev = bars[k][t - 1].close;
+    const cur = bars[k][t].close;
+    if (prev <= 0 || cur <= 0) return 0;
+    return Math.log(cur / prev);
+  }
+
+  let lastRebalance = -Infinity;
+
+  for (let t = MIN_HISTORY; t < numDays; t++) {
+    if (t - lastRebalance < REBAL_INTERVAL) continue;
+
+    // Per-asset trailing volatility over [t - VOL_LOOKBACK + 1, t] — uses only data up to t.
+    const assetVol: number[] = new Array(numStocks).fill(0);
+    let anyValid = false;
+    for (let s = 0; s < numStocks; s++) {
+      const k = s + 1;
+      const rets: number[] = [];
+      for (let d = t - VOL_LOOKBACK + 1; d <= t; d++) {
+        rets.push(logReturn(k, d));
+      }
+      const mean = rets.reduce((a, b) => a + b, 0) / rets.length;
+      const variance = rets.reduce((a, b) => a + (b - mean) * (b - mean), 0) / rets.length;
+      const vol = Math.sqrt(variance);
+      assetVol[s] = vol;
+      if (vol > 1e-8) anyValid = true;
+    }
+
+    if (!anyValid) {
+      lastRebalance = t;
+      continue;
+    }
+
+    // Inverse-volatility (risk-parity) weights across the traded stocks.
+    const invVol = assetVol.map(v => (v > 1e-8 ? 1 / v : 0));
+    const invVolSum = invVol.reduce((a, b) => a + b, 0);
+    const riskParityWeights = invVol.map(v => (invVolSum > 0 ? v / invVolSum : 0));
+
+    // Realized vol of that risk-parity mix, used purely for the exposure-scaling decision below.
+    const portRets: number[] = [];
+    for (let d = t - VOL_LOOKBACK + 1; d <= t; d++) {
+      let dayRet = 0;
+      for (let s = 0; s < numStocks; s++) {
+        dayRet += riskParityWeights[s] * logReturn(s + 1, d);
+      }
+      portRets.push(dayRet);
+    }
+    const portMeanRet = portRets.reduce((a, b) => a + b, 0) / portRets.length;
+    const portVar = portRets.reduce((a, b) => a + (b - portMeanRet) * (b - portMeanRet), 0) / portRets.length;
+    const portVolAnn = Math.sqrt(portVar) * ANNUALIZATION;
+
+    // Vol targeting: de-lever toward cash when realized vol exceeds target; never lever above 1.
+    const scale = portVolAnn > 1e-8 ? Math.min(1, TARGET_VOL_ANN / portVolAnn) : 1;
+
+    const weights = new Array(numTickers).fill(0);
+    for (let s = 0; s < numStocks; s++) {
+      weights[s + 1] = riskParityWeights[s] * scale;
+    }
+
+    signals.push({ t, weights });
+    lastRebalance = t;
+  }
+
+  return signals;
+}

package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl

@@ -0,0 +1,4 @@
+{"version":1,"record":{"status":"pending","callId":"38ff56f6-527f-49f8-91da-1c665ad23851","channel":"driver","phase":"search.proposal","actor":"proposer-shot:quant-researcher","model":"sonnet","timestamp":1784762874717,"tags":{"candidateId":"cand-001-quant-researcher"}}}
+{"version":1,"record":{"status":"settled","callId":"38ff56f6-527f-49f8-91da-1c665ad23851","channel":"driver","phase":"search.proposal","actor":"proposer-shot:quant-researcher","model":"sonnet","timestamp":1784762874717,"inputTokens":2,"outputTokens":13962,"cachedTokens":14853,"costUsd":0.5023599000000001,"costUnknown":false,"usageUnknown":false,"actualCostUsd":0.5023599000000001,"tags":{"candidateId":"cand-001-quant-researcher"}}}
+{"version":1,"record":{"status":"pending","callId":"8364c35d-7568-4a46-b357-842596bad828","channel":"driver","phase":"audit.leak","actor":"leak-auditor:claude","model":"haiku","timestamp":1784763022945,"tags":{"candidateId":"cand-001-quant-researcher","attempt":"0"}}}
+{"version":1,"record":{"status":"settled","callId":"8364c35d-7568-4a46-b357-842596bad828","channel":"driver","phase":"audit.leak","actor":"leak-auditor:claude","model":"haiku","timestamp":1784763022945,"inputTokens":10,"outputTokens":2486,"cachedTokens":10942,"costUsd":0.0736982,"costUnknown":false,"usageUnknown":false,"actualCostUsd":0.0736982,"tags":{"candidateId":"cand-001-quant-researcher","attempt":"0"}}}

package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl

@@ -0,0 +1,2 @@
+{"schema":"quant-arena.baselines.v1","at":"2026-07-22T23:27:54.712Z","seed":20260722,"costBps":10,"slippageBps":5,"windows":[{"start":199,"end":703,"startDate":"2017-04-06","endDate":"2019-03-12"},{"start":468,"end":972,"startDate":"2018-04-18","endDate":"2020-03-23"},{"start":546,"end":1050,"startDate":"2018-08-06","endDate":"2020-07-09"},{"start":594,"end":1098,"startDate":"2018-10-11","endDate":"2020-09-15"},{"start":680,"end":1184,"startDate":"2019-02-08","endDate":"2021-01-13"},{"start":1014,"end":1518,"startDate":"2020-05-21","endDate":"2022-04-26"},{"start":1057,"end":1561,"startDate":"2020-07-21","endDate":"2022-06-24"},{"start":1449,"end":1953,"startDate":"2022-01-20","endDate":"2023-12-26"}],"perWindowSharpe":{"buy-hold-index":[1.209402818407734,0.26747034145574655,0.02223192919061442,-0.025934457758002974,0.30763713767392015,0.1729701215613101,0.08485225461739357,0.7850882647054723],"equal-weight":[1.1461411611212873,0.6060778021251142,0.2849112842132356,0.17899050927362395,0.6606783099823891,0.5185933734753903,0.5223786523399545,1.2149406938872762],"sma-crossover":[1.1148336642070824,0.09109854752089735,-0.05350102847078379,-0.014563215257672965,0.35981476938731394,0.2503571546107575,0.20375008642895226,1.208060697185852]},"bestPerWindow":[1.209402818407734,0.6060778021251142,0.2849112842132356,0.17899050927362395,0.6606783099823891,0.5185933734753903,0.5223786523399545,1.2149406938872762],"fullSample":{"buy-hold-index":{"start":0,"end":2086,"days":2086,"totalReturn":1.1713024249031854,"maxDrawdown":0.2837597681759013,"sharpe":0.5894270856356978,"tradeCount":1,"turnover":1},"equal-weight":{"start":0,"end":2086,"days":2086,"totalReturn":2.7565834221442085,"maxDrawdown":0.260030673705944,"sharpe":0.8405199991639724,"tradeCount":1099,"turnover":6.9338160520867085},"sma-crossover":{"start":0,"end":2086,"days":2086,"totalReturn":1.774871914165919,"maxDrawdown":0.35749030377671454,"sharpe":0.6590297667154418,"tradeCount":1789,"turnover":111.15341469281668}}}
+{"schema":"quant-arena.candidate.v1","at":"2026-07-22T23:27:54.716Z","candidateId":"cand-001-quant-researcher","proposer":"quant-researcher","authorModel":"sonnet","strategyPath":"/tmp/quant-arena-v2/bench/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts","sha256":"sha256:befe1bfde740e579288c498d01cab91771ccbe492441e94660887d166291125d","authoringCostUsd":0.5023599000000001,"nTried":1,"leakAudit":{"truncation":{"clean":true,"cutoffs":[121,514,906,1299,1691,2084],"divergence":null},"llm":{"verdict":"clean","evidence":""}},"eval":{"perWindow":[{"start":199,"end":703,"startDate":"2017-04-06","endDate":"2019-03-12","sharpe":1.176350979324448,"bestBaselineSharpe":1.209402818407734,"excess":-0.033051839083285905},{"start":468,"end":972,"startDate":"2018-04-18","endDate":"2020-03-23","sharpe":0.3540080230521845,"bestBaselineSharpe":0.6060778021251142,"excess":-0.2520697790729297},{"start":546,"end":1050,"startDate":"2018-08-06","endDate":"2020-07-09","sharpe":0.0830954098341174,"bestBaselineSharpe":0.2849112842132356,"excess":-0.2018158743791182},{"start":594,"end":1098,"startDate":"2018-10-11","endDate":"2020-09-15","sharpe":-0.159946116561702,"bestBaselineSharpe":0.17899050927362395,"excess":-0.3389366258353259},{"start":680,"end":1184,"startDate":"2019-02-08","endDate":"2021-01-13","sharpe":0.39355206968094464,"bestBaselineSharpe":0.6606783099823891,"excess":-0.26712624030144444},{"start":1014,"end":1518,"startDate":"2020-05-21","endDate":"2022-04-26","sharpe":0.06629822864040066,"bestBaselineSharpe":0.5185933734753903,"excess":-0.45229514483498956},{"start":1057,"end":1561,"startDate":"2020-07-21","endDate":"2022-06-24","sharpe":0.08314391874329552,"bestBaselineSharpe":0.5223786523399545,"excess":-0.439234733596659},{"start":1449,"end":1953,"startDate":"2022-01-20","endDate":"2023-12-26","sharpe":1.1420267333798133,"bestBaselineSharpe":1.2149406938872762,"excess":-0.0729139605074629}],"meanExcessSharpe":-0.25718052470140196,"wins":0,"requiredWins":6,"threshold":0.1},"inSampleFull":{"start":0,"end":2086,"days":2086,"totalReturn":1.0793328708686873,"maxDrawdown":0.2547803880567502,"sharpe":0.598115214813146,"tradeCount":1000,"turnover":23.81524755112226},"verdict":"rejected-no-edge","reasons":["consistency: beat the best baseline in only 0/8 windows (need 6)","multiplicity: mean excess Sharpe -0.257 < required 0.100 (bar after 1 tried candidate)"]}

package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json

@@ -0,0 +1,84 @@
+{
+  "schema": "quant-arena.rollout.v1",
+  "outDir": "/tmp/quant-arena-v2/bench/src/quant-arena/fixtures/demo-campaign-v2",
+  "at": "2026-07-22T23:30:50.804Z",
+  "notebookRows": 2,
+  "strategies": [
+    {
+      "label": "baseline-buy-hold-index",
+      "campaignDir": "/tmp/quant-arena-v2/bench/src/quant-arena/fixtures/demo-campaign-v2/campaign/baseline-buy-hold-index",
+      "cells": 8,
+      "scenarios": [
+        "window-1014-1518",
+        "window-1057-1561",
+        "window-1449-1953",
+        "window-199-703",
+        "window-468-972",
+        "window-546-1050",
+        "window-594-1098",
+        "window-680-1184"
+      ]
+    },
+    {
+      "label": "baseline-equal-weight",
+      "campaignDir": "/tmp/quant-arena-v2/bench/src/quant-arena/fixtures/demo-campaign-v2/campaign/baseline-equal-weight",
+      "cells": 8,
+      "scenarios": [
+        "window-1014-1518",
+        "window-1057-1561",
+        "window-1449-1953",
+        "window-199-703",
+        "window-468-972",
+        "window-546-1050",
+        "window-594-1098",
+        "window-680-1184"
+      ]
+    },
+    {
+      "label": "baseline-sma-crossover",
+      "campaignDir": "/tmp/quant-arena-v2/bench/src/quant-arena/fixtures/demo-campaign-v2/campaign/baseline-sma-crossover",
+      "cells": 8,
+      "scenarios": [
+        "window-1014-1518",
+        "window-1057-1561",
+        "window-1449-1953",
+        "window-199-703",
+        "window-468-972",
+        "window-546-1050",
+        "window-594-1098",
+        "window-680-1184"
+      ]
+    },
+    {
+      "label": "cand-001-quant-researcher",
+      "campaignDir": "/tmp/quant-arena-v2/bench/src/quant-arena/fixtures/demo-campaign-v2/campaign/cand-001-quant-researcher",
+      "cells": 8,
+      "scenarios": [
+        "window-1014-1518",
+        "window-1057-1561",
+        "window-1449-1953",
+        "window-199-703",
+        "window-468-972",
+        "window-546-1050",
+        "window-594-1098",
+        "window-680-1184"
+      ]
+    }
+  ],
+  "receipts": [
+    {
+      "callId": "38ff56f6-527f-49f8-91da-1c665ad23851",
+      "phase": "search.proposal",
+      "actor": "proposer-shot:quant-researcher",
+      "model": "sonnet",
+      "costUsd": 0.5023599000000001
+    },
+    {
+      "callId": "8364c35d-7568-4a46-b357-842596bad828",
+      "phase": "audit.leak",
+      "actor": "leak-auditor:claude",
+      "model": "haiku",
+      "costUsd": 0.0736982
+    }
+  ]
+}

package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts

@@ -0,0 +1,117 @@
+export interface Bar {
+  date: string
+  open: number
+  high: number
+  low: number
+  close: number
+  volume: number
+}
+
+export interface StrategyContext {
+  symbols: string[]
+  t: number
+  history: Bar[][]
+  weights: number[]
+  equity: number
+}
+
+export interface TargetPosition {
+  symbol: string
+  weight: number
+}
+
+// ECONOMIC RATIONALE
+// -------------------
+// Core effect (risk parity / low-volatility anomaly): within the 10-name universe,
+// weight each name inversely to its trailing realized volatility instead of equal
+// weight. Lower-vol names have historically offered better risk-adjusted (not
+// necessarily raw) returns, so tilting the book toward them raises the portfolio's
+// Sharpe by shrinking variance faster than it gives up mean return -- a
+// cross-sectional statistical property of the return distribution, not a level
+// fitted to this dataset, so it should generalize out of sample.
+//
+// Overlay (trend-based de-risking): scale TOTAL exposure down when the index is
+// below its own ~6-month moving average. Sustained downtrends carry the worst
+// left-tail days; cutting exposure in that regime avoids most of that tail and
+// improves Sharpe without needing to predict the turn. This is the same
+// mechanism behind classic tactical/trend-following allocation (Faber 2007;
+// time-series momentum, Moskowitz/Ooi/Pedersen 2012).
+//
+// Turnover control: both the vol estimate and the trend regime move slowly, so
+// re-deciding weights only once a month (not daily) captures nearly all the
+// signal while keeping realized turnover, and the 15bps-per-side cost, low.
+
+const VOL_LOOKBACK = 60 // ~1 quarter of trading days for the realized-vol estimate
+const TREND_LOOKBACK = 120 // ~6 months for the index trend line
+const MIN_RETURNS_FOR_VOL = 10 // below this, a vol estimate is too noisy to trust -> equal weight
+const MIN_BARS_FOR_TREND = 20 // below this, no evidence of a downtrend yet -> assume uptrend
+const REBALANCE_PERIOD = 21 // ~1 trading month cadence, keeps churn cheap relative to the edge
+const DEFENSIVE_FRACTION = 0.3 // exposure kept on when the index is below its trend line
+const MAX_STOCK_WEIGHT = 0.25 // guardrail against one noisy low-vol reading dominating the book
+const EPS = 1e-6
+
+function stdev(returns: number[]): number {
+  const n = returns.length
+  const mean = returns.reduce((s, r) => s + r, 0) / n
+  const variance = returns.reduce((s, r) => s + (r - mean) * (r - mean), 0) / (n - 1)
+  return Math.sqrt(Math.max(variance, 0))
+}
+
+function dailyReturns(bars: Bar[], lookback: number): number[] {
+  const start = Math.max(1, bars.length - lookback)
+  const out: number[] = []
+  for (let i = start; i < bars.length; i++) {
+    const prev = bars[i - 1].close
+    const cur = bars[i].close
+    if (prev > 0) out.push(cur / prev - 1)
+  }
+  return out
+}
+
+export function onBar(ctx: StrategyContext): TargetPosition[] | null {
+  const { symbols, t, history } = ctx
+
+  // Establish the initial book on day 0; otherwise only re-decide on a fixed
+  // monthly cadence so 15bps churn cannot eat a vol/trend edge that itself
+  // only moves on a weeks-to-months timescale.
+  if (t !== 0 && t % REBALANCE_PERIOD !== 0) return null
+
+  // --- regime filter: is the index above its own trend line? ---
+  const idxBars = history[0]
+  let uptrend = true
+  if (idxBars.length >= MIN_BARS_FOR_TREND) {
+    const window = idxBars.slice(Math.max(0, idxBars.length - TREND_LOOKBACK))
+    const sma = window.reduce((s, b) => s + b.close, 0) / window.length
+    uptrend = idxBars[idxBars.length - 1].close > sma
+  }
+  const exposure = uptrend ? 1 : DEFENSIVE_FRACTION
+
+  // --- risk-parity book across the 10 constituents (symbols[1..10]) ---
+  const stockIdx: number[] = []
+  for (let k = 1; k < symbols.length; k++) stockIdx.push(k)
+
+  const invVols: number[] = stockIdx.map((k) => {
+    const rets = dailyReturns(history[k], VOL_LOOKBACK)
+    if (rets.length < MIN_RETURNS_FOR_VOL) return NaN
+    return 1 / (stdev(rets) + EPS)
+  })
+
+  const validMask = invVols.map((v) => !Number.isNaN(v))
+  const anyValid = validMask.some(Boolean)
+
+  let baseWeights: number[]
+  if (!anyValid) {
+    baseWeights = stockIdx.map(() => 1 / stockIdx.length)
+  } else {
+    const sumInv = invVols.reduce((s, v, i) => s + (validMask[i] ? v : 0), 0)
+    baseWeights = invVols.map((v, i) => (validMask[i] ? v / sumInv : 0))
+  }
+
+  const positions: TargetPosition[] = []
+  for (let i = 0; i < stockIdx.length; i++) {
+    const w = Math.min(baseWeights[i] * exposure, MAX_STOCK_WEIGHT)
+    positions.push({ symbol: symbols[stockIdx[i]], weight: w })
+  }
+
+  return positions
+}

package/src/quant-arena/holdout-certify.mts

@@ -0,0 +1,206 @@
+/**
+ * Holdout certification — the ONLY code path that reads fixtures/data/holdout/
+ * (the final 2 years). Operator-invoked, once per winning strategy:
+ *
+ *   tsx src/quant-arena/holdout-certify.mts --strategy <path/to/strategy.ts> --out <campaign dir> [--force]
+ *
+ * Protocol:
+ *  - the strategy is backtested over in-sample + holdout concatenated (so its
+ *    lookbacks are warm when the holdout period begins), but SCORED only on
+ *    the holdout days — a true walk-forward on data no candidate ever saw;
+ *  - scoring is the OFFICIAL vectorbt worker (vbt-client.ts); the TS engine
+ *    runs first as fail-closed contract prefilter only;
+ *  - the three pinned baselines run under the identical protocol; the bar is
+ *    the best baseline's holdout Sharpe;
+ *  - the truncation leak audit re-runs on the full axis first (a leak that
+ *    only pays off out-of-sample would otherwise slip through);
+ *  - the verdict is appended to the campaign notebook with in-sample and
+ *    out-of-sample stats side by side, and a certification record is written.
+ *    A second run for the same strategy hash refuses without --force — the
+ *    holdout answers once; re-rolling it is how holdouts die.
+ */
+
+import { createHash } from 'node:crypto'
+import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import process from 'node:process'
+import { runBacktest, statsForRange, type BacktestConfig, type BacktestResult, type RangeStats } from './backtest.ts'
+import { concatForCertification, loadBarsDir, loadHoldout, loadInSample } from './data.ts'
+import { loadStrategyFile } from './driver.ts'
+import { truncationInvariance } from './leak-audit.ts'
+import { PINNED_BASELINES } from './quant-loop.mts'
+import { scoreSignals, VbtWorker, type VbtWindowStats } from './vbt-client.ts'
+import type { Bar, Signal } from './types.ts'
+
+export const CERTIFICATION_SCHEMA = 'quant-arena.certification.v1'
+
+/** Official scores for [0, split) and [split, T) from the vectorbt worker.
+ *  The TS engine runs first as fail-closed contract prefilter and supplies
+ *  turnover (not carried by the worker protocol); every other number is the
+ *  worker's. */
+async function scoreOnRanges(
+  worker: VbtWorker,
+  bars: Bar[][],
+  signals: Signal[],
+  btConfig: BacktestConfig,
+  split: number,
+  T: number,
+): Promise<{ inSample: RangeStats; holdout: RangeStats }> {
+  const prefilter: BacktestResult = runBacktest(bars, signals, btConfig)
+  const vbt = await scoreSignals(worker, bars, signals, btConfig, [
+    [0, split],
+    [split, T],
+  ])
+  const toRange = (stats: VbtWindowStats, start: number, end: number): RangeStats => ({
+    start,
+    end,
+    days: end - start,
+    totalReturn: stats.totalReturn,
+    maxDrawdown: stats.maxDD,
+    sharpe: stats.sharpe,
+    tradeCount: stats.trades,
+    turnover: statsForRange(prefilter, start, end).turnover,
+  })
+  return { inSample: toRange(vbt.windows[0]!, 0, split), holdout: toRange(vbt.windows[1]!, split, T) }
+}
+
+export interface CertificationRecord {
+  schema: typeof CERTIFICATION_SCHEMA
+  at: string
+  strategyPath: string
+  sha256: string
+  costBps: number
+  slippageBps: number
+  holdoutStart: string
+  holdoutEnd: string
+  truncationClean: boolean
+  inSample: RangeStats
+  holdout: RangeStats
+  baselinesHoldout: Record<string, RangeStats>
+  bestBaselineHoldoutSharpe: number
+  holdoutExcessSharpe: number
+  certified: boolean
+  reasons: string[]
+}
+
+export async function certifyOnHoldout(opts: {
+  strategyPath: string
+  outDir: string
+  costBps?: number
+  slippageBps?: number
+  force?: boolean
+  /** Test seam ONLY: alternate data directories so the certification code
+   *  path can be exercised end-to-end WITHOUT reading the real holdout. The
+   *  CLI never sets these — the real run always uses the vendored split. */
+  insampleDir?: string
+  holdoutDir?: string
+  /** Reuse an already-warm vectorbt worker (tests / batch certification).
+   *  When omitted, one is spawned and closed inside this call. */
+  worker?: VbtWorker
+}): Promise<CertificationRecord> {
+  const code = await readFile(opts.strategyPath, 'utf8')
+  const sha256 = `sha256:${createHash('sha256').update(code).digest('hex')}`
+  const certDir = join(opts.outDir, 'holdout-certification')
+  await mkdir(certDir, { recursive: true })
+  const recordPath = join(certDir, `certification-${sha256.slice(7, 17)}.json`)
+  if (existsSync(recordPath) && !opts.force) {
+    throw new Error(
+      `holdout-certify: ${recordPath} already exists — the holdout answers once per strategy. ` +
+        'Re-running it turns the holdout into another in-sample set. Use --force only if the prior run was broken.',
+    )
+  }
+
+  const insample = opts.insampleDir ? await loadBarsDir(opts.insampleDir) : await loadInSample()
+  const holdout = opts.holdoutDir ? await loadBarsDir(opts.holdoutDir) : await loadHoldout()
+  const { aligned, holdoutStartIndex } = concatForCertification(insample, holdout)
+  const btConfig: BacktestConfig = { costBps: opts.costBps ?? 10, slippageBps: opts.slippageBps ?? 5 }
+  const T = aligned.dates.length
+
+  // v2 (`onBar`) and v1 (`generateSignals`) modules both certify; v2 runs
+  // through the incremental driver with structurally truncated history.
+  const { generateSignals: strategy } = await loadStrategyFile(opts.strategyPath, {
+    symbols: aligned.tickers,
+    costBps: btConfig.costBps,
+    slippageBps: btConfig.slippageBps,
+  })
+
+  const truncation = truncationInvariance(strategy, aligned.bars, { warmupDays: 120 })
+
+  const worker = opts.worker ?? new VbtWorker()
+  let inSampleStats: RangeStats
+  let holdoutStats: RangeStats
+  const baselinesHoldout: Record<string, RangeStats> = {}
+  try {
+    const scored = await scoreOnRanges(worker, aligned.bars, strategy(aligned.bars), btConfig, holdoutStartIndex, T)
+    inSampleStats = scored.inSample
+    holdoutStats = scored.holdout
+    for (const [name, baseline] of Object.entries(PINNED_BASELINES)) {
+      const baseScored = await scoreOnRanges(worker, aligned.bars, baseline(aligned.bars), btConfig, holdoutStartIndex, T)
+      baselinesHoldout[name] = baseScored.holdout
+    }
+  } finally {
+    if (opts.worker === undefined) await worker.close()
+  }
+  const bestBaselineHoldoutSharpe = Math.max(...Object.values(baselinesHoldout).map((s) => s.sharpe))
+  const holdoutExcessSharpe = holdoutStats.sharpe - bestBaselineHoldoutSharpe
+
+  const reasons: string[] = []
+  if (!truncation.clean) {
+    reasons.push(`leak: truncation divergence at cutoff ${truncation.divergence!.cutoff}: ${truncation.divergence!.detail}`)
+  }
+  if (holdoutExcessSharpe <= 0) {
+    reasons.push(
+      `no out-of-sample edge: holdout Sharpe ${holdoutStats.sharpe.toFixed(3)} <= best baseline ${bestBaselineHoldoutSharpe.toFixed(3)}`,
+    )
+  }
+  const certified = reasons.length === 0
+  if (certified) {
+    reasons.push(
+      `certified: holdout Sharpe ${holdoutStats.sharpe.toFixed(3)} beats best baseline ` +
+        `${bestBaselineHoldoutSharpe.toFixed(3)} on ${holdoutStats.days} untouched days`,
+    )
+  }
+
+  const record: CertificationRecord = {
+    schema: CERTIFICATION_SCHEMA,
+    at: new Date().toISOString(),
+    strategyPath: opts.strategyPath,
+    sha256,
+    costBps: btConfig.costBps,
+    slippageBps: btConfig.slippageBps,
+    holdoutStart: aligned.dates[holdoutStartIndex]!,
+    holdoutEnd: aligned.dates[T - 1]!,
+    truncationClean: truncation.clean,
+    inSample: inSampleStats,
+    holdout: holdoutStats,
+    baselinesHoldout,
+    bestBaselineHoldoutSharpe,
+    holdoutExcessSharpe,
+    certified,
+    reasons,
+  }
+  await writeFile(recordPath, JSON.stringify(record, null, 2))
+  await appendFile(join(opts.outDir, 'notebook.jsonl'), JSON.stringify(record) + '\n')
+  return record
+}
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const argv = process.argv.slice(2)
+  const flag = (name: string): string | undefined => {
+    const i = argv.indexOf(name)
+    return i !== -1 ? argv[i + 1] : undefined
+  }
+  const strategyPath = flag('--strategy')
+  const outDir = flag('--out')
+  if (!strategyPath || !outDir) {
+    console.error('usage: tsx src/quant-arena/holdout-certify.mts --strategy <strategy.ts> --out <campaign dir> [--force]')
+    process.exit(2)
+  }
+  const record = await certifyOnHoldout({ strategyPath, outDir, force: argv.includes('--force') })
+  console.log(JSON.stringify({ certified: record.certified, reasons: record.reasons, holdout: record.holdout, inSample: record.inSample }, null, 2))
+  process.exitCode = record.certified ? 0 : 1
+}

package/src/quant-arena/holdout-certify.test.mts

@@ -0,0 +1,82 @@
+import { mkdtemp, readFile, writeFile, mkdir } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { certifyOnHoldout } from './holdout-certify.mts'
+import { VbtWorker } from './vbt-client.ts'
+
+/** Synthetic Stooq-format CSVs on real calendar dates — the certification
+ *  path exercised end-to-end WITHOUT ever reading the real vendored holdout. */
+const isoDate = (dayOffset: number): string => {
+  const d = new Date(Date.UTC(2020, 0, 1))
+  d.setUTCDate(d.getUTCDate() + dayOffset)
+  return d.toISOString().slice(0, 10)
+}
+
+const csvFor = (phase: number, from: number, to: number): string => {
+  const lines = ['Date,Open,High,Low,Close,Volume']
+  for (let t = from; t < to; t++) {
+    const close = 100 + 10 * Math.sin(t / 9 + phase) + 0.03 * t
+    const open = 100 + 10 * Math.sin((t - 0.5) / 9 + phase) + 0.03 * t
+    lines.push(
+      `${isoDate(t)},${open.toFixed(4)},${(Math.max(open, close) + 1).toFixed(4)},${(Math.min(open, close) - 1).toFixed(4)},${close.toFixed(4)},1000`,
+    )
+  }
+  return lines.join('\n') + '\n'
+}
+
+const BUY_HOLD_PATH = fileURLToPath(new URL('./strategies/buy-hold-index/strategy.ts', import.meta.url))
+
+// Certification scores through the official vectorbt worker; without `uv`
+// (the pinned env launcher) this suite skips with that reason.
+describe.skipIf(!VbtWorker.isAvailable())('certifyOnHoldout (synthetic stand-in dirs — the real holdout stays untouched)', () => {
+  let outDir: string
+  let insampleDir: string
+  let holdoutDir: string
+  let worker: VbtWorker
+
+  afterAll(async () => {
+    await worker?.close()
+  })
+
+  beforeAll(async () => {
+    worker = new VbtWorker()
+    await worker.ping() // absorb cold numba JIT before test timeouts apply
+    const root = await mkdtemp(join(tmpdir(), 'quant-arena-cert-'))
+    outDir = join(root, 'out')
+    insampleDir = join(root, 'insample')
+    holdoutDir = join(root, 'holdout')
+    await mkdir(outDir, { recursive: true })
+    await mkdir(insampleDir, { recursive: true })
+    await mkdir(holdoutDir, { recursive: true })
+    for (const [ticker, phase] of [['IDX', 0], ['S01', 2]] as const) {
+      await writeFile(join(insampleDir, `${ticker}.csv`), csvFor(phase, 0, 300))
+      await writeFile(join(holdoutDir, `${ticker}.csv`), csvFor(phase, 300, 400))
+    }
+  }, 240_000)
+
+  it('scores in-sample and holdout side by side and appends a notebook row', { timeout: 120_000 }, async () => {
+    const record = await certifyOnHoldout({ strategyPath: BUY_HOLD_PATH, outDir, insampleDir, holdoutDir, worker })
+    expect(record.truncationClean).toBe(true)
+    expect(record.inSample.days).toBe(300)
+    expect(record.holdout.days).toBe(100)
+    expect(record.holdoutStart).toBe(isoDate(300))
+    // Buy-and-hold IS one of the pinned baselines, so its excess over the
+    // best baseline can never be positive: deterministically not certified.
+    expect(record.holdoutExcessSharpe).toBeLessThanOrEqual(0)
+    expect(record.certified).toBe(false)
+    expect(record.reasons[0]).toMatch(/no out-of-sample edge/)
+    const notebook = await readFile(join(outDir, 'notebook.jsonl'), 'utf8')
+    const rows = notebook.trim().split('\n').map((l) => JSON.parse(l) as { schema: string })
+    expect(rows.some((r) => r.schema === 'quant-arena.certification.v1')).toBe(true)
+  })
+
+  it('refuses a second certification for the same strategy hash without --force', { timeout: 120_000 }, async () => {
+    await expect(certifyOnHoldout({ strategyPath: BUY_HOLD_PATH, outDir, insampleDir, holdoutDir, worker })).rejects.toThrow(
+      /answers once/,
+    )
+    const forced = await certifyOnHoldout({ strategyPath: BUY_HOLD_PATH, outDir, insampleDir, holdoutDir, worker, force: true })
+    expect(forced.schema).toBe('quant-arena.certification.v1')
+  })
+})