@tangle-network/agent-bench 0.3.7 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/HARNESS.md +43 -0
- package/README.md +7 -0
- package/dist/adapters.js +23 -23
- package/dist/benchmarks/_harness.d.ts +1 -1
- package/dist/benchmarks/_harness.js +1 -1
- package/dist/benchmarks/aec-bench.js +2 -2
- package/dist/benchmarks/agentbench.js +2 -2
- package/dist/benchmarks/appworld.js +2 -2
- package/dist/benchmarks/bfcl.js +2 -2
- package/dist/benchmarks/commit0.js +2 -2
- package/dist/benchmarks/crag.js +2 -2
- package/dist/benchmarks/dabstep.js +2 -2
- package/dist/benchmarks/enterpriseops-gym.js +2 -2
- package/dist/benchmarks/finresearchbench.js +2 -2
- package/dist/benchmarks/nomiracl.js +2 -2
- package/dist/benchmarks/open-rag-bench.js +2 -2
- package/dist/benchmarks/programbench.js +2 -2
- package/dist/benchmarks/ragbench.js +2 -2
- package/dist/benchmarks/swe-bench.js +2 -2
- package/dist/benchmarks/t2-ragbench.js +2 -2
- package/dist/benchmarks/tau-bench-shared.js +2 -2
- package/dist/benchmarks/tau2-bench.js +3 -3
- package/dist/benchmarks/tau3-banking.js +3 -3
- package/dist/benchmarks/terminal-bench.js +2 -2
- package/dist/benchmarks/toollm.js +2 -2
- package/dist/benchmarks/webarena-verified.js +2 -2
- package/dist/{chunk-CKUVRZ2T.js → chunk-3U5TXJZS.js} +2 -2
- package/dist/{chunk-YCGY7UIZ.js → chunk-7GRVHU22.js} +2 -2
- package/dist/{chunk-Z7ML6L77.js → chunk-HWST3SED.js} +2 -2
- package/dist/{chunk-SYDW647C.js → chunk-IA2FBTWC.js} +2 -2
- package/dist/{chunk-R67DFVLO.js → chunk-IFVINJ4B.js} +2 -2
- package/dist/{chunk-R36V2VP7.js → chunk-IZ5M6OAC.js} +2 -2
- package/dist/{chunk-ODT47UAY.js → chunk-K3BQGZCT.js} +2 -2
- package/dist/{chunk-IFAV6KEM.js → chunk-KP5KD6EN.js} +2 -2
- package/dist/{chunk-ZEWMTR5M.js → chunk-MQMRLGOG.js} +2 -2
- package/dist/{chunk-TSWPNOYM.js → chunk-NQG5XDSB.js} +2 -2
- package/dist/{chunk-7WSD27QQ.js → chunk-PB64GYIG.js} +2 -2
- package/dist/{chunk-HBSWHQNJ.js → chunk-RCYQEFNX.js} +3 -3
- package/dist/{chunk-J3KDJNX2.js → chunk-RH5F53JT.js} +2 -2
- package/dist/{chunk-UAIOHCUK.js → chunk-SFLA7OH3.js} +3 -3
- package/dist/{chunk-Y6O2OCUO.js → chunk-SHM6MRRF.js} +2 -2
- package/dist/{chunk-KDIKRJGB.js → chunk-SHYIRB7I.js} +2 -2
- package/dist/{chunk-HHXFIHXC.js → chunk-SVR2LKYI.js} +2 -2
- package/dist/{chunk-PWQVGAJB.js → chunk-V7AEBY6U.js} +21 -21
- package/dist/{chunk-LRRD7NAG.js → chunk-WSKWVEQB.js} +18 -2
- package/dist/chunk-WSKWVEQB.js.map +1 -0
- package/dist/{chunk-2PVVP7GN.js → chunk-XKEFIFIC.js} +2 -2
- package/dist/{chunk-JRWWGMK7.js → chunk-XYA4XSNU.js} +2 -2
- package/dist/{chunk-X5YKXC6V.js → chunk-YSMEKBTD.js} +2 -2
- package/dist/{chunk-2XU6OGEN.js → chunk-Z4TZ76N7.js} +2 -2
- package/dist/index.js +23 -23
- package/package.json +5 -5
- package/scripts/trata-hedge/README.md +6 -5
- package/scripts/verify-packed-consumer.mjs +1 -1
- package/src/benchmarks/_harness.ts +20 -2
- package/src/benchmarks/humaneval.test.mts +2 -2
- package/src/gate.ts +1 -1
- package/src/hev-eval.mts +5 -2
- package/src/hev-improve.mts +118 -73
- package/src/official-optimizer-config.mts +89 -0
- package/src/official-optimizer-config.test.mts +88 -0
- package/src/profiles.ts +2 -2
- package/src/quant-arena/README.md +144 -0
- package/src/quant-arena/backtest.test.mts +135 -0
- package/src/quant-arena/backtest.ts +218 -0
- package/src/quant-arena/data.test.mts +44 -0
- package/src/quant-arena/data.ts +141 -0
- package/src/quant-arena/driver.test.mts +253 -0
- package/src/quant-arena/driver.ts +219 -0
- package/src/quant-arena/fixtures/data/PROVENANCE.md +26 -0
- package/src/quant-arena/fixtures/data/holdout/IDX.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S01.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S02.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S03.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S04.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S05.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S06.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S07.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S08.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S09.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S10.csv +523 -0
- package/src/quant-arena/fixtures/data/insample/IDX.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S01.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S02.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S03.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S04.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S05.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S06.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S07.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S08.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S09.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S10.csv +2087 -0
- package/src/quant-arena/fixtures/demo-campaign/cost-ledger.jsonl +16 -0
- package/src/quant-arena/fixtures/demo-campaign/notebook.jsonl +5 -0
- package/src/quant-arena/fixtures/demo-campaign/rollout-manifest.json +171 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-001-default-author/strategy.ts +119 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-002-default-author/strategy.ts +119 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-003-quant-researcher/strategy.ts +105 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts +102 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl +4 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl +2 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json +84 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts +117 -0
- package/src/quant-arena/holdout-certify.mts +206 -0
- package/src/quant-arena/holdout-certify.test.mts +82 -0
- package/src/quant-arena/leak-audit.test.mts +79 -0
- package/src/quant-arena/leak-audit.ts +95 -0
- package/src/quant-arena/make-fixtures.mts +161 -0
- package/src/quant-arena/multiplicity.test.mts +68 -0
- package/src/quant-arena/multiplicity.ts +87 -0
- package/src/quant-arena/nautilus-certify.ts +31 -0
- package/src/quant-arena/oms.ts +90 -0
- package/src/quant-arena/profiles/quant-researcher.profile.json +7 -0
- package/src/quant-arena/python/pyproject.toml +8 -0
- package/src/quant-arena/python/uv.lock +1297 -0
- package/src/quant-arena/python/vbt-worker.py +192 -0
- package/src/quant-arena/quant-loop.mts +813 -0
- package/src/quant-arena/quant-loop.test.mts +75 -0
- package/src/quant-arena/strategies/buy-hold-index/strategy.ts +11 -0
- package/src/quant-arena/strategies/equal-weight/strategy.ts +20 -0
- package/src/quant-arena/strategies/sma-crossover/strategy.ts +42 -0
- package/src/quant-arena/types.ts +133 -0
- package/src/quant-arena/vbt-client.ts +321 -0
- package/src/quant-arena/vbt-parity.test.mts +183 -0
- package/src/quant-arena/windows.test.mts +45 -0
- package/src/quant-arena/windows.ts +54 -0
- package/src/rollout-ledger/backfill-swe-arena.mts +606 -0
- package/src/rollout-ledger/backfill-swe-arena.test.mts +342 -0
- package/src/rollout-ledger/settle-capture.mts +442 -0
- package/src/rollout-ledger/settle-capture.test.mts +270 -0
- package/src/smoke-structural-rollout.mts +15 -9
- package/src/swe-arena/activation.mts +225 -0
- package/src/swe-arena/activation.test.mts +300 -0
- package/src/swe-arena/arms.ts +30 -14
- package/src/swe-arena/briefing.mts +217 -0
- package/src/swe-arena/briefing.test.mts +178 -0
- package/src/swe-arena/calibrate.ts +102 -1
- package/src/swe-arena/cell-evidence.mts +41 -9
- package/src/swe-arena/factory-command-container.ts +284 -0
- package/src/swe-arena/factory-judge-child.mts +228 -0
- package/src/swe-arena/factory.test.mts +643 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/calibration.md +51 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/manifest.json +29 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/spec.md +64 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/calibration.md +48 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/manifest.json +29 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/spec.md +48 -0
- package/src/swe-arena/fixtures/factory/loops-28/calibration.md +47 -0
- package/src/swe-arena/fixtures/factory/loops-28/manifest.json +30 -0
- package/src/swe-arena/fixtures/factory/loops-28/spec.md +50 -0
- package/src/swe-arena/fixtures/run-report/README.md +43 -0
- package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json +173 -0
- package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md +100 -0
- package/src/swe-arena/fixtures/run-report/gen3-rollup.json +551 -0
- package/src/swe-arena/fixtures/run-report/gen3-rollup.md +64 -0
- package/src/swe-arena/fixtures.ts +135 -2
- package/src/swe-arena/gepa-seat.mts +877 -0
- package/src/swe-arena/gepa-seat.test.mts +1059 -0
- package/src/swe-arena/implementation-ref.test.mts +64 -0
- package/src/swe-arena/implementation-ref.ts +62 -0
- package/src/swe-arena/ledger-orphans.mts +77 -0
- package/src/swe-arena/ledger-orphans.test.mts +147 -0
- package/src/swe-arena/outer-loop.mts +710 -33
- package/src/swe-arena/premeasured-from-cells.mts +281 -0
- package/src/swe-arena/premeasured-from-cells.test.mts +180 -0
- package/src/swe-arena/proposer-fanout.mts +327 -31
- package/src/swe-arena/proposer-fanout.test.mts +246 -0
- package/src/swe-arena/proposer-provenance.mts +172 -0
- package/src/swe-arena/proposer-provenance.test.mts +106 -0
- package/src/swe-arena/run-experiment.mts +367 -1
- package/src/swe-arena/run-report.mts +75 -0
- package/src/swe-arena/run-supervisor.test.mts +5 -3
- package/src/swe-arena/score-split.mts +140 -0
- package/src/swe-arena/score-split.test.mts +123 -0
- package/src/swe-arena/scratch-worktree.test.mts +55 -0
- package/src/swe-arena/scratch-worktree.ts +34 -0
- package/src/swe-arena/types.ts +52 -0
- package/src/swe-code-improve.mts +24 -25
- package/src/swe-improve.mts +129 -96
- package/src/swe-local-proof.mts +6 -1
- package/src/swe-stream.mts +4 -2
- package/src/swe-structural.mts +245 -837
- package/src/tb-container-executor.test.mts +30 -6
- package/src/tb-supervisor-sidecar.mts +2 -1
- package/src/trata-gepa.mts +182 -245
- package/dist/chunk-LRRD7NAG.js.map +0 -1
- package/src/live-improve-campaign-mbpp.mts +0 -641
- package/src/live-improve-campaign.mts +0 -500
- /package/dist/{chunk-CKUVRZ2T.js.map → chunk-3U5TXJZS.js.map} +0 -0
- /package/dist/{chunk-YCGY7UIZ.js.map → chunk-7GRVHU22.js.map} +0 -0
- /package/dist/{chunk-Z7ML6L77.js.map → chunk-HWST3SED.js.map} +0 -0
- /package/dist/{chunk-SYDW647C.js.map → chunk-IA2FBTWC.js.map} +0 -0
- /package/dist/{chunk-R67DFVLO.js.map → chunk-IFVINJ4B.js.map} +0 -0
- /package/dist/{chunk-R36V2VP7.js.map → chunk-IZ5M6OAC.js.map} +0 -0
- /package/dist/{chunk-ODT47UAY.js.map → chunk-K3BQGZCT.js.map} +0 -0
- /package/dist/{chunk-IFAV6KEM.js.map → chunk-KP5KD6EN.js.map} +0 -0
- /package/dist/{chunk-ZEWMTR5M.js.map → chunk-MQMRLGOG.js.map} +0 -0
- /package/dist/{chunk-TSWPNOYM.js.map → chunk-NQG5XDSB.js.map} +0 -0
- /package/dist/{chunk-7WSD27QQ.js.map → chunk-PB64GYIG.js.map} +0 -0
- /package/dist/{chunk-HBSWHQNJ.js.map → chunk-RCYQEFNX.js.map} +0 -0
- /package/dist/{chunk-J3KDJNX2.js.map → chunk-RH5F53JT.js.map} +0 -0
- /package/dist/{chunk-UAIOHCUK.js.map → chunk-SFLA7OH3.js.map} +0 -0
- /package/dist/{chunk-Y6O2OCUO.js.map → chunk-SHM6MRRF.js.map} +0 -0
- /package/dist/{chunk-KDIKRJGB.js.map → chunk-SHYIRB7I.js.map} +0 -0
- /package/dist/{chunk-HHXFIHXC.js.map → chunk-SVR2LKYI.js.map} +0 -0
- /package/dist/{chunk-PWQVGAJB.js.map → chunk-V7AEBY6U.js.map} +0 -0
- /package/dist/{chunk-2PVVP7GN.js.map → chunk-XKEFIFIC.js.map} +0 -0
- /package/dist/{chunk-JRWWGMK7.js.map → chunk-XYA4XSNU.js.map} +0 -0
- /package/dist/{chunk-X5YKXC6V.js.map → chunk-YSMEKBTD.js.map} +0 -0
- /package/dist/{chunk-2XU6OGEN.js.map → chunk-Z4TZ76N7.js.map} +0 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest'
|
|
2
|
+
import { truncationCutoffs, truncationInvariance } from './leak-audit.ts'
|
|
3
|
+
import { generateSignals as smaCrossover } from './strategies/sma-crossover/strategy.ts'
|
|
4
|
+
import { generateSignals as buyHold } from './strategies/buy-hold-index/strategy.ts'
|
|
5
|
+
import type { Bar, Signal } from './types.ts'
|
|
6
|
+
|
|
7
|
+
/** Deterministic wiggly price path (no RNG — the test must be reproducible). */
|
|
8
|
+
function syntheticBars(n: number, tickers: number): Bar[][] {
|
|
9
|
+
return Array.from({ length: tickers }, (_, k) =>
|
|
10
|
+
Array.from({ length: n }, (_, t) => {
|
|
11
|
+
const close = 100 + 10 * Math.sin(t / 7 + k) + 0.05 * t
|
|
12
|
+
const open = 100 + 10 * Math.sin((t - 0.5) / 7 + k) + 0.05 * t
|
|
13
|
+
return {
|
|
14
|
+
date: `d${String(t).padStart(4, '0')}`,
|
|
15
|
+
open,
|
|
16
|
+
high: Math.max(open, close) + 1,
|
|
17
|
+
low: Math.min(open, close) - 1,
|
|
18
|
+
close,
|
|
19
|
+
volume: 1000,
|
|
20
|
+
}
|
|
21
|
+
}),
|
|
22
|
+
)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Deliberately leaky: sizes today's weight by TOMORROW's return. Exactly the
|
|
26
|
+
* bug class the truncation check exists to kill. */
|
|
27
|
+
const peekAheadStrategy = (bars: Bar[][]): Signal[] => {
|
|
28
|
+
const T = bars[0]!.length
|
|
29
|
+
const signals: Signal[] = []
|
|
30
|
+
for (let t = 0; t < T - 1; t++) {
|
|
31
|
+
const up = bars[0]![t + 1]!.close > bars[0]![t]!.close
|
|
32
|
+
signals.push({ t, weights: bars.map((_, k) => (k === 0 && up ? 1 : 0)) })
|
|
33
|
+
}
|
|
34
|
+
return signals
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Leaky via whole-series statistics: normalizes by the FULL-sample max close. */
|
|
38
|
+
const fullSampleMaxStrategy = (bars: Bar[][]): Signal[] => {
|
|
39
|
+
const maxClose = Math.max(...bars[0]!.map((b) => b.close))
|
|
40
|
+
const T = bars[0]!.length
|
|
41
|
+
const signals: Signal[] = []
|
|
42
|
+
for (let t = 0; t < T; t++) {
|
|
43
|
+
const w = bars[0]![t]!.close / maxClose
|
|
44
|
+
signals.push({ t, weights: bars.map((_, k) => (k === 0 ? w : 0)) })
|
|
45
|
+
}
|
|
46
|
+
return signals
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
describe('truncationInvariance — the mechanical look-ahead guard', () => {
|
|
50
|
+
const bars = syntheticBars(400, 3)
|
|
51
|
+
|
|
52
|
+
it('catches a strategy that peeks at tomorrow', () => {
|
|
53
|
+
const report = truncationInvariance(peekAheadStrategy, bars, { warmupDays: 50 })
|
|
54
|
+
expect(report.clean).toBe(false)
|
|
55
|
+
expect(report.divergence).not.toBeNull()
|
|
56
|
+
expect(report.divergence!.detail).toMatch(/decision changed|exists only/)
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
it('catches a strategy leaking through full-sample statistics', () => {
|
|
60
|
+
const report = truncationInvariance(fullSampleMaxStrategy, bars, { warmupDays: 50 })
|
|
61
|
+
expect(report.clean).toBe(false)
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('passes the pinned baselines (no look-ahead by construction)', () => {
|
|
65
|
+
expect(truncationInvariance(smaCrossover, bars, { warmupDays: 120 }).clean).toBe(true)
|
|
66
|
+
expect(truncationInvariance(buyHold, bars, { warmupDays: 50 }).clean).toBe(true)
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
it('cutoffs are inside (warmup, T-1) and deduplicated ascending', () => {
|
|
70
|
+
const cutoffs = truncationCutoffs(400, 50)
|
|
71
|
+
expect(cutoffs.length).toBeGreaterThanOrEqual(2)
|
|
72
|
+
for (const c of cutoffs) {
|
|
73
|
+
expect(c).toBeGreaterThan(50)
|
|
74
|
+
expect(c).toBeLessThan(399)
|
|
75
|
+
}
|
|
76
|
+
expect(cutoffs).toEqual([...cutoffs].sort((a, b) => a - b))
|
|
77
|
+
expect(new Set(cutoffs).size).toBe(cutoffs.length)
|
|
78
|
+
})
|
|
79
|
+
})
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Look-ahead (leak) audit — the deterministic half.
|
|
3
|
+
*
|
|
4
|
+
* The strongest guard against look-ahead bias is mechanical: run the strategy
|
|
5
|
+
* on data truncated at day c and on the full history. Under the contract
|
|
6
|
+
* ("the signal at t uses only bars[0..t]"), every signal with t <= c must be
|
|
7
|
+
* BIT-IDENTICAL in both runs. Any divergence proves the strategy read past
|
|
8
|
+
* its decision day — future closes, whole-series statistics, end-anchored
|
|
9
|
+
* indexing — regardless of how the leak was written.
|
|
10
|
+
*
|
|
11
|
+
* This check is necessary, not sufficient (a strategy hardcoding calendar
|
|
12
|
+
* dates it memorized survives truncation), which is why the campaign ALSO
|
|
13
|
+
* runs an adversarial LLM read of the source (see quant-loop.mts). Kill on
|
|
14
|
+
* either.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { Bar, GenerateSignals, Signal } from './types.ts'
|
|
18
|
+
|
|
19
|
+
export interface TruncationDivergence {
|
|
20
|
+
cutoff: number
|
|
21
|
+
t: number
|
|
22
|
+
detail: string
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface TruncationReport {
|
|
26
|
+
clean: boolean
|
|
27
|
+
cutoffs: number[]
|
|
28
|
+
divergence: TruncationDivergence | null
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Evenly spaced cutoffs across (warmupDays, totalDays - 1). */
|
|
32
|
+
export function truncationCutoffs(totalDays: number, warmupDays: number, count = 6): number[] {
|
|
33
|
+
const lo = Math.max(warmupDays + 1, 1)
|
|
34
|
+
const hi = totalDays - 2
|
|
35
|
+
if (hi <= lo) throw new Error(`truncationCutoffs: no room between warmup ${warmupDays} and ${totalDays} days`)
|
|
36
|
+
const cutoffs = new Set<number>()
|
|
37
|
+
for (let i = 0; i < count; i++) {
|
|
38
|
+
cutoffs.add(Math.round(lo + ((hi - lo) * i) / Math.max(count - 1, 1)))
|
|
39
|
+
}
|
|
40
|
+
return [...cutoffs].sort((a, b) => a - b)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const signalMap = (signals: Signal[], maxT: number): Map<number, number[]> => {
|
|
44
|
+
const map = new Map<number, number[]>()
|
|
45
|
+
for (const s of signals) if (s.t <= maxT) map.set(s.t, s.weights)
|
|
46
|
+
return map
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function compareAt(cutoff: number, full: Signal[], truncated: Signal[]): TruncationDivergence | null {
|
|
50
|
+
const fullMap = signalMap(full, cutoff)
|
|
51
|
+
const truncMap = signalMap(truncated, cutoff)
|
|
52
|
+
const ts = new Set([...fullMap.keys(), ...truncMap.keys()])
|
|
53
|
+
for (const t of [...ts].sort((a, b) => a - b)) {
|
|
54
|
+
const a = fullMap.get(t)
|
|
55
|
+
const b = truncMap.get(t)
|
|
56
|
+
if (a === undefined || b === undefined) {
|
|
57
|
+
return {
|
|
58
|
+
cutoff,
|
|
59
|
+
t,
|
|
60
|
+
detail: `signal at t=${t} exists only in the ${a === undefined ? 'truncated' : 'full'} run`,
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (a.length !== b.length) {
|
|
64
|
+
return { cutoff, t, detail: `weight vector length ${a.length} vs ${b.length} at t=${t}` }
|
|
65
|
+
}
|
|
66
|
+
for (let k = 0; k < a.length; k++) {
|
|
67
|
+
if (!Object.is(a[k], b[k])) {
|
|
68
|
+
return {
|
|
69
|
+
cutoff,
|
|
70
|
+
t,
|
|
71
|
+
detail: `weights[${k}] at t=${t}: full=${a[k]} truncated=${b[k]} — the decision changed when future bars were removed`,
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return null
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Run the truncation-invariance check. Deterministic; throws only if the
|
|
80
|
+
* strategy itself throws. */
|
|
81
|
+
export function truncationInvariance(
|
|
82
|
+
strategy: GenerateSignals,
|
|
83
|
+
bars: Bar[][],
|
|
84
|
+
opts: { warmupDays: number; cutoffs?: number[] },
|
|
85
|
+
): TruncationReport {
|
|
86
|
+
const totalDays = bars[0]?.length ?? 0
|
|
87
|
+
const cutoffs = opts.cutoffs ?? truncationCutoffs(totalDays, opts.warmupDays)
|
|
88
|
+
const full = strategy(bars)
|
|
89
|
+
for (const cutoff of cutoffs) {
|
|
90
|
+
const truncated = strategy(bars.map((series) => series.slice(0, cutoff + 1)))
|
|
91
|
+
const divergence = compareAt(cutoff, full, truncated)
|
|
92
|
+
if (divergence !== null) return { clean: false, cutoffs, divergence }
|
|
93
|
+
}
|
|
94
|
+
return { clean: true, cutoffs, divergence: null }
|
|
95
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic synthetic market generator — writes the vendored fixtures.
|
|
3
|
+
*
|
|
4
|
+
* tsx src/quant-arena/make-fixtures.mts # regenerates fixtures/data/**
|
|
5
|
+
*
|
|
6
|
+
* Why synthetic (see fixtures/data/PROVENANCE.md): the free daily-bar sources
|
|
7
|
+
* we checked (Stooq) license their data for personal use only and sit behind
|
|
8
|
+
* an anti-bot wall, so vendoring real bars into a redistributable repo is not
|
|
9
|
+
* clean. Instead the repo commits 10 years of clearly-labeled synthetic daily
|
|
10
|
+
* bars for an index (IDX) plus 10 stocks (S01..S10), produced by THIS file
|
|
11
|
+
* with a fixed seed — anyone can regenerate and diff them.
|
|
12
|
+
*
|
|
13
|
+
* The generator aims for realistic structure, not any real market's path:
|
|
14
|
+
* - a two-state regime-switching market factor (calm: +9%/yr drift, 13% vol;
|
|
15
|
+
* stress: -12%/yr drift, 32% vol; sticky transitions),
|
|
16
|
+
* - per-stock beta in [0.6, 1.5], idiosyncratic vol 15-35%/yr, small alpha,
|
|
17
|
+
* rare idiosyncratic jumps,
|
|
18
|
+
* - OHLC built from overnight gaps + intraday ranges, lognormal volume.
|
|
19
|
+
*
|
|
20
|
+
* Split: weekdays 2016-07-01 .. 2026-06-30; the FINAL 2 YEARS (>= 2024-07-01)
|
|
21
|
+
* go to fixtures/data/holdout/, the rest to fixtures/data/insample/.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { mkdir, writeFile } from 'node:fs/promises'
|
|
25
|
+
import { join } from 'node:path'
|
|
26
|
+
import { pathToFileURL } from 'node:url'
|
|
27
|
+
import { HOLDOUT_DIR, HOLDOUT_START, IN_SAMPLE_DIR, INDEX_TICKER } from './data.ts'
|
|
28
|
+
import { mulberry32 } from './windows.ts'
|
|
29
|
+
|
|
30
|
+
export const FIXTURE_SEED = 20260722
|
|
31
|
+
|
|
32
|
+
const START_DATE = '2016-07-01'
|
|
33
|
+
const END_DATE = '2026-06-30'
|
|
34
|
+
const DAYS_PER_YEAR = 252
|
|
35
|
+
|
|
36
|
+
interface Regime {
|
|
37
|
+
driftAnnual: number
|
|
38
|
+
volAnnual: number
|
|
39
|
+
stayProb: number
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const CALM: Regime = { driftAnnual: 0.09, volAnnual: 0.13, stayProb: 0.985 }
|
|
43
|
+
const STRESS: Regime = { driftAnnual: -0.12, volAnnual: 0.32, stayProb: 0.92 }
|
|
44
|
+
|
|
45
|
+
/** Seeded standard normal via Box-Muller over mulberry32. */
|
|
46
|
+
function gaussian(rand: () => number): () => number {
|
|
47
|
+
let spare: number | null = null
|
|
48
|
+
return () => {
|
|
49
|
+
if (spare !== null) {
|
|
50
|
+
const v = spare
|
|
51
|
+
spare = null
|
|
52
|
+
return v
|
|
53
|
+
}
|
|
54
|
+
let u = 0
|
|
55
|
+
let v = 0
|
|
56
|
+
while (u === 0) u = rand()
|
|
57
|
+
while (v === 0) v = rand()
|
|
58
|
+
const mag = Math.sqrt(-2 * Math.log(u))
|
|
59
|
+
spare = mag * Math.sin(2 * Math.PI * v)
|
|
60
|
+
return mag * Math.cos(2 * Math.PI * v)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function weekdays(startDate: string, endDate: string): string[] {
|
|
65
|
+
const dates: string[] = []
|
|
66
|
+
const d = new Date(`${startDate}T00:00:00Z`)
|
|
67
|
+
const end = new Date(`${endDate}T00:00:00Z`)
|
|
68
|
+
while (d.getTime() <= end.getTime()) {
|
|
69
|
+
const dow = d.getUTCDay()
|
|
70
|
+
if (dow !== 0 && dow !== 6) dates.push(d.toISOString().slice(0, 10))
|
|
71
|
+
d.setUTCDate(d.getUTCDate() + 1)
|
|
72
|
+
}
|
|
73
|
+
return dates
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
interface SyntheticSeries {
|
|
77
|
+
ticker: string
|
|
78
|
+
rows: Array<{ date: string; open: number; high: number; low: number; close: number; volume: number }>
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function generateUniverse(seed: number): SyntheticSeries[] {
|
|
82
|
+
const dates = weekdays(START_DATE, END_DATE)
|
|
83
|
+
const T = dates.length
|
|
84
|
+
const rand = mulberry32(seed)
|
|
85
|
+
const norm = gaussian(rand)
|
|
86
|
+
|
|
87
|
+
// Market factor daily log-returns under the regime chain.
|
|
88
|
+
let regime: Regime = CALM
|
|
89
|
+
const marketLogRet: number[] = new Array(T).fill(0)
|
|
90
|
+
for (let t = 1; t < T; t++) {
|
|
91
|
+
if (rand() > regime.stayProb) regime = regime === CALM ? STRESS : CALM
|
|
92
|
+
marketLogRet[t] = regime.driftAnnual / DAYS_PER_YEAR + (regime.volAnnual / Math.sqrt(DAYS_PER_YEAR)) * norm()
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const specs = [
|
|
96
|
+
{ ticker: INDEX_TICKER, beta: 1, idioVol: 0.02, alpha: 0, jumpProb: 0, basePrice: 200 },
|
|
97
|
+
...Array.from({ length: 10 }, (_, i) => ({
|
|
98
|
+
ticker: `S${String(i + 1).padStart(2, '0')}`,
|
|
99
|
+
beta: 0.6 + rand() * 0.9,
|
|
100
|
+
idioVol: 0.15 + rand() * 0.2,
|
|
101
|
+
alpha: -0.02 + rand() * 0.08,
|
|
102
|
+
jumpProb: 0.004,
|
|
103
|
+
basePrice: 20 + rand() * 180,
|
|
104
|
+
})),
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
return specs.map((spec) => {
|
|
108
|
+
const dailyIdio = spec.idioVol / Math.sqrt(DAYS_PER_YEAR)
|
|
109
|
+
const closes: number[] = new Array(T).fill(0)
|
|
110
|
+
closes[0] = spec.basePrice
|
|
111
|
+
for (let t = 1; t < T; t++) {
|
|
112
|
+
let logRet = spec.alpha / DAYS_PER_YEAR + spec.beta * marketLogRet[t]! + dailyIdio * norm()
|
|
113
|
+
if (spec.jumpProb > 0 && rand() < spec.jumpProb) {
|
|
114
|
+
logRet += (rand() < 0.5 ? -1 : 1) * (3 + rand() * 5) * dailyIdio
|
|
115
|
+
}
|
|
116
|
+
closes[t] = closes[t - 1]! * Math.exp(logRet)
|
|
117
|
+
}
|
|
118
|
+
const baseVolume = 1e6 * (0.5 + rand() * 4)
|
|
119
|
+
const rows = dates.map((date, t) => {
|
|
120
|
+
const prevClose = t === 0 ? closes[0]! : closes[t - 1]!
|
|
121
|
+
const close = closes[t]!
|
|
122
|
+
const gap = t === 0 ? 0 : 0.3 * dailyIdio * norm()
|
|
123
|
+
const open = round4(prevClose * Math.exp(gap))
|
|
124
|
+
const range = Math.abs(norm()) * 0.5 * dailyIdio
|
|
125
|
+
const high = round4(Math.max(open, close) * Math.exp(range))
|
|
126
|
+
const low = round4(Math.min(open, close) * Math.exp(-Math.abs(norm()) * 0.5 * dailyIdio))
|
|
127
|
+
const ret = t === 0 ? 0 : Math.abs(close / prevClose - 1)
|
|
128
|
+
const volume = Math.round(baseVolume * (1 + 3 * ret) * Math.exp(0.2 * norm()))
|
|
129
|
+
return { date, open, high, low, close: round4(close), volume }
|
|
130
|
+
})
|
|
131
|
+
return { ticker: spec.ticker, rows }
|
|
132
|
+
})
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const round4 = (x: number): number => Math.round(x * 10_000) / 10_000
|
|
136
|
+
|
|
137
|
+
function toCsv(series: SyntheticSeries, filter: (date: string) => boolean): string {
|
|
138
|
+
const lines = ['Date,Open,High,Low,Close,Volume']
|
|
139
|
+
for (const r of series.rows) {
|
|
140
|
+
if (!filter(r.date)) continue
|
|
141
|
+
lines.push(`${r.date},${r.open},${r.high},${r.low},${r.close},${r.volume}`)
|
|
142
|
+
}
|
|
143
|
+
return lines.join('\n') + '\n'
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
|
|
147
|
+
|
|
148
|
+
if (isMain) {
|
|
149
|
+
const universe = generateUniverse(FIXTURE_SEED)
|
|
150
|
+
await mkdir(IN_SAMPLE_DIR, { recursive: true })
|
|
151
|
+
await mkdir(HOLDOUT_DIR, { recursive: true })
|
|
152
|
+
for (const series of universe) {
|
|
153
|
+
await writeFile(join(IN_SAMPLE_DIR, `${series.ticker}.csv`), toCsv(series, (d) => d < HOLDOUT_START))
|
|
154
|
+
await writeFile(join(HOLDOUT_DIR, `${series.ticker}.csv`), toCsv(series, (d) => d >= HOLDOUT_START))
|
|
155
|
+
}
|
|
156
|
+
const inDays = universe[0]!.rows.filter((r) => r.date < HOLDOUT_START).length
|
|
157
|
+
const outDays = universe[0]!.rows.length - inDays
|
|
158
|
+
console.log(
|
|
159
|
+
`wrote ${universe.length} tickers: ${inDays} in-sample days (< ${HOLDOUT_START}) + ${outDays} holdout days, seed ${FIXTURE_SEED}`,
|
|
160
|
+
)
|
|
161
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest'
|
|
2
|
+
import {
|
|
3
|
+
BASE_EXCESS_SHARPE,
|
|
4
|
+
MULTIPLICITY_SPREAD,
|
|
5
|
+
decideAcceptance,
|
|
6
|
+
requiredExcessSharpe,
|
|
7
|
+
} from './multiplicity.ts'
|
|
8
|
+
|
|
9
|
+
describe('requiredExcessSharpe — the rising bar', () => {
|
|
10
|
+
it('equals the base floor for the very first candidate', () => {
|
|
11
|
+
expect(requiredExcessSharpe(1)).toBeCloseTo(BASE_EXCESS_SHARPE, 12)
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
it('matches the documented formula base + spread * sqrt(2 ln n)', () => {
|
|
15
|
+
for (const n of [2, 4, 10, 50, 200]) {
|
|
16
|
+
expect(requiredExcessSharpe(n)).toBeCloseTo(BASE_EXCESS_SHARPE + MULTIPLICITY_SPREAD * Math.sqrt(2 * Math.log(n)), 12)
|
|
17
|
+
}
|
|
18
|
+
// Spot values so a silent constant change fails a test, not just a diff.
|
|
19
|
+
expect(requiredExcessSharpe(10)).toBeCloseTo(0.1 + 0.15 * Math.sqrt(2 * Math.log(10)), 12)
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
it('is strictly increasing in candidates tried', () => {
|
|
23
|
+
let prev = requiredExcessSharpe(1)
|
|
24
|
+
for (let n = 2; n <= 100; n++) {
|
|
25
|
+
const cur = requiredExcessSharpe(n)
|
|
26
|
+
expect(cur).toBeGreaterThan(prev)
|
|
27
|
+
prev = cur
|
|
28
|
+
}
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
it('rejects a non-positive or fractional try count', () => {
|
|
32
|
+
expect(() => requiredExcessSharpe(0)).toThrow(/positive integer/)
|
|
33
|
+
expect(() => requiredExcessSharpe(2.5)).toThrow(/positive integer/)
|
|
34
|
+
})
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
describe('decideAcceptance — consistency AND multiplicity', () => {
|
|
38
|
+
const goodWindows = [0.9, 0.8, 1.0, 0.7, 0.85, 0.95, 0.6, 0.75]
|
|
39
|
+
|
|
40
|
+
it('accepts a consistent, large edge early in the campaign', () => {
|
|
41
|
+
const d = decideAcceptance({ perWindowExcess: goodWindows, nTried: 1 })
|
|
42
|
+
expect(d.accepted).toBe(true)
|
|
43
|
+
expect(d.wins).toBe(8)
|
|
44
|
+
expect(d.requiredWins).toBe(6)
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
it('rejects on consistency when the edge is one lucky stretch (5/8 wins)', () => {
|
|
48
|
+
const d = decideAcceptance({ perWindowExcess: [3, 3, 3, -0.1, -0.1, -0.1, 0.5, 0.5], nTried: 1 })
|
|
49
|
+
expect(d.accepted).toBe(false)
|
|
50
|
+
expect(d.wins).toBe(5)
|
|
51
|
+
expect(d.reasons[0]).toMatch(/consistency/)
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
it('the SAME evidence passes at n=1 and fails once enough candidates were tried', () => {
|
|
55
|
+
// Mean excess 0.30: above the n=1 bar (0.10), below the n=50 bar (~0.52).
|
|
56
|
+
const marginal = [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3]
|
|
57
|
+
expect(decideAcceptance({ perWindowExcess: marginal, nTried: 1 }).accepted).toBe(true)
|
|
58
|
+
const late = decideAcceptance({ perWindowExcess: marginal, nTried: 50 })
|
|
59
|
+
expect(late.accepted).toBe(false)
|
|
60
|
+
expect(late.reasons[0]).toMatch(/multiplicity/)
|
|
61
|
+
expect(late.threshold).toBeGreaterThan(0.3)
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('fails closed on non-finite evidence and empty windows', () => {
|
|
65
|
+
expect(() => decideAcceptance({ perWindowExcess: [0.5, Number.NaN], nTried: 1 })).toThrow(/non-finite/)
|
|
66
|
+
expect(() => decideAcceptance({ perWindowExcess: [], nTried: 1 })).toThrow(/no windows/)
|
|
67
|
+
})
|
|
68
|
+
})
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multiplicity-adjusted acceptance: the bar RISES with every candidate tried.
|
|
3
|
+
*
|
|
4
|
+
* Why: if you test N random strategies against the same data, the best one
|
|
5
|
+
* looks good by luck alone. Under the null (no skill), the expected maximum
|
|
6
|
+
* Sharpe across N independent tries grows like sqrt(2 ln N) times the
|
|
7
|
+
* cross-trial dispersion — that is the term the Deflated Sharpe Ratio of
|
|
8
|
+
* Bailey & Lopez de Prado ("The Deflated Sharpe Ratio", Journal of Portfolio
|
|
9
|
+
* Management, 2014) corrects for. We implement the simple, auditable version:
|
|
10
|
+
*
|
|
11
|
+
* requiredExcessSharpe(n) = BASE + SPREAD * sqrt(2 * ln(n))
|
|
12
|
+
*
|
|
13
|
+
* where n counts EVERY candidate ever tried this campaign (including ones
|
|
14
|
+
* killed by the leak audit — they were still draws from the search), BASE is
|
|
15
|
+
* the floor a single try must clear, and SPREAD calibrates how fast luck
|
|
16
|
+
* accumulates. Every tried candidate is a permanent notebook row, so n can
|
|
17
|
+
* never be quietly reset.
|
|
18
|
+
*
|
|
19
|
+
* A candidate is accepted only if BOTH hold:
|
|
20
|
+
* 1. it beats the best pinned baseline's Sharpe in >= ceil(minWinFraction*K)
|
|
21
|
+
* of the K in-sample windows (consistency, not one lucky stretch), and
|
|
22
|
+
* 2. its mean excess Sharpe across the K windows >= requiredExcessSharpe(n).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** Annualized excess-Sharpe floor for the very first candidate (n = 1). */
|
|
26
|
+
export const BASE_EXCESS_SHARPE = 0.1
|
|
27
|
+
/** Luck-accumulation rate: how fast the bar rises with tries. */
|
|
28
|
+
export const MULTIPLICITY_SPREAD = 0.15
|
|
29
|
+
|
|
30
|
+
export function requiredExcessSharpe(nTried: number): number {
|
|
31
|
+
if (!Number.isInteger(nTried) || nTried < 1) {
|
|
32
|
+
throw new Error(`requiredExcessSharpe: nTried must be a positive integer, got ${nTried}`)
|
|
33
|
+
}
|
|
34
|
+
return BASE_EXCESS_SHARPE + MULTIPLICITY_SPREAD * Math.sqrt(2 * Math.log(nTried))
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface AcceptanceDecision {
|
|
38
|
+
accepted: boolean
|
|
39
|
+
wins: number
|
|
40
|
+
windows: number
|
|
41
|
+
requiredWins: number
|
|
42
|
+
meanExcessSharpe: number
|
|
43
|
+
threshold: number
|
|
44
|
+
nTried: number
|
|
45
|
+
reasons: string[]
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface AcceptanceInput {
|
|
49
|
+
/** Candidate Sharpe minus best-baseline Sharpe, one entry per window. */
|
|
50
|
+
perWindowExcess: number[]
|
|
51
|
+
/** Total candidates tried this campaign INCLUDING this one. */
|
|
52
|
+
nTried: number
|
|
53
|
+
/** Fraction of windows the candidate must win. Default 0.75 (6 of 8). */
|
|
54
|
+
minWinFraction?: number
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function decideAcceptance(input: AcceptanceInput): AcceptanceDecision {
|
|
58
|
+
const { perWindowExcess, nTried } = input
|
|
59
|
+
const windows = perWindowExcess.length
|
|
60
|
+
if (windows === 0) throw new Error('decideAcceptance: no windows')
|
|
61
|
+
for (const e of perWindowExcess) {
|
|
62
|
+
if (!Number.isFinite(e)) throw new Error('decideAcceptance: non-finite excess Sharpe — fail-closed')
|
|
63
|
+
}
|
|
64
|
+
const minWinFraction = input.minWinFraction ?? 0.75
|
|
65
|
+
const requiredWins = Math.ceil(minWinFraction * windows)
|
|
66
|
+
const wins = perWindowExcess.filter((e) => e > 0).length
|
|
67
|
+
const meanExcessSharpe = perWindowExcess.reduce((s, e) => s + e, 0) / windows
|
|
68
|
+
const threshold = requiredExcessSharpe(nTried)
|
|
69
|
+
const reasons: string[] = []
|
|
70
|
+
if (wins < requiredWins) {
|
|
71
|
+
reasons.push(`consistency: beat the best baseline in only ${wins}/${windows} windows (need ${requiredWins})`)
|
|
72
|
+
}
|
|
73
|
+
if (meanExcessSharpe < threshold) {
|
|
74
|
+
reasons.push(
|
|
75
|
+
`multiplicity: mean excess Sharpe ${meanExcessSharpe.toFixed(3)} < required ${threshold.toFixed(3)} ` +
|
|
76
|
+
`(bar after ${nTried} tried candidate${nTried === 1 ? '' : 's'})`,
|
|
77
|
+
)
|
|
78
|
+
}
|
|
79
|
+
const accepted = reasons.length === 0
|
|
80
|
+
if (accepted) {
|
|
81
|
+
reasons.push(
|
|
82
|
+
`accepted: won ${wins}/${windows} windows, mean excess Sharpe ${meanExcessSharpe.toFixed(3)} ` +
|
|
83
|
+
`>= ${threshold.toFixed(3)} at n=${nTried}`,
|
|
84
|
+
)
|
|
85
|
+
}
|
|
86
|
+
return { accepted, wins, windows, requiredWins, meanExcessSharpe, threshold, nTried, reasons }
|
|
87
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 3 stub — Nautilus Trader certification adapter (NOT IMPLEMENTED).
|
|
3
|
+
*
|
|
4
|
+
* Per the quant-arena v2 decision memo, the final certification tier runs a
|
|
5
|
+
* winning strategy's decision record through Nautilus Trader's event-driven
|
|
6
|
+
* engine (real OMS semantics: order lifecycle, partial fills, latency
|
|
7
|
+
* models) as an independent confirmation of the vectorbt score before
|
|
8
|
+
* anything is promoted. Phases 1-2 (this PR) end at the vectorbt worker;
|
|
9
|
+
* this seam exists so the campaign plumbing has a stable name to call.
|
|
10
|
+
*
|
|
11
|
+
* TODO(phase-3, quant-arena v2 memo): implement as a python worker sibling
|
|
12
|
+
* of python/vbt-worker.py (same JSONL protocol shape, own pinned uv env),
|
|
13
|
+
* translating the OMS `Order` stream from driver.ts — not raw weights —
|
|
14
|
+
* into Nautilus order submissions, and reconciling final equity against the
|
|
15
|
+
* vectorbt curve within a documented tolerance.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { Order } from './types.ts'
|
|
19
|
+
|
|
20
|
+
export interface NautilusCertification {
|
|
21
|
+
finalEquity: number
|
|
22
|
+
ordersAccepted: number
|
|
23
|
+
ordersRejected: number
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function certifyWithNautilus(_orders: Order[]): Promise<NautilusCertification> {
|
|
27
|
+
throw new Error(
|
|
28
|
+
'quant-arena: Nautilus certification is Phase 3 of the v2 memo and is not implemented yet — ' +
|
|
29
|
+
'the vectorbt worker (vbt-client.ts) is the current official scorer.',
|
|
30
|
+
)
|
|
31
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shared OMS rebalancer — the ONLY component that turns a strategy's
|
|
3
|
+
* target weights into orders. Strategies never construct `Order`s; keeping
|
|
4
|
+
* sizing in one place means every candidate and baseline trades under the
|
|
5
|
+
* same rules, and the leak/contract audits only ever have to reason about
|
|
6
|
+
* weights.
|
|
7
|
+
*
|
|
8
|
+
* Sizing (LEAN `CalculateOrderQuantity` style, long-only v2):
|
|
9
|
+
* qty = (targetWeight * equity - currentPositionValue) / price
|
|
10
|
+
* at decision-time (close of day t) prices. Positive -> buy, negative ->
|
|
11
|
+
* sell. Quantities are fractional: the backtest engines track dollar
|
|
12
|
+
* positions, so lot rounding would only introduce parity noise between the
|
|
13
|
+
* TS prefilter and the vectorbt scorer. No fee buffer is reserved (fees are
|
|
14
|
+
* financed at 0% by the engines — conservative, never flattering).
|
|
15
|
+
*
|
|
16
|
+
* Long-only enforcement is fail-closed: a negative target weight, a weight
|
|
17
|
+
* sum > 1, or an unknown symbol throws rather than clamping.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import type { Order, Rebalancer, TargetPosition } from './types.ts'
|
|
21
|
+
|
|
22
|
+
const WEIGHT_EPS = 1e-9
|
|
23
|
+
/** Trades below this fraction of equity are dust — no order is emitted. */
|
|
24
|
+
const MIN_TRADE_FRACTION = 1e-9
|
|
25
|
+
|
|
26
|
+
/** Validate targets and expand them to a full per-symbol weight vector
|
|
27
|
+
* (omitted symbols -> 0). Throws on contract violations. */
|
|
28
|
+
export function targetsToWeights(targets: TargetPosition[], symbols: string[]): number[] {
|
|
29
|
+
const index = new Map(symbols.map((s, k) => [s, k]))
|
|
30
|
+
const weights = new Array<number>(symbols.length).fill(0)
|
|
31
|
+
const seen = new Set<string>()
|
|
32
|
+
let sum = 0
|
|
33
|
+
for (const target of targets) {
|
|
34
|
+
const k = index.get(target.symbol)
|
|
35
|
+
if (k === undefined) {
|
|
36
|
+
throw new Error(`oms: target names unknown symbol '${target.symbol}' (universe: ${symbols.join(', ')})`)
|
|
37
|
+
}
|
|
38
|
+
if (seen.has(target.symbol)) {
|
|
39
|
+
throw new Error(`oms: symbol '${target.symbol}' targeted twice in one rebalance`)
|
|
40
|
+
}
|
|
41
|
+
seen.add(target.symbol)
|
|
42
|
+
if (!Number.isFinite(target.weight)) throw new Error(`oms: non-finite weight for '${target.symbol}'`)
|
|
43
|
+
if (target.weight < -WEIGHT_EPS) {
|
|
44
|
+
throw new Error(`oms: negative weight ${target.weight} for '${target.symbol}' — long-only, no shorting`)
|
|
45
|
+
}
|
|
46
|
+
weights[k] = Math.max(target.weight, 0)
|
|
47
|
+
sum += weights[k]!
|
|
48
|
+
}
|
|
49
|
+
if (sum > 1 + 1e-6) {
|
|
50
|
+
throw new Error(`oms: target weights sum ${sum} > 1 — no leverage`)
|
|
51
|
+
}
|
|
52
|
+
return weights
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The one shared rebalancer. See module header for the sizing rule. */
|
|
56
|
+
export const rebalance: Rebalancer = ({ targets, symbols, equity, prices, positionValues, t, tag }) => {
|
|
57
|
+
if (prices.length !== symbols.length || positionValues.length !== symbols.length) {
|
|
58
|
+
throw new Error(
|
|
59
|
+
`oms: state arrays disagree with the universe (${symbols.length} symbols, ` +
|
|
60
|
+
`${prices.length} prices, ${positionValues.length} positions)`,
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
if (!Number.isFinite(equity) || equity <= 0) throw new Error(`oms: nonpositive equity ${equity}`)
|
|
64
|
+
const weights = targetsToWeights(targets, symbols)
|
|
65
|
+
const orders: Order[] = []
|
|
66
|
+
for (let k = 0; k < symbols.length; k++) {
|
|
67
|
+
const price = prices[k]!
|
|
68
|
+
if (!Number.isFinite(price) || price <= 0) throw new Error(`oms: nonpositive price for '${symbols[k]}'`)
|
|
69
|
+
const desiredValue = weights[k]! * equity
|
|
70
|
+
const deltaValue = desiredValue - positionValues[k]!
|
|
71
|
+
if (Math.abs(deltaValue) <= MIN_TRADE_FRACTION * equity) continue
|
|
72
|
+
const qty = Math.abs(deltaValue) / price
|
|
73
|
+
const side = deltaValue > 0 ? 'buy' : 'sell'
|
|
74
|
+
if (side === 'sell' && deltaValue < -positionValues[k]! - MIN_TRADE_FRACTION * equity) {
|
|
75
|
+
// Cannot happen when weights >= 0, but keep the invariant explicit:
|
|
76
|
+
// a sell may never exceed the current long position.
|
|
77
|
+
throw new Error(`oms: sell of ${qty} '${symbols[k]}' exceeds current position — long-only`)
|
|
78
|
+
}
|
|
79
|
+
orders.push({
|
|
80
|
+
clientOrderId: `qa-t${t}-${symbols[k]}`,
|
|
81
|
+
symbol: symbols[k]!,
|
|
82
|
+
side,
|
|
83
|
+
qty,
|
|
84
|
+
type: 'market',
|
|
85
|
+
tif: 'day',
|
|
86
|
+
...(tag !== undefined ? { tag } : {}),
|
|
87
|
+
})
|
|
88
|
+
}
|
|
89
|
+
return orders
|
|
90
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "quant-researcher",
|
|
3
|
+
"description": "Strategy-author lens that speaks quant language: regimes, turnover, capacity, transaction costs. Same claude harness as default-author; only the system prompt differs, so any performance delta is attributable to the lens.",
|
|
4
|
+
"prompt": {
|
|
5
|
+
"systemPrompt": "You are a systematic-trading researcher at a mid-frequency equity desk. You think in regimes (trend vs chop, calm vs stress), you treat turnover as a cost center (every rebalance pays spread + impact), you size positions for capacity, and you are allergic to backtest overfitting — you prefer one robust effect over five fitted parameters. When you write a strategy you state, in comments, the economic rationale the rule is harvesting and why it should survive out-of-sample."
|
|
6
|
+
}
|
|
7
|
+
}
|