@tangle-network/agent-bench 0.11.2 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/HARNESS.md +6 -2
  3. package/README.md +1 -4
  4. package/dist/benchmarks/swe-bench.js +4 -9
  5. package/dist/benchmarks/swe-bench.js.map +1 -1
  6. package/package.json +5 -5
  7. package/scripts/run-package-tests.mjs +2 -2
  8. package/src/benchmarks/swe-bench.test.mts +49 -0
  9. package/src/benchmarks/swe-bench.ts +4 -9
  10. package/src/quant-arena/README.md +0 -144
  11. package/src/quant-arena/backtest.test.mts +0 -135
  12. package/src/quant-arena/backtest.ts +0 -218
  13. package/src/quant-arena/data.test.mts +0 -44
  14. package/src/quant-arena/data.ts +0 -141
  15. package/src/quant-arena/driver.test.mts +0 -253
  16. package/src/quant-arena/driver.ts +0 -219
  17. package/src/quant-arena/fixtures/data/PROVENANCE.md +0 -26
  18. package/src/quant-arena/fixtures/data/holdout/IDX.csv +0 -523
  19. package/src/quant-arena/fixtures/data/holdout/S01.csv +0 -523
  20. package/src/quant-arena/fixtures/data/holdout/S02.csv +0 -523
  21. package/src/quant-arena/fixtures/data/holdout/S03.csv +0 -523
  22. package/src/quant-arena/fixtures/data/holdout/S04.csv +0 -523
  23. package/src/quant-arena/fixtures/data/holdout/S05.csv +0 -523
  24. package/src/quant-arena/fixtures/data/holdout/S06.csv +0 -523
  25. package/src/quant-arena/fixtures/data/holdout/S07.csv +0 -523
  26. package/src/quant-arena/fixtures/data/holdout/S08.csv +0 -523
  27. package/src/quant-arena/fixtures/data/holdout/S09.csv +0 -523
  28. package/src/quant-arena/fixtures/data/holdout/S10.csv +0 -523
  29. package/src/quant-arena/fixtures/data/insample/IDX.csv +0 -2087
  30. package/src/quant-arena/fixtures/data/insample/S01.csv +0 -2087
  31. package/src/quant-arena/fixtures/data/insample/S02.csv +0 -2087
  32. package/src/quant-arena/fixtures/data/insample/S03.csv +0 -2087
  33. package/src/quant-arena/fixtures/data/insample/S04.csv +0 -2087
  34. package/src/quant-arena/fixtures/data/insample/S05.csv +0 -2087
  35. package/src/quant-arena/fixtures/data/insample/S06.csv +0 -2087
  36. package/src/quant-arena/fixtures/data/insample/S07.csv +0 -2087
  37. package/src/quant-arena/fixtures/data/insample/S08.csv +0 -2087
  38. package/src/quant-arena/fixtures/data/insample/S09.csv +0 -2087
  39. package/src/quant-arena/fixtures/data/insample/S10.csv +0 -2087
  40. package/src/quant-arena/fixtures/demo-campaign/cost-ledger.jsonl +0 -16
  41. package/src/quant-arena/fixtures/demo-campaign/notebook.jsonl +0 -5
  42. package/src/quant-arena/fixtures/demo-campaign/rollout-manifest.json +0 -171
  43. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-001-default-author/strategy.ts +0 -119
  44. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-002-default-author/strategy.ts +0 -119
  45. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-003-quant-researcher/strategy.ts +0 -105
  46. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts +0 -102
  47. package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl +0 -4
  48. package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl +0 -2
  49. package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json +0 -84
  50. package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts +0 -117
  51. package/src/quant-arena/holdout-certify.mts +0 -206
  52. package/src/quant-arena/holdout-certify.test.mts +0 -82
  53. package/src/quant-arena/leak-audit.test.mts +0 -79
  54. package/src/quant-arena/leak-audit.ts +0 -95
  55. package/src/quant-arena/make-fixtures.mts +0 -161
  56. package/src/quant-arena/multiplicity.test.mts +0 -68
  57. package/src/quant-arena/multiplicity.ts +0 -87
  58. package/src/quant-arena/nautilus-certify.ts +0 -31
  59. package/src/quant-arena/oms.ts +0 -90
  60. package/src/quant-arena/profiles/quant-researcher.profile.json +0 -12
  61. package/src/quant-arena/python/pyproject.toml +0 -8
  62. package/src/quant-arena/python/uv.lock +0 -1297
  63. package/src/quant-arena/python/vbt-worker.py +0 -192
  64. package/src/quant-arena/quant-loop.mts +0 -840
  65. package/src/quant-arena/quant-loop.test.mts +0 -75
  66. package/src/quant-arena/strategies/buy-hold-index/strategy.ts +0 -11
  67. package/src/quant-arena/strategies/equal-weight/strategy.ts +0 -20
  68. package/src/quant-arena/strategies/sma-crossover/strategy.ts +0 -42
  69. package/src/quant-arena/types.ts +0 -133
  70. package/src/quant-arena/vbt-client.ts +0 -321
  71. package/src/quant-arena/vbt-parity.test.mts +0 -183
  72. package/src/quant-arena/windows.test.mts +0 -45
  73. package/src/quant-arena/windows.ts +0 -54
  74. package/src/rollout-ledger/backfill-swe-arena.mts +0 -610
  75. package/src/rollout-ledger/backfill-swe-arena.test.mts +0 -347
  76. package/src/rollout-ledger/settle-capture.mts +0 -448
  77. package/src/rollout-ledger/settle-capture.test.mts +0 -270
  78. package/src/swe-arena/activation.mts +0 -225
  79. package/src/swe-arena/activation.test.mts +0 -300
  80. package/src/swe-arena/analyze.ts +0 -211
  81. package/src/swe-arena/arms.ts +0 -862
  82. package/src/swe-arena/bootstrap-meta.mts +0 -188
  83. package/src/swe-arena/bootstrap-meta.test.mts +0 -51
  84. package/src/swe-arena/briefing.mts +0 -217
  85. package/src/swe-arena/briefing.test.mts +0 -179
  86. package/src/swe-arena/calibrate.ts +0 -217
  87. package/src/swe-arena/capabilities.mts +0 -76
  88. package/src/swe-arena/capabilities.test.mts +0 -57
  89. package/src/swe-arena/capacity.ts +0 -198
  90. package/src/swe-arena/cell-evidence.mts +0 -437
  91. package/src/swe-arena/cell-evidence.test.mts +0 -248
  92. package/src/swe-arena/diagnosis-ensemble.test.mts +0 -210
  93. package/src/swe-arena/diagnosis-ensemble.ts +0 -523
  94. package/src/swe-arena/execution.test.mts +0 -1171
  95. package/src/swe-arena/factory-command-container.ts +0 -284
  96. package/src/swe-arena/factory-judge-child.mts +0 -228
  97. package/src/swe-arena/factory.test.mts +0 -645
  98. package/src/swe-arena/fixtures/analyze.py +0 -80
  99. package/src/swe-arena/fixtures/excludes.txt +0 -8
  100. package/src/swe-arena/fixtures/factory/agent-eval-309/calibration.md +0 -51
  101. package/src/swe-arena/fixtures/factory/agent-eval-309/manifest.json +0 -29
  102. package/src/swe-arena/fixtures/factory/agent-eval-309/spec.md +0 -64
  103. package/src/swe-arena/fixtures/factory/agent-runtime-232/calibration.md +0 -48
  104. package/src/swe-arena/fixtures/factory/agent-runtime-232/manifest.json +0 -29
  105. package/src/swe-arena/fixtures/factory/agent-runtime-232/spec.md +0 -48
  106. package/src/swe-arena/fixtures/factory/loops-28/calibration.md +0 -47
  107. package/src/swe-arena/fixtures/factory/loops-28/manifest.json +0 -30
  108. package/src/swe-arena/fixtures/factory/loops-28/spec.md +0 -50
  109. package/src/swe-arena/fixtures/gen1-salvage/README.md +0 -45
  110. package/src/swe-arena/fixtures/gen1-salvage/cand0-e6d7361.diff +0 -116
  111. package/src/swe-arena/fixtures/gen1-salvage/cand1-76a8590.diff +0 -293
  112. package/src/swe-arena/fixtures/holdout-preregister.log +0 -12
  113. package/src/swe-arena/fixtures/holdout.json +0 -44
  114. package/src/swe-arena/fixtures/instances.json +0 -146
  115. package/src/swe-arena/fixtures/ledger.jsonl +0 -12
  116. package/src/swe-arena/fixtures/patches/pallets__flask-5014.solo.patch +0 -36
  117. package/src/swe-arena/fixtures/patches/pydata__xarray-4687.sup.patch +0 -33
  118. package/src/swe-arena/fixtures/rejudge.jsonl +0 -15
  119. package/src/swe-arena/fixtures/rematch.jsonl +0 -3
  120. package/src/swe-arena/fixtures/rematch2.jsonl +0 -3
  121. package/src/swe-arena/fixtures/rematch3.jsonl +0 -3
  122. package/src/swe-arena/fixtures/run-report/README.md +0 -43
  123. package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json +0 -173
  124. package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md +0 -100
  125. package/src/swe-arena/fixtures/run-report/gen3-rollup.json +0 -551
  126. package/src/swe-arena/fixtures/run-report/gen3-rollup.md +0 -64
  127. package/src/swe-arena/fixtures/sup-journal-true.json +0 -19
  128. package/src/swe-arena/fixtures/verify/astropy__astropy-13033.sh +0 -48
  129. package/src/swe-arena/fixtures/verify/django__django-11532.sh +0 -50
  130. package/src/swe-arena/fixtures/verify/matplotlib__matplotlib-20826.sh +0 -76
  131. package/src/swe-arena/fixtures/verify/pydata__xarray-4687.sh +0 -44
  132. package/src/swe-arena/fixtures/verify/pytest-dev__pytest-6197.sh +0 -32
  133. package/src/swe-arena/fixtures/verify/sphinx-doc__sphinx-9658.sh +0 -51
  134. package/src/swe-arena/fixtures/worker-tokens.json +0 -42
  135. package/src/swe-arena/fixtures.ts +0 -237
  136. package/src/swe-arena/gepa-seat.mts +0 -886
  137. package/src/swe-arena/gepa-seat.test.mts +0 -1136
  138. package/src/swe-arena/holdout-certify.mts +0 -408
  139. package/src/swe-arena/holdout-certify.test.mts +0 -160
  140. package/src/swe-arena/implementation-ref.test.mts +0 -64
  141. package/src/swe-arena/implementation-ref.ts +0 -62
  142. package/src/swe-arena/judge-child.mts +0 -37
  143. package/src/swe-arena/ledger-orphans.mts +0 -77
  144. package/src/swe-arena/ledger-orphans.test.mts +0 -149
  145. package/src/swe-arena/manifest.mts +0 -293
  146. package/src/swe-arena/manifest.test.mts +0 -169
  147. package/src/swe-arena/materialize.ts +0 -142
  148. package/src/swe-arena/outer-loop.mts +0 -2854
  149. package/src/swe-arena/outer-loop.test.mts +0 -714
  150. package/src/swe-arena/parity.test.mts +0 -87
  151. package/src/swe-arena/premeasured-from-cells.mts +0 -296
  152. package/src/swe-arena/premeasured-from-cells.test.mts +0 -201
  153. package/src/swe-arena/proc.test.mts +0 -172
  154. package/src/swe-arena/proc.ts +0 -260
  155. package/src/swe-arena/profiles/deepseek-author.profile.json +0 -12
  156. package/src/swe-arena/profiles/default-author.profile.json +0 -12
  157. package/src/swe-arena/proposer-fanout.mts +0 -736
  158. package/src/swe-arena/proposer-fanout.test.mts +0 -660
  159. package/src/swe-arena/proposer-provenance.mts +0 -176
  160. package/src/swe-arena/proposer-provenance.test.mts +0 -106
  161. package/src/swe-arena/reconcile.ts +0 -0
  162. package/src/swe-arena/replay.mts +0 -183
  163. package/src/swe-arena/replay.test.mts +0 -300
  164. package/src/swe-arena/run-experiment.mts +0 -729
  165. package/src/swe-arena/run-report.mts +0 -75
  166. package/src/swe-arena/run-supervisor.mjs +0 -297
  167. package/src/swe-arena/run-supervisor.test.mts +0 -539
  168. package/src/swe-arena/score-split.mts +0 -140
  169. package/src/swe-arena/score-split.test.mts +0 -123
  170. package/src/swe-arena/scratch-worktree-serialization.test.mts +0 -72
  171. package/src/swe-arena/scratch-worktree.test.mts +0 -56
  172. package/src/swe-arena/scratch-worktree.ts +0 -64
  173. package/src/swe-arena/serialized-judge.ts +0 -414
  174. package/src/swe-arena/types.ts +0 -218
@@ -1,218 +0,0 @@
1
- /**
2
- * Daily-bar event-loop backtester. Zero dependencies beyond the Node stdlib
3
- * (in fact: zero imports at all except the local types) so a reviewer can
4
- * audit the whole execution model in one file.
5
- *
6
- * Execution model (conservative by construction):
7
- * - A signal decided at the close of day t is FILLED at the open of day t+1.
8
- * Nothing trades on the bar that produced the decision.
9
- * - Every rebalance pays `costBps + slippageBps` (basis points, one-way) on
10
- * the dollars traded, deducted from cash at the fill.
11
- * - No shorting, no leverage: weights >= 0, sum <= 1. Violations THROW —
12
- * a strategy that asks for leverage is a broken candidate, not a clamped one.
13
- * - Cash earns 0. A fully-invested target therefore carries a small negative
14
- * cash balance equal to accumulated fees (fees financed at 0% — this only
15
- * ever understates performance, never flatters it).
16
- *
17
- * Determinism: pure function of (bars, signals, config). No clock, no RNG.
18
- */
19
-
20
- import type { Bar, Signal } from './types.ts'
21
-
22
- export interface BacktestConfig {
23
- /** One-way transaction cost, basis points of traded dollars. */
24
- costBps: number
25
- /** One-way slippage, basis points of traded dollars. */
26
- slippageBps: number
27
- }
28
-
29
- export const TRADING_DAYS_PER_YEAR = 252
30
-
31
- export interface RangeStats {
32
- /** First/last day index of the scored range (inclusive start, exclusive end). */
33
- start: number
34
- end: number
35
- days: number
36
- totalReturn: number
37
- maxDrawdown: number
38
- /** Annualized Sharpe of daily returns (rf = 0). 0 when volatility is 0. */
39
- sharpe: number
40
- /** Fills with |trade| > 1e-9 × equity, summed over the range. */
41
- tradeCount: number
42
- /** Σ traded dollars / equity at each rebalance (two-sided turnover). */
43
- turnover: number
44
- }
45
-
46
- export interface BacktestResult {
47
- dates: string[]
48
- /** Equity at each day's close; equity[0] = 1 (all cash at the first close). */
49
- equity: number[]
50
- /** dailyReturns[i] = equity[i+1] / equity[i] - 1. */
51
- dailyReturns: number[]
52
- /** Per-day fill count / turnover fraction (index = fill day). */
53
- fillsByDay: number[]
54
- turnoverByDay: number[]
55
- stats: RangeStats
56
- }
57
-
58
- const WEIGHT_EPS = 1e-9
59
-
60
- export function assertAligned(bars: Bar[][]): void {
61
- if (bars.length === 0) throw new Error('backtest: empty universe')
62
- const T = bars[0]!.length
63
- if (T < 2) throw new Error('backtest: need at least 2 days of bars')
64
- for (let k = 1; k < bars.length; k++) {
65
- if (bars[k]!.length !== T) {
66
- throw new Error(`backtest: ticker ${k} has ${bars[k]!.length} bars, ticker 0 has ${T} — unaligned`)
67
- }
68
- for (let t = 0; t < T; t++) {
69
- if (bars[k]![t]!.date !== bars[0]![t]!.date) {
70
- throw new Error(`backtest: date mismatch at t=${t}: ${bars[k]![t]!.date} vs ${bars[0]![t]!.date}`)
71
- }
72
- }
73
- }
74
- }
75
-
76
- /** Validate one signal against the contract. Throws on violation. */
77
- export function assertSignal(signal: Signal, tickers: number, totalDays: number): void {
78
- if (!Number.isInteger(signal.t) || signal.t < 0 || signal.t >= totalDays) {
79
- throw new Error(`backtest: signal.t=${signal.t} outside [0, ${totalDays})`)
80
- }
81
- if (signal.weights.length !== tickers) {
82
- throw new Error(`backtest: signal at t=${signal.t} has ${signal.weights.length} weights, universe has ${tickers}`)
83
- }
84
- let sum = 0
85
- for (const w of signal.weights) {
86
- if (!Number.isFinite(w)) throw new Error(`backtest: non-finite weight at t=${signal.t}`)
87
- if (w < -WEIGHT_EPS) throw new Error(`backtest: negative weight ${w} at t=${signal.t} — no shorting`)
88
- sum += w
89
- }
90
- if (sum > 1 + 1e-6) throw new Error(`backtest: weights sum ${sum} > 1 at t=${signal.t} — no leverage`)
91
- }
92
-
93
- /** Map of decision day -> target weights. A rebalance is executed ONLY at the
94
- * open following an emitted signal; between signals, positions drift with
95
- * prices (no silent daily re-targeting, no hidden fee drag). */
96
- export function fillSchedule(signals: Signal[], tickers: number, totalDays: number): Map<number, number[]> {
97
- const sorted = [...signals].sort((a, b) => a.t - b.t)
98
- for (let i = 1; i < sorted.length; i++) {
99
- if (sorted[i]!.t === sorted[i - 1]!.t) {
100
- throw new Error(`backtest: two signals share t=${sorted[i]!.t} — a strategy emits at most one target per day`)
101
- }
102
- }
103
- const schedule = new Map<number, number[]>()
104
- for (const s of sorted) {
105
- assertSignal(s, tickers, totalDays)
106
- schedule.set(s.t, s.weights)
107
- }
108
- return schedule
109
- }
110
-
111
- export function runBacktest(bars: Bar[][], signals: Signal[], config: BacktestConfig): BacktestResult {
112
- assertAligned(bars)
113
- const N = bars.length
114
- const T = bars[0]!.length
115
- const fills = fillSchedule(signals, N, T)
116
- const feeRate = (config.costBps + config.slippageBps) / 10_000
117
-
118
- const equity: number[] = new Array(T).fill(0)
119
- const fillsByDay: number[] = new Array(T).fill(0)
120
- const turnoverByDay: number[] = new Array(T).fill(0)
121
- let cash = 1
122
- const pos: number[] = new Array(N).fill(0)
123
- equity[0] = 1
124
-
125
- for (let t = 1; t < T; t++) {
126
- // Overnight: yesterday's close -> today's open.
127
- for (let k = 0; k < N; k++) {
128
- const prevClose = bars[k]![t - 1]!.close
129
- if (prevClose <= 0) throw new Error(`backtest: nonpositive close for ticker ${k} at t=${t - 1}`)
130
- pos[k] = pos[k]! * (bars[k]![t]!.open / prevClose)
131
- }
132
- // Fill the target decided at yesterday's close, at today's open.
133
- const target = fills.get(t - 1)
134
- if (target !== undefined) {
135
- let equityOpen = cash
136
- for (let k = 0; k < N; k++) equityOpen += pos[k]!
137
- if (equityOpen <= 0) throw new Error(`backtest: equity wiped out at t=${t}`)
138
- let traded = 0
139
- let fills = 0
140
- for (let k = 0; k < N; k++) {
141
- const desired = target[k]! * equityOpen
142
- const delta = desired - pos[k]!
143
- traded += Math.abs(delta)
144
- if (Math.abs(delta) > WEIGHT_EPS * equityOpen) fills++
145
- cash -= delta
146
- pos[k] = desired
147
- }
148
- const fee = traded * feeRate
149
- cash -= fee
150
- fillsByDay[t] = fills
151
- turnoverByDay[t] = traded / equityOpen
152
- }
153
- // Intraday: today's open -> today's close.
154
- for (let k = 0; k < N; k++) {
155
- const open = bars[k]![t]!.open
156
- if (open <= 0) throw new Error(`backtest: nonpositive open for ticker ${k} at t=${t}`)
157
- pos[k] = pos[k]! * (bars[k]![t]!.close / open)
158
- }
159
- let eq = cash
160
- for (let k = 0; k < N; k++) eq += pos[k]!
161
- equity[t] = eq
162
- }
163
-
164
- const dailyReturns: number[] = new Array(T - 1)
165
- for (let i = 0; i < T - 1; i++) dailyReturns[i] = equity[i + 1]! / equity[i]! - 1
166
-
167
- const result: BacktestResult = {
168
- dates: bars[0]!.map((b) => b.date),
169
- equity,
170
- dailyReturns,
171
- fillsByDay,
172
- turnoverByDay,
173
- stats: undefined as unknown as RangeStats,
174
- }
175
- result.stats = statsForRange(result, 0, T)
176
- return result
177
- }
178
-
179
- /** Score a sub-range [start, end) of an existing backtest — the walk-forward
180
- * window view. Positions carried into the window count; nothing after `end`
181
- * leaks in. */
182
- export function statsForRange(result: BacktestResult, start: number, end: number): RangeStats {
183
- const T = result.equity.length
184
- if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end > T || end - start < 2) {
185
- throw new Error(`statsForRange: bad range [${start}, ${end}) over ${T} days`)
186
- }
187
- const eq = result.equity.slice(start, end)
188
- const totalReturn = eq[eq.length - 1]! / eq[0]! - 1
189
- let peak = eq[0]!
190
- let maxDrawdown = 0
191
- for (const e of eq) {
192
- if (e > peak) peak = e
193
- const dd = (peak - e) / peak
194
- if (dd > maxDrawdown) maxDrawdown = dd
195
- }
196
- const returns = result.dailyReturns.slice(start, end - 1)
197
- const n = returns.length
198
- const mean = returns.reduce((s, r) => s + r, 0) / n
199
- const variance = n > 1 ? returns.reduce((s, r) => s + (r - mean) ** 2, 0) / (n - 1) : 0
200
- const std = Math.sqrt(variance)
201
- const sharpe = std > 0 ? (mean / std) * Math.sqrt(TRADING_DAYS_PER_YEAR) : 0
202
- let tradeCount = 0
203
- let turnover = 0
204
- for (let t = start + 1; t < end; t++) {
205
- tradeCount += result.fillsByDay[t]!
206
- turnover += result.turnoverByDay[t]!
207
- }
208
- return {
209
- start,
210
- end,
211
- days: end - start,
212
- totalReturn,
213
- maxDrawdown,
214
- sharpe,
215
- tradeCount,
216
- turnover,
217
- }
218
- }
@@ -1,44 +0,0 @@
1
- import { describe, expect, it } from 'vitest'
2
- import { readdir } from 'node:fs/promises'
3
- import {
4
- HOLDOUT_DIR,
5
- HOLDOUT_START,
6
- IN_SAMPLE_DIR,
7
- INDEX_TICKER,
8
- loadInSample,
9
- parseStooqCsv,
10
- } from './data.ts'
11
-
12
- describe('parseStooqCsv', () => {
13
- it('parses the vendored CSV format', () => {
14
- const bars = parseStooqCsv('Date,Open,High,Low,Close,Volume\n2020-01-02,10,11,9,10.5,12345\n2020-01-03,10.5,12,10,11,999\n', 'test')
15
- expect(bars).toHaveLength(2)
16
- expect(bars[0]).toEqual({ date: '2020-01-02', open: 10, high: 11, low: 9, close: 10.5, volume: 12345 })
17
- })
18
-
19
- it('fails loud on descending dates, bad prices, and foreign headers', () => {
20
- expect(() => parseStooqCsv('Date,Open,High,Low,Close,Volume\n2020-01-03,10,11,9,10,1\n2020-01-02,10,11,9,10,1\n', 't')).toThrow(/ascending/)
21
- expect(() => parseStooqCsv('Date,Open,High,Low,Close,Volume\n2020-01-02,0,11,9,10,1\n', 't')).toThrow(/nonpositive/)
22
- expect(() => parseStooqCsv('Ticker,Per,Date\nX,D,2020\n', 't')).toThrow(/header/)
23
- })
24
- })
25
-
26
- describe('vendored fixture integrity (in-sample side only — the holdout is not read here)', () => {
27
- it('in-sample and holdout are physically separate directories with the same tickers', async () => {
28
- const inFiles = (await readdir(IN_SAMPLE_DIR)).filter((f) => f.endsWith('.csv')).sort()
29
- const outFiles = (await readdir(HOLDOUT_DIR)).filter((f) => f.endsWith('.csv')).sort()
30
- expect(inFiles).toEqual(outFiles)
31
- expect(inFiles).toContain(`${INDEX_TICKER}.csv`)
32
- expect(inFiles.length).toBe(11)
33
- })
34
-
35
- it('loadInSample aligns the universe, leads with the index, and never crosses into the holdout era', async () => {
36
- const aligned = await loadInSample()
37
- expect(aligned.tickers[0]).toBe(INDEX_TICKER)
38
- expect(aligned.tickers).toHaveLength(11)
39
- expect(aligned.bars).toHaveLength(11)
40
- for (const series of aligned.bars) expect(series).toHaveLength(aligned.dates.length)
41
- expect(aligned.dates.length).toBeGreaterThan(1800)
42
- expect(aligned.dates[aligned.dates.length - 1]! < HOLDOUT_START).toBe(true)
43
- })
44
- })
@@ -1,141 +0,0 @@
1
- /**
2
- * Vendored daily-bar data: Stooq-format CSVs (Date,Open,High,Low,Close,Volume)
3
- * committed under fixtures/data/. Two physically separate directories:
4
- *
5
- * fixtures/data/insample/ — everything the campaign loop may read.
6
- * fixtures/data/holdout/ — the FINAL 2 years. Loaded ONLY by the
7
- * certification path (holdout-certify.mts).
8
- * quant-loop.mts never imports `loadHoldout`.
9
- *
10
- * See fixtures/data/PROVENANCE.md for where the series come from.
11
- */
12
-
13
- import { readdir, readFile } from 'node:fs/promises'
14
- import { basename, join } from 'node:path'
15
- import { fileURLToPath } from 'node:url'
16
- import type { Bar } from './types.ts'
17
-
18
- export const DATA_DIR = fileURLToPath(new URL('./fixtures/data', import.meta.url))
19
- export const IN_SAMPLE_DIR = join(DATA_DIR, 'insample')
20
- export const HOLDOUT_DIR = join(DATA_DIR, 'holdout')
21
- /** First holdout date — insample bars must all be strictly before this. */
22
- export const HOLDOUT_START = '2024-07-01'
23
- /** The benchmark index ticker; always bars[0] in the aligned universe. */
24
- export const INDEX_TICKER = 'IDX'
25
-
26
- export interface AlignedBars {
27
- /** tickers[0] === INDEX_TICKER; the rest sorted alphabetically. */
28
- tickers: string[]
29
- dates: string[]
30
- /** bars[k] belongs to tickers[k]; every series shares the date axis. */
31
- bars: Bar[][]
32
- }
33
-
34
- export function parseStooqCsv(text: string, source: string): Bar[] {
35
- const lines = text.split('\n').map((l) => l.trim()).filter((l) => l.length > 0)
36
- if (lines.length < 2) throw new Error(`${source}: empty CSV`)
37
- const header = lines[0]!.toLowerCase()
38
- if (!header.startsWith('date,open,high,low,close')) {
39
- throw new Error(`${source}: unexpected header '${lines[0]}'`)
40
- }
41
- const bars: Bar[] = []
42
- for (const line of lines.slice(1)) {
43
- const cells = line.split(',')
44
- if (cells.length < 5) throw new Error(`${source}: bad row '${line}'`)
45
- const [date, open, high, low, close, volume] = cells
46
- const bar: Bar = {
47
- date: date!,
48
- open: Number(open),
49
- high: Number(high),
50
- low: Number(low),
51
- close: Number(close),
52
- volume: volume !== undefined ? Number(volume) : 0,
53
- }
54
- if (!/^\d{4}-\d{2}-\d{2}$/.test(bar.date)) throw new Error(`${source}: bad date '${bar.date}'`)
55
- for (const v of [bar.open, bar.high, bar.low, bar.close]) {
56
- if (!Number.isFinite(v) || v <= 0) throw new Error(`${source}: nonpositive price on ${bar.date}`)
57
- }
58
- bars.push(bar)
59
- }
60
- for (let i = 1; i < bars.length; i++) {
61
- if (bars[i]!.date <= bars[i - 1]!.date) {
62
- throw new Error(`${source}: dates not strictly ascending at ${bars[i]!.date}`)
63
- }
64
- }
65
- return bars
66
- }
67
-
68
- /** Load every `<TICKER>.csv` in a directory and align on the intersection of
69
- * dates. The index ticker leads; the rest follow alphabetically. */
70
- export async function loadBarsDir(dir: string): Promise<AlignedBars> {
71
- const files = (await readdir(dir)).filter((f) => f.endsWith('.csv')).sort()
72
- if (files.length === 0) throw new Error(`loadBarsDir: no CSVs in ${dir}`)
73
- const byTicker = new Map<string, Bar[]>()
74
- for (const file of files) {
75
- const ticker = basename(file, '.csv')
76
- byTicker.set(ticker, parseStooqCsv(await readFile(join(dir, file), 'utf8'), join(dir, file)))
77
- }
78
- if (!byTicker.has(INDEX_TICKER)) {
79
- throw new Error(`loadBarsDir: ${dir} has no ${INDEX_TICKER}.csv — the universe needs its benchmark index`)
80
- }
81
- const tickers = [INDEX_TICKER, ...[...byTicker.keys()].filter((t) => t !== INDEX_TICKER).sort()]
82
- let shared: Set<string> | null = null
83
- for (const ticker of tickers) {
84
- const tickerDates = new Set(byTicker.get(ticker)!.map((b) => b.date))
85
- if (shared === null) {
86
- shared = tickerDates
87
- } else {
88
- const carried: Set<string> = shared
89
- shared = new Set([...carried].filter((d) => tickerDates.has(d)))
90
- }
91
- }
92
- const dates = [...shared!].sort()
93
- if (dates.length < 2) throw new Error(`loadBarsDir: fewer than 2 shared dates across ${dir}`)
94
- const bars = tickers.map((ticker) => {
95
- const wanted = new Set(dates)
96
- return byTicker.get(ticker)!.filter((b) => wanted.has(b.date))
97
- })
98
- return { tickers, dates, bars }
99
- }
100
-
101
- /** The campaign loop's data. Fails loud if any bar strays into the holdout era. */
102
- export async function loadInSample(): Promise<AlignedBars> {
103
- const aligned = await loadBarsDir(IN_SAMPLE_DIR)
104
- const last = aligned.dates[aligned.dates.length - 1]!
105
- if (last >= HOLDOUT_START) {
106
- throw new Error(`loadInSample: in-sample data reaches ${last}, at/past the holdout start ${HOLDOUT_START}`)
107
- }
108
- return aligned
109
- }
110
-
111
- /** CERTIFICATION PATH ONLY (holdout-certify.mts). The final 2 years. */
112
- export async function loadHoldout(): Promise<AlignedBars> {
113
- const aligned = await loadBarsDir(HOLDOUT_DIR)
114
- const first = aligned.dates[0]!
115
- if (first < HOLDOUT_START) {
116
- throw new Error(`loadHoldout: holdout data starts ${first}, before the holdout start ${HOLDOUT_START}`)
117
- }
118
- return aligned
119
- }
120
-
121
- /** In-sample followed by holdout on one axis — what the certification run
122
- * backtests so lookback indicators are warm when the holdout period begins.
123
- * Returns the concatenated universe plus the index of the first holdout day. */
124
- export function concatForCertification(insample: AlignedBars, holdout: AlignedBars): { aligned: AlignedBars; holdoutStartIndex: number } {
125
- if (insample.tickers.join(',') !== holdout.tickers.join(',')) {
126
- throw new Error('concatForCertification: in-sample and holdout universes differ')
127
- }
128
- const lastIn = insample.dates[insample.dates.length - 1]!
129
- const firstOut = holdout.dates[0]!
130
- if (firstOut <= lastIn) {
131
- throw new Error(`concatForCertification: holdout starts ${firstOut}, not after in-sample end ${lastIn}`)
132
- }
133
- return {
134
- aligned: {
135
- tickers: insample.tickers,
136
- dates: [...insample.dates, ...holdout.dates],
137
- bars: insample.bars.map((series, k) => [...series, ...holdout.bars[k]!]),
138
- },
139
- holdoutStartIndex: insample.dates.length,
140
- }
141
- }
@@ -1,253 +0,0 @@
1
- import { describe, expect, it } from 'vitest'
2
- import { runBacktest } from './backtest.ts'
3
- import { generateSignalsFromStrategy, runIncremental, strategyFromGenerateSignals } from './driver.ts'
4
- import { truncationInvariance } from './leak-audit.ts'
5
- import { rebalance, targetsToWeights } from './oms.ts'
6
- import { generateSignals as buyHold } from './strategies/buy-hold-index/strategy.ts'
7
- import { generateSignals as equalWeight } from './strategies/equal-weight/strategy.ts'
8
- import { generateSignals as smaCrossover } from './strategies/sma-crossover/strategy.ts'
9
- import type { Bar, Signal, Strategy, StrategyContext } from './types.ts'
10
-
11
- /** Deterministic wiggly price path (no RNG — reproducible). */
12
- function syntheticBars(n: number, tickers: number): Bar[][] {
13
- return Array.from({ length: tickers }, (_, k) =>
14
- Array.from({ length: n }, (_, t) => {
15
- const close = 100 + 10 * Math.sin(t / 7 + k) + 0.05 * t
16
- const open = 100 + 10 * Math.sin((t - 0.5) / 7 + k) + 0.05 * t
17
- return {
18
- date: `d${String(t).padStart(4, '0')}`,
19
- open,
20
- high: Math.max(open, close) + 1,
21
- low: Math.min(open, close) - 1,
22
- close,
23
- volume: 1000,
24
- }
25
- }),
26
- )
27
- }
28
-
29
- const symbolsFor = (bars: Bar[][]): string[] => bars.map((_, k) => (k === 0 ? 'IDX' : `T${k}`))
30
- const config = (bars: Bar[][]) => ({ costBps: 10, slippageBps: 5, symbols: symbolsFor(bars) })
31
-
32
- describe('runIncremental — structural history visibility', () => {
33
- const bars = syntheticBars(120, 3)
34
-
35
- it('every onBar call sees exactly bars[0..t] and nothing beyond', () => {
36
- let calls = 0
37
- const spy: Strategy = {
38
- onBar(ctx: StrategyContext) {
39
- calls++
40
- for (let k = 0; k < ctx.history.length; k++) {
41
- expect(ctx.history[k]!.length).toBe(ctx.t + 1)
42
- // The future is ABSENT, not merely guarded.
43
- expect(ctx.history[k]![ctx.t + 1]).toBeUndefined()
44
- expect(ctx.history[k]![ctx.t]!.date).toBe(bars[k]![ctx.t]!.date)
45
- }
46
- return null
47
- },
48
- }
49
- runIncremental(bars, spy, config(bars))
50
- expect(calls).toBe(120)
51
- })
52
-
53
- it('driver equity path is bit-identical to runBacktest on the collected decisions', () => {
54
- const strategy = strategyFromGenerateSignals(smaCrossover)
55
- const run = runIncremental(bars, strategy, config(bars))
56
- const reference = runBacktest(bars, run.decisions, { costBps: 10, slippageBps: 5 })
57
- expect(run.equityByDay.length).toBe(reference.equity.length)
58
- for (let t = 0; t < run.equityByDay.length; t++) {
59
- expect(run.equityByDay[t]).toBe(reference.equity[t])
60
- }
61
- })
62
-
63
- it('ctx.weights and ctx.equity reflect the drifted portfolio', () => {
64
- const seen: Array<{ t: number; equity: number; weightSum: number }> = []
65
- const holdIndex: Strategy = {
66
- onBar(ctx) {
67
- seen.push({ t: ctx.t, equity: ctx.equity, weightSum: ctx.weights.reduce((s, w) => s + w, 0) })
68
- return ctx.t === 0 ? [{ symbol: 'IDX', weight: 1 }] : null
69
- },
70
- }
71
- const run = runIncremental(bars, holdIndex, config(bars))
72
- expect(seen[0]!.equity).toBe(1)
73
- expect(seen[0]!.weightSum).toBe(0) // all cash before the first fill
74
- // After the t=0 decision fills at t=1's open, the book is ~fully invested.
75
- expect(seen[1]!.weightSum).toBeGreaterThan(0.99)
76
- expect(run.decisions).toEqual([{ t: 0, weights: [1, 0, 0] }])
77
- })
78
-
79
- it('a decision on the final bar is recorded but produces no fill', () => {
80
- const lastBarOnly: Strategy = {
81
- onBar(ctx) {
82
- return ctx.t === bars[0]!.length - 1 ? [{ symbol: 'IDX', weight: 1 }] : null
83
- },
84
- }
85
- const run = runIncremental(bars, lastBarOnly, config(bars))
86
- expect(run.decisions.length).toBe(1)
87
- expect(run.equityByDay.every((e) => e === 1)).toBe(true) // never traded
88
- })
89
-
90
- it('fail-closed on contract violations from onBar', () => {
91
- const short: Strategy = { onBar: () => [{ symbol: 'IDX', weight: -0.2 }] }
92
- expect(() => runIncremental(bars, short, config(bars))).toThrow(/long-only|negative/)
93
- const levered: Strategy = {
94
- onBar: () => [
95
- { symbol: 'IDX', weight: 0.8 },
96
- { symbol: 'T1', weight: 0.5 },
97
- ],
98
- }
99
- expect(() => runIncremental(bars, levered, config(bars))).toThrow(/sum.*> 1/)
100
- const unknown: Strategy = { onBar: () => [{ symbol: 'NOPE', weight: 0.5 }] }
101
- expect(() => runIncremental(bars, unknown, config(bars))).toThrow(/unknown symbol/)
102
- })
103
- })
104
-
105
- describe('batch-compat shim — the 3 pinned baselines run unchanged', () => {
106
- const bars = syntheticBars(400, 4)
107
-
108
- for (const [name, generateSignals] of [
109
- ['buy-hold-index', buyHold],
110
- ['equal-weight', equalWeight],
111
- ['sma-crossover', smaCrossover],
112
- ] as const) {
113
- it(`${name}: incremental drive reproduces the batch signals exactly`, () => {
114
- const batch = generateSignals(bars)
115
- const incremental = generateSignalsFromStrategy(strategyFromGenerateSignals(generateSignals))(bars)
116
- expect(incremental).toEqual(batch)
117
- })
118
- }
119
- })
120
-
121
- describe('leak audit under the v2 harness', () => {
122
- const bars = syntheticBars(400, 3)
123
-
124
- /** Deliberately leaky: sizes today's weight by TOMORROW's return (same
125
- * fixture class as leak-audit.test.mts, which keeps guarding the v1 path). */
126
- const peekAhead = (input: Bar[][]): Signal[] => {
127
- const T = input[0]!.length
128
- const signals: Signal[] = []
129
- for (let t = 0; t < T - 1; t++) {
130
- const up = input[0]![t + 1]!.close > input[0]![t]!.close
131
- signals.push({ t, weights: input.map((_, k) => (k === 0 && up ? 1 : 0)) })
132
- }
133
- return signals
134
- }
135
-
136
- /** Leaky via whole-series statistics over the FULL sample. */
137
- const fullSampleMax = (input: Bar[][]): Signal[] => {
138
- const maxClose = Math.max(...input[0]!.map((b) => b.close))
139
- const T = input[0]!.length
140
- const signals: Signal[] = []
141
- for (let t = 0; t < T; t++) {
142
- signals.push({ t, weights: input.map((_, k) => (k === 0 ? input[0]![t]!.close / maxClose : 0)) })
143
- }
144
- return signals
145
- }
146
-
147
- it('v1 truncation audit still catches both leaky strategies as written', () => {
148
- expect(truncationInvariance(peekAhead, bars, { warmupDays: 50 }).clean).toBe(false)
149
- expect(truncationInvariance(fullSampleMax, bars, { warmupDays: 50 }).clean).toBe(false)
150
- })
151
-
152
- it('the incremental harness neutralizes the batch peek-ahead (its signal can never fire)', () => {
153
- // Under the driver the shim hands the strategy TRUNCATED history: the
154
- // peek-ahead can only emit a signal for day t once it has seen t+1, so
155
- // "today's" signal never exists at decision time — zero trades, leak
156
- // structurally neutralized (and still caught as written by the v1 audit).
157
- const run = runIncremental(bars, strategyFromGenerateSignals(peekAhead), config(bars))
158
- expect(run.decisions).toEqual([])
159
- expect(run.orders).toEqual([])
160
- })
161
-
162
- it('a v2-native strategy reaching past today crashes on absent data — fail-closed', () => {
163
- const v2Peeker: Strategy = {
164
- onBar(ctx) {
165
- // ctx.history physically ends at today; tomorrow is undefined.
166
- const up = ctx.history[0]![ctx.t + 1]!.close > ctx.history[0]![ctx.t]!.close
167
- return up ? [{ symbol: 'IDX', weight: 1 }] : null
168
- },
169
- }
170
- expect(() => runIncremental(bars, v2Peeker, config(bars))).toThrow()
171
- })
172
-
173
- it('the incremental harness defuses whole-series leaks into causal decisions', () => {
174
- // Driven bar-by-bar, "max over the whole series" becomes "max up to
175
- // today": different decisions than the batch run, but causal — the
176
- // truncation audit on the DRIVEN strategy is clean by construction.
177
- const driven = generateSignalsFromStrategy(strategyFromGenerateSignals(fullSampleMax))
178
- expect(truncationInvariance(driven, bars, { warmupDays: 50 }).clean).toBe(true)
179
- expect(driven(bars)).not.toEqual(fullSampleMax(bars))
180
- })
181
- })
182
-
183
- describe('oms — the shared rebalancer', () => {
184
- it('targetsToWeights expands sparse targets and enforces the contract', () => {
185
- const symbols = ['IDX', 'A', 'B']
186
- expect(targetsToWeights([{ symbol: 'A', weight: 0.6 }], symbols)).toEqual([0, 0.6, 0])
187
- expect(() => targetsToWeights([{ symbol: 'A', weight: 0.6 }, { symbol: 'A', weight: 0.1 }], symbols)).toThrow(
188
- /twice/,
189
- )
190
- expect(() => targetsToWeights([{ symbol: 'A', weight: Number.NaN }], symbols)).toThrow(/non-finite/)
191
- })
192
-
193
- it('sizes orders LEAN-style: qty = (targetWeight * equity - positionValue) / price', () => {
194
- const orders = rebalance({
195
- targets: [
196
- { symbol: 'A', weight: 0.5 },
197
- { symbol: 'B', weight: 0 },
198
- ],
199
- symbols: ['IDX', 'A', 'B'],
200
- equity: 2,
201
- prices: [100, 50, 20],
202
- positionValues: [0, 0.5, 0.5],
203
- t: 42,
204
- tag: 'cand-x',
205
- })
206
- expect(orders).toEqual([
207
- // buy A: (0.5 * 2 - 0.5) / 50 = 0.01 shares
208
- { clientOrderId: 'qa-t42-A', symbol: 'A', side: 'buy', qty: 0.01, type: 'market', tif: 'day', tag: 'cand-x' },
209
- // sell B to flat: 0.5 / 20 = 0.025 shares
210
- { clientOrderId: 'qa-t42-B', symbol: 'B', side: 'sell', qty: 0.025, type: 'market', tif: 'day', tag: 'cand-x' },
211
- ])
212
- })
213
-
214
- it('suppresses dust and emits nothing on a no-op rebalance', () => {
215
- const orders = rebalance({
216
- targets: [{ symbol: 'A', weight: 0.5 }],
217
- symbols: ['IDX', 'A'],
218
- equity: 1,
219
- prices: [100, 50],
220
- positionValues: [0, 0.5],
221
- t: 7,
222
- })
223
- expect(orders).toEqual([])
224
- })
225
-
226
- it('order ids are deterministic per (t, symbol)', () => {
227
- const make = () =>
228
- rebalance({
229
- targets: [{ symbol: 'A', weight: 0.3 }],
230
- symbols: ['IDX', 'A'],
231
- equity: 1,
232
- prices: [100, 50],
233
- positionValues: [0, 0],
234
- t: 9,
235
- })
236
- expect(make()).toEqual(make())
237
- expect(make()[0]!.clientOrderId).toBe('qa-t9-A')
238
- })
239
-
240
- it('strategies cannot smuggle orders: only weights cross the seam', () => {
241
- // The driver hands onBar's return value to targetsToWeights, which
242
- // rejects anything that is not {symbol, weight} rows summing sanely.
243
- const bars = syntheticBars(30, 2)
244
- const smuggler = {
245
- onBar: () => [{ symbol: 'IDX', weight: 0.5, side: 'buy', qty: 999 }],
246
- } as unknown as Strategy
247
- // Extra fields are ignored — the OMS reads ONLY symbol + weight; the
248
- // emitted order comes out of the shared sizing rule, not the strategy.
249
- const run = runIncremental(bars, smuggler, { costBps: 10, slippageBps: 5, symbols: ['IDX', 'T1'] })
250
- expect(run.orders.every((o) => o.qty < 1)).toBe(true)
251
- expect(run.orders[0]!.clientOrderId).toMatch(/^qa-t0-IDX$/)
252
- })
253
- })