@tangle-network/agent-bench 0.11.2 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/HARNESS.md +6 -2
  3. package/README.md +1 -4
  4. package/dist/benchmarks/swe-bench.js +4 -9
  5. package/dist/benchmarks/swe-bench.js.map +1 -1
  6. package/package.json +5 -5
  7. package/scripts/run-package-tests.mjs +2 -2
  8. package/src/benchmarks/swe-bench.test.mts +49 -0
  9. package/src/benchmarks/swe-bench.ts +4 -9
  10. package/src/quant-arena/README.md +0 -144
  11. package/src/quant-arena/backtest.test.mts +0 -135
  12. package/src/quant-arena/backtest.ts +0 -218
  13. package/src/quant-arena/data.test.mts +0 -44
  14. package/src/quant-arena/data.ts +0 -141
  15. package/src/quant-arena/driver.test.mts +0 -253
  16. package/src/quant-arena/driver.ts +0 -219
  17. package/src/quant-arena/fixtures/data/PROVENANCE.md +0 -26
  18. package/src/quant-arena/fixtures/data/holdout/IDX.csv +0 -523
  19. package/src/quant-arena/fixtures/data/holdout/S01.csv +0 -523
  20. package/src/quant-arena/fixtures/data/holdout/S02.csv +0 -523
  21. package/src/quant-arena/fixtures/data/holdout/S03.csv +0 -523
  22. package/src/quant-arena/fixtures/data/holdout/S04.csv +0 -523
  23. package/src/quant-arena/fixtures/data/holdout/S05.csv +0 -523
  24. package/src/quant-arena/fixtures/data/holdout/S06.csv +0 -523
  25. package/src/quant-arena/fixtures/data/holdout/S07.csv +0 -523
  26. package/src/quant-arena/fixtures/data/holdout/S08.csv +0 -523
  27. package/src/quant-arena/fixtures/data/holdout/S09.csv +0 -523
  28. package/src/quant-arena/fixtures/data/holdout/S10.csv +0 -523
  29. package/src/quant-arena/fixtures/data/insample/IDX.csv +0 -2087
  30. package/src/quant-arena/fixtures/data/insample/S01.csv +0 -2087
  31. package/src/quant-arena/fixtures/data/insample/S02.csv +0 -2087
  32. package/src/quant-arena/fixtures/data/insample/S03.csv +0 -2087
  33. package/src/quant-arena/fixtures/data/insample/S04.csv +0 -2087
  34. package/src/quant-arena/fixtures/data/insample/S05.csv +0 -2087
  35. package/src/quant-arena/fixtures/data/insample/S06.csv +0 -2087
  36. package/src/quant-arena/fixtures/data/insample/S07.csv +0 -2087
  37. package/src/quant-arena/fixtures/data/insample/S08.csv +0 -2087
  38. package/src/quant-arena/fixtures/data/insample/S09.csv +0 -2087
  39. package/src/quant-arena/fixtures/data/insample/S10.csv +0 -2087
  40. package/src/quant-arena/fixtures/demo-campaign/cost-ledger.jsonl +0 -16
  41. package/src/quant-arena/fixtures/demo-campaign/notebook.jsonl +0 -5
  42. package/src/quant-arena/fixtures/demo-campaign/rollout-manifest.json +0 -171
  43. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-001-default-author/strategy.ts +0 -119
  44. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-002-default-author/strategy.ts +0 -119
  45. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-003-quant-researcher/strategy.ts +0 -105
  46. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts +0 -102
  47. package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl +0 -4
  48. package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl +0 -2
  49. package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json +0 -84
  50. package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts +0 -117
  51. package/src/quant-arena/holdout-certify.mts +0 -206
  52. package/src/quant-arena/holdout-certify.test.mts +0 -82
  53. package/src/quant-arena/leak-audit.test.mts +0 -79
  54. package/src/quant-arena/leak-audit.ts +0 -95
  55. package/src/quant-arena/make-fixtures.mts +0 -161
  56. package/src/quant-arena/multiplicity.test.mts +0 -68
  57. package/src/quant-arena/multiplicity.ts +0 -87
  58. package/src/quant-arena/nautilus-certify.ts +0 -31
  59. package/src/quant-arena/oms.ts +0 -90
  60. package/src/quant-arena/profiles/quant-researcher.profile.json +0 -12
  61. package/src/quant-arena/python/pyproject.toml +0 -8
  62. package/src/quant-arena/python/uv.lock +0 -1297
  63. package/src/quant-arena/python/vbt-worker.py +0 -192
  64. package/src/quant-arena/quant-loop.mts +0 -840
  65. package/src/quant-arena/quant-loop.test.mts +0 -75
  66. package/src/quant-arena/strategies/buy-hold-index/strategy.ts +0 -11
  67. package/src/quant-arena/strategies/equal-weight/strategy.ts +0 -20
  68. package/src/quant-arena/strategies/sma-crossover/strategy.ts +0 -42
  69. package/src/quant-arena/types.ts +0 -133
  70. package/src/quant-arena/vbt-client.ts +0 -321
  71. package/src/quant-arena/vbt-parity.test.mts +0 -183
  72. package/src/quant-arena/windows.test.mts +0 -45
  73. package/src/quant-arena/windows.ts +0 -54
  74. package/src/rollout-ledger/backfill-swe-arena.mts +0 -610
  75. package/src/rollout-ledger/backfill-swe-arena.test.mts +0 -347
  76. package/src/rollout-ledger/settle-capture.mts +0 -448
  77. package/src/rollout-ledger/settle-capture.test.mts +0 -270
  78. package/src/swe-arena/activation.mts +0 -225
  79. package/src/swe-arena/activation.test.mts +0 -300
  80. package/src/swe-arena/analyze.ts +0 -211
  81. package/src/swe-arena/arms.ts +0 -862
  82. package/src/swe-arena/bootstrap-meta.mts +0 -188
  83. package/src/swe-arena/bootstrap-meta.test.mts +0 -51
  84. package/src/swe-arena/briefing.mts +0 -217
  85. package/src/swe-arena/briefing.test.mts +0 -179
  86. package/src/swe-arena/calibrate.ts +0 -217
  87. package/src/swe-arena/capabilities.mts +0 -76
  88. package/src/swe-arena/capabilities.test.mts +0 -57
  89. package/src/swe-arena/capacity.ts +0 -198
  90. package/src/swe-arena/cell-evidence.mts +0 -437
  91. package/src/swe-arena/cell-evidence.test.mts +0 -248
  92. package/src/swe-arena/diagnosis-ensemble.test.mts +0 -210
  93. package/src/swe-arena/diagnosis-ensemble.ts +0 -523
  94. package/src/swe-arena/execution.test.mts +0 -1171
  95. package/src/swe-arena/factory-command-container.ts +0 -284
  96. package/src/swe-arena/factory-judge-child.mts +0 -228
  97. package/src/swe-arena/factory.test.mts +0 -645
  98. package/src/swe-arena/fixtures/analyze.py +0 -80
  99. package/src/swe-arena/fixtures/excludes.txt +0 -8
  100. package/src/swe-arena/fixtures/factory/agent-eval-309/calibration.md +0 -51
  101. package/src/swe-arena/fixtures/factory/agent-eval-309/manifest.json +0 -29
  102. package/src/swe-arena/fixtures/factory/agent-eval-309/spec.md +0 -64
  103. package/src/swe-arena/fixtures/factory/agent-runtime-232/calibration.md +0 -48
  104. package/src/swe-arena/fixtures/factory/agent-runtime-232/manifest.json +0 -29
  105. package/src/swe-arena/fixtures/factory/agent-runtime-232/spec.md +0 -48
  106. package/src/swe-arena/fixtures/factory/loops-28/calibration.md +0 -47
  107. package/src/swe-arena/fixtures/factory/loops-28/manifest.json +0 -30
  108. package/src/swe-arena/fixtures/factory/loops-28/spec.md +0 -50
  109. package/src/swe-arena/fixtures/gen1-salvage/README.md +0 -45
  110. package/src/swe-arena/fixtures/gen1-salvage/cand0-e6d7361.diff +0 -116
  111. package/src/swe-arena/fixtures/gen1-salvage/cand1-76a8590.diff +0 -293
  112. package/src/swe-arena/fixtures/holdout-preregister.log +0 -12
  113. package/src/swe-arena/fixtures/holdout.json +0 -44
  114. package/src/swe-arena/fixtures/instances.json +0 -146
  115. package/src/swe-arena/fixtures/ledger.jsonl +0 -12
  116. package/src/swe-arena/fixtures/patches/pallets__flask-5014.solo.patch +0 -36
  117. package/src/swe-arena/fixtures/patches/pydata__xarray-4687.sup.patch +0 -33
  118. package/src/swe-arena/fixtures/rejudge.jsonl +0 -15
  119. package/src/swe-arena/fixtures/rematch.jsonl +0 -3
  120. package/src/swe-arena/fixtures/rematch2.jsonl +0 -3
  121. package/src/swe-arena/fixtures/rematch3.jsonl +0 -3
  122. package/src/swe-arena/fixtures/run-report/README.md +0 -43
  123. package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json +0 -173
  124. package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md +0 -100
  125. package/src/swe-arena/fixtures/run-report/gen3-rollup.json +0 -551
  126. package/src/swe-arena/fixtures/run-report/gen3-rollup.md +0 -64
  127. package/src/swe-arena/fixtures/sup-journal-true.json +0 -19
  128. package/src/swe-arena/fixtures/verify/astropy__astropy-13033.sh +0 -48
  129. package/src/swe-arena/fixtures/verify/django__django-11532.sh +0 -50
  130. package/src/swe-arena/fixtures/verify/matplotlib__matplotlib-20826.sh +0 -76
  131. package/src/swe-arena/fixtures/verify/pydata__xarray-4687.sh +0 -44
  132. package/src/swe-arena/fixtures/verify/pytest-dev__pytest-6197.sh +0 -32
  133. package/src/swe-arena/fixtures/verify/sphinx-doc__sphinx-9658.sh +0 -51
  134. package/src/swe-arena/fixtures/worker-tokens.json +0 -42
  135. package/src/swe-arena/fixtures.ts +0 -237
  136. package/src/swe-arena/gepa-seat.mts +0 -886
  137. package/src/swe-arena/gepa-seat.test.mts +0 -1136
  138. package/src/swe-arena/holdout-certify.mts +0 -408
  139. package/src/swe-arena/holdout-certify.test.mts +0 -160
  140. package/src/swe-arena/implementation-ref.test.mts +0 -64
  141. package/src/swe-arena/implementation-ref.ts +0 -62
  142. package/src/swe-arena/judge-child.mts +0 -37
  143. package/src/swe-arena/ledger-orphans.mts +0 -77
  144. package/src/swe-arena/ledger-orphans.test.mts +0 -149
  145. package/src/swe-arena/manifest.mts +0 -293
  146. package/src/swe-arena/manifest.test.mts +0 -169
  147. package/src/swe-arena/materialize.ts +0 -142
  148. package/src/swe-arena/outer-loop.mts +0 -2854
  149. package/src/swe-arena/outer-loop.test.mts +0 -714
  150. package/src/swe-arena/parity.test.mts +0 -87
  151. package/src/swe-arena/premeasured-from-cells.mts +0 -296
  152. package/src/swe-arena/premeasured-from-cells.test.mts +0 -201
  153. package/src/swe-arena/proc.test.mts +0 -172
  154. package/src/swe-arena/proc.ts +0 -260
  155. package/src/swe-arena/profiles/deepseek-author.profile.json +0 -12
  156. package/src/swe-arena/profiles/default-author.profile.json +0 -12
  157. package/src/swe-arena/proposer-fanout.mts +0 -736
  158. package/src/swe-arena/proposer-fanout.test.mts +0 -660
  159. package/src/swe-arena/proposer-provenance.mts +0 -176
  160. package/src/swe-arena/proposer-provenance.test.mts +0 -106
  161. package/src/swe-arena/reconcile.ts +0 -0
  162. package/src/swe-arena/replay.mts +0 -183
  163. package/src/swe-arena/replay.test.mts +0 -300
  164. package/src/swe-arena/run-experiment.mts +0 -729
  165. package/src/swe-arena/run-report.mts +0 -75
  166. package/src/swe-arena/run-supervisor.mjs +0 -297
  167. package/src/swe-arena/run-supervisor.test.mts +0 -539
  168. package/src/swe-arena/score-split.mts +0 -140
  169. package/src/swe-arena/score-split.test.mts +0 -123
  170. package/src/swe-arena/scratch-worktree-serialization.test.mts +0 -72
  171. package/src/swe-arena/scratch-worktree.test.mts +0 -56
  172. package/src/swe-arena/scratch-worktree.ts +0 -64
  173. package/src/swe-arena/serialized-judge.ts +0 -414
  174. package/src/swe-arena/types.ts +0 -218
@@ -1,79 +0,0 @@
1
- import { describe, expect, it } from 'vitest'
2
- import { truncationCutoffs, truncationInvariance } from './leak-audit.ts'
3
- import { generateSignals as smaCrossover } from './strategies/sma-crossover/strategy.ts'
4
- import { generateSignals as buyHold } from './strategies/buy-hold-index/strategy.ts'
5
- import type { Bar, Signal } from './types.ts'
6
-
7
- /** Deterministic wiggly price path (no RNG — the test must be reproducible). */
8
- function syntheticBars(n: number, tickers: number): Bar[][] {
9
- return Array.from({ length: tickers }, (_, k) =>
10
- Array.from({ length: n }, (_, t) => {
11
- const close = 100 + 10 * Math.sin(t / 7 + k) + 0.05 * t
12
- const open = 100 + 10 * Math.sin((t - 0.5) / 7 + k) + 0.05 * t
13
- return {
14
- date: `d${String(t).padStart(4, '0')}`,
15
- open,
16
- high: Math.max(open, close) + 1,
17
- low: Math.min(open, close) - 1,
18
- close,
19
- volume: 1000,
20
- }
21
- }),
22
- )
23
- }
24
-
25
- /** Deliberately leaky: sizes today's weight by TOMORROW's return. Exactly the
26
- * bug class the truncation check exists to kill. */
27
- const peekAheadStrategy = (bars: Bar[][]): Signal[] => {
28
- const T = bars[0]!.length
29
- const signals: Signal[] = []
30
- for (let t = 0; t < T - 1; t++) {
31
- const up = bars[0]![t + 1]!.close > bars[0]![t]!.close
32
- signals.push({ t, weights: bars.map((_, k) => (k === 0 && up ? 1 : 0)) })
33
- }
34
- return signals
35
- }
36
-
37
- /** Leaky via whole-series statistics: normalizes by the FULL-sample max close. */
38
- const fullSampleMaxStrategy = (bars: Bar[][]): Signal[] => {
39
- const maxClose = Math.max(...bars[0]!.map((b) => b.close))
40
- const T = bars[0]!.length
41
- const signals: Signal[] = []
42
- for (let t = 0; t < T; t++) {
43
- const w = bars[0]![t]!.close / maxClose
44
- signals.push({ t, weights: bars.map((_, k) => (k === 0 ? w : 0)) })
45
- }
46
- return signals
47
- }
48
-
49
- describe('truncationInvariance — the mechanical look-ahead guard', () => {
50
- const bars = syntheticBars(400, 3)
51
-
52
- it('catches a strategy that peeks at tomorrow', () => {
53
- const report = truncationInvariance(peekAheadStrategy, bars, { warmupDays: 50 })
54
- expect(report.clean).toBe(false)
55
- expect(report.divergence).not.toBeNull()
56
- expect(report.divergence!.detail).toMatch(/decision changed|exists only/)
57
- })
58
-
59
- it('catches a strategy leaking through full-sample statistics', () => {
60
- const report = truncationInvariance(fullSampleMaxStrategy, bars, { warmupDays: 50 })
61
- expect(report.clean).toBe(false)
62
- })
63
-
64
- it('passes the pinned baselines (no look-ahead by construction)', () => {
65
- expect(truncationInvariance(smaCrossover, bars, { warmupDays: 120 }).clean).toBe(true)
66
- expect(truncationInvariance(buyHold, bars, { warmupDays: 50 }).clean).toBe(true)
67
- })
68
-
69
- it('cutoffs are inside (warmup, T-1) and deduplicated ascending', () => {
70
- const cutoffs = truncationCutoffs(400, 50)
71
- expect(cutoffs.length).toBeGreaterThanOrEqual(2)
72
- for (const c of cutoffs) {
73
- expect(c).toBeGreaterThan(50)
74
- expect(c).toBeLessThan(399)
75
- }
76
- expect(cutoffs).toEqual([...cutoffs].sort((a, b) => a - b))
77
- expect(new Set(cutoffs).size).toBe(cutoffs.length)
78
- })
79
- })
@@ -1,95 +0,0 @@
1
- /**
2
- * Look-ahead (leak) audit — the deterministic half.
3
- *
4
- * The strongest guard against look-ahead bias is mechanical: run the strategy
5
- * on data truncated at day c and on the full history. Under the contract
6
- * ("the signal at t uses only bars[0..t]"), every signal with t <= c must be
7
- * BIT-IDENTICAL in both runs. Any divergence proves the strategy read past
8
- * its decision day — future closes, whole-series statistics, end-anchored
9
- * indexing — regardless of how the leak was written.
10
- *
11
- * This check is necessary, not sufficient (a strategy hardcoding calendar
12
- * dates it memorized survives truncation), which is why the campaign ALSO
13
- * runs an adversarial LLM read of the source (see quant-loop.mts). Kill on
14
- * either.
15
- */
16
-
17
- import type { Bar, GenerateSignals, Signal } from './types.ts'
18
-
19
- export interface TruncationDivergence {
20
- cutoff: number
21
- t: number
22
- detail: string
23
- }
24
-
25
- export interface TruncationReport {
26
- clean: boolean
27
- cutoffs: number[]
28
- divergence: TruncationDivergence | null
29
- }
30
-
31
- /** Evenly spaced cutoffs across (warmupDays, totalDays - 1). */
32
- export function truncationCutoffs(totalDays: number, warmupDays: number, count = 6): number[] {
33
- const lo = Math.max(warmupDays + 1, 1)
34
- const hi = totalDays - 2
35
- if (hi <= lo) throw new Error(`truncationCutoffs: no room between warmup ${warmupDays} and ${totalDays} days`)
36
- const cutoffs = new Set<number>()
37
- for (let i = 0; i < count; i++) {
38
- cutoffs.add(Math.round(lo + ((hi - lo) * i) / Math.max(count - 1, 1)))
39
- }
40
- return [...cutoffs].sort((a, b) => a - b)
41
- }
42
-
43
- const signalMap = (signals: Signal[], maxT: number): Map<number, number[]> => {
44
- const map = new Map<number, number[]>()
45
- for (const s of signals) if (s.t <= maxT) map.set(s.t, s.weights)
46
- return map
47
- }
48
-
49
- function compareAt(cutoff: number, full: Signal[], truncated: Signal[]): TruncationDivergence | null {
50
- const fullMap = signalMap(full, cutoff)
51
- const truncMap = signalMap(truncated, cutoff)
52
- const ts = new Set([...fullMap.keys(), ...truncMap.keys()])
53
- for (const t of [...ts].sort((a, b) => a - b)) {
54
- const a = fullMap.get(t)
55
- const b = truncMap.get(t)
56
- if (a === undefined || b === undefined) {
57
- return {
58
- cutoff,
59
- t,
60
- detail: `signal at t=${t} exists only in the ${a === undefined ? 'truncated' : 'full'} run`,
61
- }
62
- }
63
- if (a.length !== b.length) {
64
- return { cutoff, t, detail: `weight vector length ${a.length} vs ${b.length} at t=${t}` }
65
- }
66
- for (let k = 0; k < a.length; k++) {
67
- if (!Object.is(a[k], b[k])) {
68
- return {
69
- cutoff,
70
- t,
71
- detail: `weights[${k}] at t=${t}: full=${a[k]} truncated=${b[k]} — the decision changed when future bars were removed`,
72
- }
73
- }
74
- }
75
- }
76
- return null
77
- }
78
-
79
- /** Run the truncation-invariance check. Deterministic; throws only if the
80
- * strategy itself throws. */
81
- export function truncationInvariance(
82
- strategy: GenerateSignals,
83
- bars: Bar[][],
84
- opts: { warmupDays: number; cutoffs?: number[] },
85
- ): TruncationReport {
86
- const totalDays = bars[0]?.length ?? 0
87
- const cutoffs = opts.cutoffs ?? truncationCutoffs(totalDays, opts.warmupDays)
88
- const full = strategy(bars)
89
- for (const cutoff of cutoffs) {
90
- const truncated = strategy(bars.map((series) => series.slice(0, cutoff + 1)))
91
- const divergence = compareAt(cutoff, full, truncated)
92
- if (divergence !== null) return { clean: false, cutoffs, divergence }
93
- }
94
- return { clean: true, cutoffs, divergence: null }
95
- }
@@ -1,161 +0,0 @@
1
- /**
2
- * Deterministic synthetic market generator — writes the vendored fixtures.
3
- *
4
- * tsx src/quant-arena/make-fixtures.mts # regenerates fixtures/data/**
5
- *
6
- * Why synthetic (see fixtures/data/PROVENANCE.md): the free daily-bar sources
7
- * we checked (Stooq) license their data for personal use only and sit behind
8
- * an anti-bot wall, so vendoring real bars into a redistributable repo is not
9
- * clean. Instead the repo commits 10 years of clearly-labeled synthetic daily
10
- * bars for an index (IDX) plus 10 stocks (S01..S10), produced by THIS file
11
- * with a fixed seed — anyone can regenerate and diff them.
12
- *
13
- * The generator aims for realistic structure, not any real market's path:
14
- * - a two-state regime-switching market factor (calm: +9%/yr drift, 13% vol;
15
- * stress: -12%/yr drift, 32% vol; sticky transitions),
16
- * - per-stock beta in [0.6, 1.5], idiosyncratic vol 15-35%/yr, small alpha,
17
- * rare idiosyncratic jumps,
18
- * - OHLC built from overnight gaps + intraday ranges, lognormal volume.
19
- *
20
- * Split: weekdays 2016-07-01 .. 2026-06-30; the FINAL 2 YEARS (>= 2024-07-01)
21
- * go to fixtures/data/holdout/, the rest to fixtures/data/insample/.
22
- */
23
-
24
- import { mkdir, writeFile } from 'node:fs/promises'
25
- import { join } from 'node:path'
26
- import { pathToFileURL } from 'node:url'
27
- import { HOLDOUT_DIR, HOLDOUT_START, IN_SAMPLE_DIR, INDEX_TICKER } from './data.ts'
28
- import { mulberry32 } from './windows.ts'
29
-
30
- export const FIXTURE_SEED = 20260722
31
-
32
- const START_DATE = '2016-07-01'
33
- const END_DATE = '2026-06-30'
34
- const DAYS_PER_YEAR = 252
35
-
36
- interface Regime {
37
- driftAnnual: number
38
- volAnnual: number
39
- stayProb: number
40
- }
41
-
42
- const CALM: Regime = { driftAnnual: 0.09, volAnnual: 0.13, stayProb: 0.985 }
43
- const STRESS: Regime = { driftAnnual: -0.12, volAnnual: 0.32, stayProb: 0.92 }
44
-
45
- /** Seeded standard normal via Box-Muller over mulberry32. */
46
- function gaussian(rand: () => number): () => number {
47
- let spare: number | null = null
48
- return () => {
49
- if (spare !== null) {
50
- const v = spare
51
- spare = null
52
- return v
53
- }
54
- let u = 0
55
- let v = 0
56
- while (u === 0) u = rand()
57
- while (v === 0) v = rand()
58
- const mag = Math.sqrt(-2 * Math.log(u))
59
- spare = mag * Math.sin(2 * Math.PI * v)
60
- return mag * Math.cos(2 * Math.PI * v)
61
- }
62
- }
63
-
64
- export function weekdays(startDate: string, endDate: string): string[] {
65
- const dates: string[] = []
66
- const d = new Date(`${startDate}T00:00:00Z`)
67
- const end = new Date(`${endDate}T00:00:00Z`)
68
- while (d.getTime() <= end.getTime()) {
69
- const dow = d.getUTCDay()
70
- if (dow !== 0 && dow !== 6) dates.push(d.toISOString().slice(0, 10))
71
- d.setUTCDate(d.getUTCDate() + 1)
72
- }
73
- return dates
74
- }
75
-
76
- interface SyntheticSeries {
77
- ticker: string
78
- rows: Array<{ date: string; open: number; high: number; low: number; close: number; volume: number }>
79
- }
80
-
81
- export function generateUniverse(seed: number): SyntheticSeries[] {
82
- const dates = weekdays(START_DATE, END_DATE)
83
- const T = dates.length
84
- const rand = mulberry32(seed)
85
- const norm = gaussian(rand)
86
-
87
- // Market factor daily log-returns under the regime chain.
88
- let regime: Regime = CALM
89
- const marketLogRet: number[] = new Array(T).fill(0)
90
- for (let t = 1; t < T; t++) {
91
- if (rand() > regime.stayProb) regime = regime === CALM ? STRESS : CALM
92
- marketLogRet[t] = regime.driftAnnual / DAYS_PER_YEAR + (regime.volAnnual / Math.sqrt(DAYS_PER_YEAR)) * norm()
93
- }
94
-
95
- const specs = [
96
- { ticker: INDEX_TICKER, beta: 1, idioVol: 0.02, alpha: 0, jumpProb: 0, basePrice: 200 },
97
- ...Array.from({ length: 10 }, (_, i) => ({
98
- ticker: `S${String(i + 1).padStart(2, '0')}`,
99
- beta: 0.6 + rand() * 0.9,
100
- idioVol: 0.15 + rand() * 0.2,
101
- alpha: -0.02 + rand() * 0.08,
102
- jumpProb: 0.004,
103
- basePrice: 20 + rand() * 180,
104
- })),
105
- ]
106
-
107
- return specs.map((spec) => {
108
- const dailyIdio = spec.idioVol / Math.sqrt(DAYS_PER_YEAR)
109
- const closes: number[] = new Array(T).fill(0)
110
- closes[0] = spec.basePrice
111
- for (let t = 1; t < T; t++) {
112
- let logRet = spec.alpha / DAYS_PER_YEAR + spec.beta * marketLogRet[t]! + dailyIdio * norm()
113
- if (spec.jumpProb > 0 && rand() < spec.jumpProb) {
114
- logRet += (rand() < 0.5 ? -1 : 1) * (3 + rand() * 5) * dailyIdio
115
- }
116
- closes[t] = closes[t - 1]! * Math.exp(logRet)
117
- }
118
- const baseVolume = 1e6 * (0.5 + rand() * 4)
119
- const rows = dates.map((date, t) => {
120
- const prevClose = t === 0 ? closes[0]! : closes[t - 1]!
121
- const close = closes[t]!
122
- const gap = t === 0 ? 0 : 0.3 * dailyIdio * norm()
123
- const open = round4(prevClose * Math.exp(gap))
124
- const range = Math.abs(norm()) * 0.5 * dailyIdio
125
- const high = round4(Math.max(open, close) * Math.exp(range))
126
- const low = round4(Math.min(open, close) * Math.exp(-Math.abs(norm()) * 0.5 * dailyIdio))
127
- const ret = t === 0 ? 0 : Math.abs(close / prevClose - 1)
128
- const volume = Math.round(baseVolume * (1 + 3 * ret) * Math.exp(0.2 * norm()))
129
- return { date, open, high, low, close: round4(close), volume }
130
- })
131
- return { ticker: spec.ticker, rows }
132
- })
133
- }
134
-
135
- const round4 = (x: number): number => Math.round(x * 10_000) / 10_000
136
-
137
- function toCsv(series: SyntheticSeries, filter: (date: string) => boolean): string {
138
- const lines = ['Date,Open,High,Low,Close,Volume']
139
- for (const r of series.rows) {
140
- if (!filter(r.date)) continue
141
- lines.push(`${r.date},${r.open},${r.high},${r.low},${r.close},${r.volume}`)
142
- }
143
- return lines.join('\n') + '\n'
144
- }
145
-
146
- const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
147
-
148
- if (isMain) {
149
- const universe = generateUniverse(FIXTURE_SEED)
150
- await mkdir(IN_SAMPLE_DIR, { recursive: true })
151
- await mkdir(HOLDOUT_DIR, { recursive: true })
152
- for (const series of universe) {
153
- await writeFile(join(IN_SAMPLE_DIR, `${series.ticker}.csv`), toCsv(series, (d) => d < HOLDOUT_START))
154
- await writeFile(join(HOLDOUT_DIR, `${series.ticker}.csv`), toCsv(series, (d) => d >= HOLDOUT_START))
155
- }
156
- const inDays = universe[0]!.rows.filter((r) => r.date < HOLDOUT_START).length
157
- const outDays = universe[0]!.rows.length - inDays
158
- console.log(
159
- `wrote ${universe.length} tickers: ${inDays} in-sample days (< ${HOLDOUT_START}) + ${outDays} holdout days, seed ${FIXTURE_SEED}`,
160
- )
161
- }
@@ -1,68 +0,0 @@
1
- import { describe, expect, it } from 'vitest'
2
- import {
3
- BASE_EXCESS_SHARPE,
4
- MULTIPLICITY_SPREAD,
5
- decideAcceptance,
6
- requiredExcessSharpe,
7
- } from './multiplicity.ts'
8
-
9
- describe('requiredExcessSharpe — the rising bar', () => {
10
- it('equals the base floor for the very first candidate', () => {
11
- expect(requiredExcessSharpe(1)).toBeCloseTo(BASE_EXCESS_SHARPE, 12)
12
- })
13
-
14
- it('matches the documented formula base + spread * sqrt(2 ln n)', () => {
15
- for (const n of [2, 4, 10, 50, 200]) {
16
- expect(requiredExcessSharpe(n)).toBeCloseTo(BASE_EXCESS_SHARPE + MULTIPLICITY_SPREAD * Math.sqrt(2 * Math.log(n)), 12)
17
- }
18
- // Spot values so a silent constant change fails a test, not just a diff.
19
- expect(requiredExcessSharpe(10)).toBeCloseTo(0.1 + 0.15 * Math.sqrt(2 * Math.log(10)), 12)
20
- })
21
-
22
- it('is strictly increasing in candidates tried', () => {
23
- let prev = requiredExcessSharpe(1)
24
- for (let n = 2; n <= 100; n++) {
25
- const cur = requiredExcessSharpe(n)
26
- expect(cur).toBeGreaterThan(prev)
27
- prev = cur
28
- }
29
- })
30
-
31
- it('rejects a non-positive or fractional try count', () => {
32
- expect(() => requiredExcessSharpe(0)).toThrow(/positive integer/)
33
- expect(() => requiredExcessSharpe(2.5)).toThrow(/positive integer/)
34
- })
35
- })
36
-
37
- describe('decideAcceptance — consistency AND multiplicity', () => {
38
- const goodWindows = [0.9, 0.8, 1.0, 0.7, 0.85, 0.95, 0.6, 0.75]
39
-
40
- it('accepts a consistent, large edge early in the campaign', () => {
41
- const d = decideAcceptance({ perWindowExcess: goodWindows, nTried: 1 })
42
- expect(d.accepted).toBe(true)
43
- expect(d.wins).toBe(8)
44
- expect(d.requiredWins).toBe(6)
45
- })
46
-
47
- it('rejects on consistency when the edge is one lucky stretch (5/8 wins)', () => {
48
- const d = decideAcceptance({ perWindowExcess: [3, 3, 3, -0.1, -0.1, -0.1, 0.5, 0.5], nTried: 1 })
49
- expect(d.accepted).toBe(false)
50
- expect(d.wins).toBe(5)
51
- expect(d.reasons[0]).toMatch(/consistency/)
52
- })
53
-
54
- it('the SAME evidence passes at n=1 and fails once enough candidates were tried', () => {
55
- // Mean excess 0.30: above the n=1 bar (0.10), below the n=50 bar (~0.52).
56
- const marginal = [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3]
57
- expect(decideAcceptance({ perWindowExcess: marginal, nTried: 1 }).accepted).toBe(true)
58
- const late = decideAcceptance({ perWindowExcess: marginal, nTried: 50 })
59
- expect(late.accepted).toBe(false)
60
- expect(late.reasons[0]).toMatch(/multiplicity/)
61
- expect(late.threshold).toBeGreaterThan(0.3)
62
- })
63
-
64
- it('fails closed on non-finite evidence and empty windows', () => {
65
- expect(() => decideAcceptance({ perWindowExcess: [0.5, Number.NaN], nTried: 1 })).toThrow(/non-finite/)
66
- expect(() => decideAcceptance({ perWindowExcess: [], nTried: 1 })).toThrow(/no windows/)
67
- })
68
- })
@@ -1,87 +0,0 @@
1
- /**
2
- * Multiplicity-adjusted acceptance: the bar RISES with every candidate tried.
3
- *
4
- * Why: if you test N random strategies against the same data, the best one
5
- * looks good by luck alone. Under the null (no skill), the expected maximum
6
- * Sharpe across N independent tries grows like sqrt(2 ln N) times the
7
- * cross-trial dispersion — that is the term the Deflated Sharpe Ratio of
8
- * Bailey & Lopez de Prado ("The Deflated Sharpe Ratio", Journal of Portfolio
9
- * Management, 2014) corrects for. We implement the simple, auditable version:
10
- *
11
- * requiredExcessSharpe(n) = BASE + SPREAD * sqrt(2 * ln(n))
12
- *
13
- * where n counts EVERY candidate ever tried this campaign (including ones
14
- * killed by the leak audit — they were still draws from the search), BASE is
15
- * the floor a single try must clear, and SPREAD calibrates how fast luck
16
- * accumulates. Every tried candidate is a permanent notebook row, so n can
17
- * never be quietly reset.
18
- *
19
- * A candidate is accepted only if BOTH hold:
20
- * 1. it beats the best pinned baseline's Sharpe in >= ceil(minWinFraction*K)
21
- * of the K in-sample windows (consistency, not one lucky stretch), and
22
- * 2. its mean excess Sharpe across the K windows >= requiredExcessSharpe(n).
23
- */
24
-
25
- /** Annualized excess-Sharpe floor for the very first candidate (n = 1). */
26
- export const BASE_EXCESS_SHARPE = 0.1
27
- /** Luck-accumulation rate: how fast the bar rises with tries. */
28
- export const MULTIPLICITY_SPREAD = 0.15
29
-
30
- export function requiredExcessSharpe(nTried: number): number {
31
- if (!Number.isInteger(nTried) || nTried < 1) {
32
- throw new Error(`requiredExcessSharpe: nTried must be a positive integer, got ${nTried}`)
33
- }
34
- return BASE_EXCESS_SHARPE + MULTIPLICITY_SPREAD * Math.sqrt(2 * Math.log(nTried))
35
- }
36
-
37
- export interface AcceptanceDecision {
38
- accepted: boolean
39
- wins: number
40
- windows: number
41
- requiredWins: number
42
- meanExcessSharpe: number
43
- threshold: number
44
- nTried: number
45
- reasons: string[]
46
- }
47
-
48
- export interface AcceptanceInput {
49
- /** Candidate Sharpe minus best-baseline Sharpe, one entry per window. */
50
- perWindowExcess: number[]
51
- /** Total candidates tried this campaign INCLUDING this one. */
52
- nTried: number
53
- /** Fraction of windows the candidate must win. Default 0.75 (6 of 8). */
54
- minWinFraction?: number
55
- }
56
-
57
- export function decideAcceptance(input: AcceptanceInput): AcceptanceDecision {
58
- const { perWindowExcess, nTried } = input
59
- const windows = perWindowExcess.length
60
- if (windows === 0) throw new Error('decideAcceptance: no windows')
61
- for (const e of perWindowExcess) {
62
- if (!Number.isFinite(e)) throw new Error('decideAcceptance: non-finite excess Sharpe — fail-closed')
63
- }
64
- const minWinFraction = input.minWinFraction ?? 0.75
65
- const requiredWins = Math.ceil(minWinFraction * windows)
66
- const wins = perWindowExcess.filter((e) => e > 0).length
67
- const meanExcessSharpe = perWindowExcess.reduce((s, e) => s + e, 0) / windows
68
- const threshold = requiredExcessSharpe(nTried)
69
- const reasons: string[] = []
70
- if (wins < requiredWins) {
71
- reasons.push(`consistency: beat the best baseline in only ${wins}/${windows} windows (need ${requiredWins})`)
72
- }
73
- if (meanExcessSharpe < threshold) {
74
- reasons.push(
75
- `multiplicity: mean excess Sharpe ${meanExcessSharpe.toFixed(3)} < required ${threshold.toFixed(3)} ` +
76
- `(bar after ${nTried} tried candidate${nTried === 1 ? '' : 's'})`,
77
- )
78
- }
79
- const accepted = reasons.length === 0
80
- if (accepted) {
81
- reasons.push(
82
- `accepted: won ${wins}/${windows} windows, mean excess Sharpe ${meanExcessSharpe.toFixed(3)} ` +
83
- `>= ${threshold.toFixed(3)} at n=${nTried}`,
84
- )
85
- }
86
- return { accepted, wins, windows, requiredWins, meanExcessSharpe, threshold, nTried, reasons }
87
- }
@@ -1,31 +0,0 @@
1
- /**
2
- * Phase 3 stub — Nautilus Trader certification adapter (NOT IMPLEMENTED).
3
- *
4
- * Per the quant-arena v2 decision memo, the final certification tier runs a
5
- * winning strategy's decision record through Nautilus Trader's event-driven
6
- * engine (real OMS semantics: order lifecycle, partial fills, latency
7
- * models) as an independent confirmation of the vectorbt score before
8
- * anything is promoted. Phases 1-2 (this PR) end at the vectorbt worker;
9
- * this seam exists so the campaign plumbing has a stable name to call.
10
- *
11
- * TODO(phase-3, quant-arena v2 memo): implement as a python worker sibling
12
- * of python/vbt-worker.py (same JSONL protocol shape, own pinned uv env),
13
- * translating the OMS `Order` stream from driver.ts — not raw weights —
14
- * into Nautilus order submissions, and reconciling final equity against the
15
- * vectorbt curve within a documented tolerance.
16
- */
17
-
18
- import type { Order } from './types.ts'
19
-
20
- export interface NautilusCertification {
21
- finalEquity: number
22
- ordersAccepted: number
23
- ordersRejected: number
24
- }
25
-
26
- export function certifyWithNautilus(_orders: Order[]): Promise<NautilusCertification> {
27
- throw new Error(
28
- 'quant-arena: Nautilus certification is Phase 3 of the v2 memo and is not implemented yet — ' +
29
- 'the vectorbt worker (vbt-client.ts) is the current official scorer.',
30
- )
31
- }
@@ -1,90 +0,0 @@
1
- /**
2
- * The shared OMS rebalancer — the ONLY component that turns a strategy's
3
- * target weights into orders. Strategies never construct `Order`s; keeping
4
- * sizing in one place means every candidate and baseline trades under the
5
- * same rules, and the leak/contract audits only ever have to reason about
6
- * weights.
7
- *
8
- * Sizing (LEAN `CalculateOrderQuantity` style, long-only v2):
9
- * qty = (targetWeight * equity - currentPositionValue) / price
10
- * at decision-time (close of day t) prices. Positive -> buy, negative ->
11
- * sell. Quantities are fractional: the backtest engines track dollar
12
- * positions, so lot rounding would only introduce parity noise between the
13
- * TS prefilter and the vectorbt scorer. No fee buffer is reserved (fees are
14
- * financed at 0% by the engines — conservative, never flattering).
15
- *
16
- * Long-only enforcement is fail-closed: a negative target weight, a weight
17
- * sum > 1, or an unknown symbol throws rather than clamping.
18
- */
19
-
20
- import type { Order, Rebalancer, TargetPosition } from './types.ts'
21
-
22
- const WEIGHT_EPS = 1e-9
23
- /** Trades below this fraction of equity are dust — no order is emitted. */
24
- const MIN_TRADE_FRACTION = 1e-9
25
-
26
- /** Validate targets and expand them to a full per-symbol weight vector
27
- * (omitted symbols -> 0). Throws on contract violations. */
28
- export function targetsToWeights(targets: TargetPosition[], symbols: string[]): number[] {
29
- const index = new Map(symbols.map((s, k) => [s, k]))
30
- const weights = new Array<number>(symbols.length).fill(0)
31
- const seen = new Set<string>()
32
- let sum = 0
33
- for (const target of targets) {
34
- const k = index.get(target.symbol)
35
- if (k === undefined) {
36
- throw new Error(`oms: target names unknown symbol '${target.symbol}' (universe: ${symbols.join(', ')})`)
37
- }
38
- if (seen.has(target.symbol)) {
39
- throw new Error(`oms: symbol '${target.symbol}' targeted twice in one rebalance`)
40
- }
41
- seen.add(target.symbol)
42
- if (!Number.isFinite(target.weight)) throw new Error(`oms: non-finite weight for '${target.symbol}'`)
43
- if (target.weight < -WEIGHT_EPS) {
44
- throw new Error(`oms: negative weight ${target.weight} for '${target.symbol}' — long-only, no shorting`)
45
- }
46
- weights[k] = Math.max(target.weight, 0)
47
- sum += weights[k]!
48
- }
49
- if (sum > 1 + 1e-6) {
50
- throw new Error(`oms: target weights sum ${sum} > 1 — no leverage`)
51
- }
52
- return weights
53
- }
54
-
55
- /** The one shared rebalancer. See module header for the sizing rule. */
56
- export const rebalance: Rebalancer = ({ targets, symbols, equity, prices, positionValues, t, tag }) => {
57
- if (prices.length !== symbols.length || positionValues.length !== symbols.length) {
58
- throw new Error(
59
- `oms: state arrays disagree with the universe (${symbols.length} symbols, ` +
60
- `${prices.length} prices, ${positionValues.length} positions)`,
61
- )
62
- }
63
- if (!Number.isFinite(equity) || equity <= 0) throw new Error(`oms: nonpositive equity ${equity}`)
64
- const weights = targetsToWeights(targets, symbols)
65
- const orders: Order[] = []
66
- for (let k = 0; k < symbols.length; k++) {
67
- const price = prices[k]!
68
- if (!Number.isFinite(price) || price <= 0) throw new Error(`oms: nonpositive price for '${symbols[k]}'`)
69
- const desiredValue = weights[k]! * equity
70
- const deltaValue = desiredValue - positionValues[k]!
71
- if (Math.abs(deltaValue) <= MIN_TRADE_FRACTION * equity) continue
72
- const qty = Math.abs(deltaValue) / price
73
- const side = deltaValue > 0 ? 'buy' : 'sell'
74
- if (side === 'sell' && deltaValue < -positionValues[k]! - MIN_TRADE_FRACTION * equity) {
75
- // Cannot happen when weights >= 0, but keep the invariant explicit:
76
- // a sell may never exceed the current long position.
77
- throw new Error(`oms: sell of ${qty} '${symbols[k]}' exceeds current position — long-only`)
78
- }
79
- orders.push({
80
- clientOrderId: `qa-t${t}-${symbols[k]}`,
81
- symbol: symbols[k]!,
82
- side,
83
- qty,
84
- type: 'market',
85
- tif: 'day',
86
- ...(tag !== undefined ? { tag } : {}),
87
- })
88
- }
89
- return orders
90
- }
@@ -1,12 +0,0 @@
1
- {
2
- "name": "quant-researcher",
3
- "description": "Pi strategy-author lens that speaks quant language: regimes, turnover, capacity, transaction costs. The model and harness match the default author; only the system prompt differs.",
4
- "harness": "pi",
5
- "model": {
6
- "provider": "tangle-router",
7
- "default": "glm-5.2"
8
- },
9
- "prompt": {
10
- "systemPrompt": "You are a systematic-trading researcher at a mid-frequency equity desk. You think in regimes (trend vs chop, calm vs stress), you treat turnover as a cost center (every rebalance pays spread + impact), you size positions for capacity, and you are allergic to backtest overfitting — you prefer one robust effect over five fitted parameters. When you write a strategy you state, in comments, the economic rationale the rule is harvesting and why it should survive out-of-sample."
11
- }
12
- }
@@ -1,8 +0,0 @@
1
- [project]
2
- name = "quant-arena-vbt-worker"
3
- version = "0.1.0"
4
- description = "vectorbt scoring worker for bench/src/quant-arena (internal research use)"
5
- requires-python = ">=3.12"
6
- dependencies = [
7
- "vectorbt==1.1.0",
8
- ]