@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/quant-arena/quant-loop.test.mts

@@ -0,0 +1,75 @@
+import { describe, expect, it } from 'vitest'
+import { buildAuthorPrompt, extractStrategySource, parseAuditVerdict } from './quant-loop.mts'
+import { bootstrapWindows } from './windows.ts'
+import type { AlignedBars } from './data.ts'
+
+const V2_MODULE = `
+interface Bar { date: string; open: number; high: number; low: number; close: number; volume: number }
+interface StrategyContext { symbols: string[]; t: number; history: Bar[][]; weights: number[]; equity: number }
+interface TargetPosition { symbol: string; weight: number }
+export function onBar(ctx: StrategyContext): TargetPosition[] | null {
+  if (ctx.t < 20) return null
+  return [{ symbol: ctx.symbols[0], weight: 0.5 }]
+}
+`
+
+describe('extractStrategySource — v2 contract', () => {
+  it('accepts a fenced module exporting onBar', () => {
+    const out = extractStrategySource('Here is my strategy:\n```ts\n' + V2_MODULE + '\n```\n')
+    expect(out.ok).toBe(true)
+    if (out.ok) expect(out.code).toContain('export function onBar')
+  })
+
+  it('rejects a v1 generateSignals-only reply (the contract moved)', () => {
+    const v1 = '```ts\nexport function generateSignals(bars: unknown[]): unknown[] { return [] }\n```'
+    const out = extractStrategySource(v1)
+    expect(out.ok).toBe(false)
+    if (!out.ok) expect(out.reason).toMatch(/onBar/)
+  })
+
+  it('rejects non-self-contained modules (banned identifiers)', () => {
+    const dirty = '```ts\nimport fs from "node:fs"\nexport function onBar(ctx: unknown) { return null }\n```'
+    const out = extractStrategySource(dirty)
+    expect(out.ok).toBe(false)
+    if (!out.ok) expect(out.reason).toMatch(/import/)
+  })
+
+  it('takes the LAST onBar block when the author narrates with several', () => {
+    const text = '```ts\nexport function onBar(a: unknown) { return null } // draft\n```\nbetter:\n```ts\n' + V2_MODULE + '```'
+    const out = extractStrategySource(text)
+    expect(out.ok).toBe(true)
+    if (out.ok) expect(out.code).toContain('ctx.t < 20')
+  })
+})
+
+describe('buildAuthorPrompt — authored against the v2 contract', () => {
+  it('states onBar, structural truncation, the OMS seam, and the acceptance bar', () => {
+    const universe: AlignedBars = {
+      tickers: ['IDX', 'S01'],
+      dates: Array.from({ length: 800 }, (_, i) => `d${i}`),
+      bars: [],
+    }
+    const windows = bootstrapWindows(800, { k: 4, windowDays: 200, seed: 7, warmupDays: 100 })
+    const prompt = buildAuthorPrompt({
+      universe,
+      windows,
+      baselineTable: '  (table)',
+      threshold: 0.25,
+      nTried: 3,
+    })
+    expect(prompt).toContain('export function onBar(ctx: StrategyContext)')
+    expect(prompt).toContain('ctx.history[k].length === ctx.t + 1')
+    expect(prompt).toContain('shared rebalancer')
+    expect(prompt).toContain('try #3')
+    expect(prompt).not.toContain('generateSignals')
+  })
+})
+
+describe('parseAuditVerdict', () => {
+  it('parses clean and leak verdicts, favoring the last JSON object', () => {
+    expect(parseAuditVerdict('{"verdict":"clean"}')).toEqual({ verdict: 'clean', evidence: '' })
+    const leak = parseAuditVerdict('thinking... {"verdict":"clean"} wait no {"verdict":"leak","evidence":"Date.now()"}')
+    expect(leak).toEqual({ verdict: 'leak', evidence: 'Date.now()' })
+    expect(parseAuditVerdict('no json here')).toBeNull()
+  })
+})

package/src/quant-arena/strategies/buy-hold-index/strategy.ts

@@ -0,0 +1,11 @@
+/**
+ * Pinned baseline: buy and hold the benchmark index (bars[0]) from day 0.
+ * One fill, then nothing — the cheapest possible exposure to the market.
+ */
+
+import type { Bar, Signal } from '../../types.ts'
+
+export function generateSignals(bars: Bar[][]): Signal[] {
+  const weights = bars.map((_, k) => (k === 0 ? 1 : 0))
+  return [{ t: 0, weights }]
+}

package/src/quant-arena/strategies/equal-weight/strategy.ts

@@ -0,0 +1,20 @@
+/**
+ * Pinned baseline: 1/N across the whole universe (index included), rebalanced
+ * every 21 trading days (~monthly). Harvests diversification with bounded
+ * turnover.
+ */
+
+import type { Bar, Signal } from '../../types.ts'
+
+const REBALANCE_EVERY = 21
+
+export function generateSignals(bars: Bar[][]): Signal[] {
+  const n = bars.length
+  const totalDays = bars[0]?.length ?? 0
+  const weights = bars.map(() => 1 / n)
+  const signals: Signal[] = []
+  for (let t = 0; t < totalDays; t += REBALANCE_EVERY) {
+    signals.push({ t, weights: [...weights] })
+  }
+  return signals
+}

package/src/quant-arena/strategies/sma-crossover/strategy.ts

@@ -0,0 +1,42 @@
+/**
+ * Pinned baseline: SMA(20/100) crossover momentum. Each day t (after warmup)
+ * a ticker is "trending" when its 20-day simple moving average of closes —
+ * computed from bars up to and including t — is above its 100-day SMA. Equity
+ * splits equally across trending tickers; no trend anywhere = 100% cash.
+ * Signals are emitted only when the trending set changes, so turnover stays
+ * proportional to regime changes rather than to calendar days.
+ */
+
+import type { Bar, Signal } from '../../types.ts'
+
+const FAST = 20
+const SLOW = 100
+
+export function generateSignals(bars: Bar[][]): Signal[] {
+  const n = bars.length
+  const totalDays = bars[0]?.length ?? 0
+  // Prefix sums of closes per ticker: sum[t+1] = closes[0..t] summed.
+  const prefix: number[][] = bars.map((series) => {
+    const p = new Array<number>(series.length + 1).fill(0)
+    for (let t = 0; t < series.length; t++) p[t + 1] = p[t]! + series[t]!.close
+    return p
+  })
+  const sma = (k: number, t: number, len: number): number =>
+    (prefix[k]![t + 1]! - prefix[k]![t + 1 - len]!) / len
+
+  const signals: Signal[] = []
+  let lastKey = ''
+  for (let t = SLOW - 1; t < totalDays; t++) {
+    const trending: number[] = []
+    for (let k = 0; k < n; k++) {
+      if (sma(k, t, FAST) > sma(k, t, SLOW)) trending.push(k)
+    }
+    const key = trending.join(',')
+    if (key === lastKey) continue
+    lastKey = key
+    const weights = new Array<number>(n).fill(0)
+    for (const k of trending) weights[k] = 1 / Math.max(trending.length, 1)
+    signals.push({ t, weights })
+  }
+  return signals
+}

package/src/quant-arena/types.ts

@@ -0,0 +1,133 @@
+/**
+ * QUANT-ARENA strategy contract (v2 — incremental, OMS-mediated).
+ *
+ * A strategy is ONE module exporting `onBar(ctx: StrategyContext)`. The
+ * harness calls it once per trading day, in order, and the strategy answers
+ * with either a full set of target portfolio weights or `null` ("hold").
+ *
+ *  - `ctx.history[k]` is ticker k's daily bars UP TO AND INCLUDING today
+ *    (`ctx.history[k].length === ctx.t + 1`, always). The harness physically
+ *    slices the data before every call: bars after today DO NOT EXIST in the
+ *    array, so look-ahead is structurally impossible through `history`.
+ *    `history[0]` is ALWAYS the index (the buy-and-hold benchmark asset);
+ *    `history[1..]` are the rest of the universe, in `ctx.symbols` order.
+ *  - Returning `TargetPosition[]` requests a rebalance: each entry names a
+ *    symbol and a target fraction of equity. Weights must be >= 0 and sum
+ *    to <= 1 (the remainder sits in cash earning 0). Violations kill the
+ *    candidate — fail-closed, not silently clamped. Symbols omitted from the
+ *    list are targeted to 0 (sold).
+ *  - Returning `null` means "no trade today": positions drift with prices.
+ *  - FILL TIMING: a rebalance decided at the close of day `t` fills at the
+ *    NEXT day's open. Every fill pays cost + slippage (basis points, one-way)
+ *    on traded dollars — churn is expensive.
+ *  - Strategies NEVER construct or emit `Order`s. The shared rebalancer
+ *    (oms.ts) is the only component that turns target weights into orders,
+ *    so sizing, side, and long-only enforcement are uniform across every
+ *    candidate and baseline.
+ *  - Determinism: `onBar` must be a pure function of `ctx`. No RNG, no
+ *    clock, no memory of prior calls that a re-run would not rebuild. The
+ *    lab re-runs candidates on truncated data; any divergence in decisions
+ *    up to the cutoff kills the candidate.
+ *
+ * The v1 batch contract (`generateSignals(bars): Signal[]`) remains for the
+ * pinned baselines and old fixtures; `driver.ts` wraps it so batch strategies
+ * run unchanged under the incremental harness.
+ */
+
+export interface Bar {
+  /** ISO date, e.g. '2019-03-08'. */
+  date: string
+  open: number
+  high: number
+  low: number
+  close: number
+  volume: number
+}
+
+// ---------------------------------------------------------------------------
+// v2 incremental contract.
+// ---------------------------------------------------------------------------
+
+export interface StrategyContext {
+  /** Universe tickers; symbols[0] is the benchmark index. */
+  symbols: string[]
+  /** Today's day index into the shared date axis. */
+  t: number
+  /** history[k] = bars for symbols[k], indices 0..t ONLY (length t + 1). */
+  history: Bar[][]
+  /** Current portfolio weights per symbol (drifted, marked at today's close). */
+  weights: number[]
+  /** Current equity (starts at 1). */
+  equity: number
+}
+
+export interface TargetPosition {
+  symbol: string
+  /** Target fraction of equity; >= 0, and the sum over all targets <= 1. */
+  weight: number
+}
+
+export interface Strategy {
+  /** Called once per bar. Return targets to rebalance, or null to hold. */
+  onBar(ctx: StrategyContext): TargetPosition[] | null
+}
+
+export interface StrategyModuleV2 {
+  onBar: Strategy['onBar']
+}
+
+// ---------------------------------------------------------------------------
+// OMS layer. Orders are produced ONLY by the shared rebalancer (oms.ts) from
+// the strategy's target weights — never by strategy code.
+// ---------------------------------------------------------------------------
+
+export type OrderSide = 'buy' | 'sell'
+export type OrderType = 'market' | 'limit'
+export type TimeInForce = 'day' | 'gtc' | 'ioc' | 'fok'
+
+export interface Order {
+  /** Deterministic id, unique within a run (e.g. 'qa-t42-S03'). */
+  clientOrderId: string
+  symbol: string
+  side: OrderSide
+  /** Absolute quantity in shares (fractional allowed — see oms.ts). */
+  qty: number
+  type: OrderType
+  limitPrice?: number
+  tif: TimeInForce
+  /** Free-form provenance tag, e.g. the candidate id. */
+  tag?: string
+}
+
+/** Turns target weights into orders given the portfolio state at decision
+ *  time. The one shared implementation lives in oms.ts (long-only in v2). */
+export type Rebalancer = (args: {
+  targets: TargetPosition[]
+  symbols: string[]
+  /** Equity at decision time (close of day t). */
+  equity: number
+  /** Last price per symbol at decision time (close of day t). */
+  prices: number[]
+  /** Current position value in dollars per symbol. */
+  positionValues: number[]
+  /** Decision day index — used for deterministic order ids. */
+  t: number
+  tag?: string
+}) => Order[]
+
+// ---------------------------------------------------------------------------
+// v1 batch contract (baselines + legacy fixtures; wrapped by driver.ts).
+// ---------------------------------------------------------------------------
+
+export interface Signal {
+  /** Day index into the shared date axis. */
+  t: number
+  /** Target weight per ticker (same order as `bars`). */
+  weights: number[]
+}
+
+export type GenerateSignals = (bars: Bar[][]) => Signal[]
+
+export interface StrategyModule {
+  generateSignals: GenerateSignals
+}

package/src/quant-arena/vbt-client.ts

@@ -0,0 +1,321 @@
+/**
+ * Client for the persistent vectorbt scoring worker (python/vbt-worker.py).
+ *
+ * One worker is spawned per campaign (cold numba JIT is paid once, at the
+ * `ready` handshake) and every scoring call is a JSONL request/response over
+ * stdin/stdout. The worker is the OFFICIAL scorer; the TS engine
+ * (backtest.ts) remains as contract prefilter + leak-audit substrate only.
+ *
+ * Crash discipline (the ledger-orphan posture, applied to in-flight work):
+ * if the worker dies, every pending request settles IMMEDIATELY with a
+ * `VbtWorkerCrashError` — nothing is left orphaned awaiting a dead process.
+ * The next request lazily respawns the worker, and `score()` retries a
+ * request exactly once if the crash happened while it was in flight. A
+ * second crash on the same request propagates — fail loud, not retry-loop.
+ *
+ * Environment: pinned by python/pyproject.toml + python/uv.lock and launched
+ * through `uv run --frozen`, so the scoring stack is byte-reproducible.
+ */
+
+import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from 'node:child_process'
+import { createInterface } from 'node:readline'
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import type { Bar, Signal } from './types.ts'
+
+export const VBT_PYTHON_DIR = fileURLToPath(new URL('./python', import.meta.url))
+
+export interface VbtWindowStats {
+  totalReturn: number
+  /** Positive convention (0.12 = a 12% peak-to-trough drawdown). */
+  maxDD: number
+  /** Annualized, sample-std, 252 trading days — same formula as backtest.ts. */
+  sharpe: number
+  trades: number
+}
+
+export interface VbtScore {
+  windows: VbtWindowStats[]
+  full: VbtWindowStats
+  /** Group equity per day, close-marked, equity[0] = 1. */
+  equity: number[]
+}
+
+export interface VbtScoreRequest {
+  /** T x N fill prices (next-open model). */
+  openPrices: number[][]
+  /** T x N valuation prices. */
+  closePrices: number[][]
+  /** One row per day: the decision made at that day's close, or null. */
+  weights: Array<number[] | null>
+  costBps: number
+  slippageBps: number
+  /** [start, end) day-index ranges to score. */
+  windows: Array<[number, number]>
+}
+
+export class VbtWorkerCrashError extends Error {
+  constructor(detail: string) {
+    super(`vbt worker crashed: ${detail}`)
+    this.name = 'VbtWorkerCrashError'
+  }
+}
+
+interface Pending {
+  resolve: (value: Record<string, unknown>) => void
+  reject: (error: Error) => void
+}
+
+export interface VbtWorkerOptions {
+  pythonDir?: string
+  /** Covers spawn + cold numba JIT on first start. */
+  readyTimeoutMs?: number
+  requestTimeoutMs?: number
+}
+
+export class VbtWorker {
+  private readonly pythonDir: string
+  private readonly readyTimeoutMs: number
+  private readonly requestTimeoutMs: number
+  private child: ChildProcessWithoutNullStreams | null = null
+  private ready: Promise<void> | null = null
+  private pending = new Map<number, Pending>()
+  private nextId = 1
+  private closed = false
+  private stderrTail: string[] = []
+
+  constructor(options: VbtWorkerOptions = {}) {
+    this.pythonDir = options.pythonDir ?? VBT_PYTHON_DIR
+    this.readyTimeoutMs = options.readyTimeoutMs ?? 180_000
+    this.requestTimeoutMs = options.requestTimeoutMs ?? 120_000
+  }
+
+  /** True when the pinned worker environment can be launched here: `uv` on
+   *  PATH and the lockfile present. Tests use this to skip-with-reason. */
+  static isAvailable(pythonDir = VBT_PYTHON_DIR): boolean {
+    if (!existsSync(join(pythonDir, 'uv.lock'))) return false
+    const probe = spawnSync('uv', ['--version'], { stdio: 'ignore' })
+    return probe.status === 0
+  }
+
+  private start(): Promise<void> {
+    if (this.ready !== null) return this.ready
+    if (this.closed) return Promise.reject(new Error('vbt worker: client closed'))
+    const workerPath = join(this.pythonDir, 'vbt-worker.py')
+    const child = spawn('uv', ['run', '--frozen', '--project', this.pythonDir, 'python', workerPath], {
+      cwd: this.pythonDir,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    })
+    this.child = child
+    this.stderrTail = []
+    // EPIPE on a dying worker's stdin is handled by the exit path (settle +
+    // respawn) — a stream 'error' event must not crash the host process.
+    child.stdin.on('error', () => {})
+    child.stderr.setEncoding('utf8')
+    child.stderr.on('data', (chunk: string) => {
+      this.stderrTail.push(chunk)
+      if (this.stderrTail.length > 40) this.stderrTail.shift()
+    })
+
+    this.ready = new Promise<void>((resolveReady, rejectReady) => {
+      let isReady = false
+      const readyTimer = setTimeout(() => {
+        if (!isReady) {
+          rejectReady(new Error(`vbt worker: no ready handshake within ${this.readyTimeoutMs}ms`))
+          child.kill('SIGKILL')
+        }
+      }, this.readyTimeoutMs)
+
+      const lines = createInterface({ input: child.stdout })
+      lines.on('line', (line) => {
+        let parsed: Record<string, unknown>
+        try {
+          parsed = JSON.parse(line) as Record<string, unknown>
+        } catch {
+          return // non-protocol noise on stdout is ignored, never fatal
+        }
+        if (parsed.ready === true) {
+          isReady = true
+          clearTimeout(readyTimer)
+          resolveReady()
+          return
+        }
+        const id = parsed.id
+        if (typeof id === 'number') {
+          const waiter = this.pending.get(id)
+          if (waiter !== undefined) {
+            this.pending.delete(id)
+            waiter.resolve(parsed)
+          }
+        }
+      })
+
+      child.on('exit', (code, signal) => {
+        clearTimeout(readyTimer)
+        const detail = `exit code=${code} signal=${signal} stderr: ${this.stderrTail.join('').slice(-800)}`
+        if (!isReady) rejectReady(new VbtWorkerCrashError(detail))
+        this.settleAllPending(new VbtWorkerCrashError(detail))
+        this.child = null
+        this.ready = null // next request respawns
+      })
+      child.on('error', (cause) => {
+        clearTimeout(readyTimer)
+        if (!isReady) rejectReady(new VbtWorkerCrashError(String(cause)))
+        this.settleAllPending(new VbtWorkerCrashError(String(cause)))
+        this.child = null
+        this.ready = null
+      })
+    })
+    return this.ready
+  }
+
+  /** Ledger-orphan discipline: a dead worker never leaves an unresolved
+   *  request behind — everything in flight settles NOW. */
+  private settleAllPending(error: Error): void {
+    const waiters = [...this.pending.values()]
+    this.pending.clear()
+    for (const waiter of waiters) waiter.reject(error)
+  }
+
+  private async request(payload: Record<string, unknown>): Promise<Record<string, unknown>> {
+    await this.start()
+    const child = this.child
+    if (child === null) throw new VbtWorkerCrashError('worker exited during handshake')
+    const id = this.nextId++
+    const reply = await new Promise<Record<string, unknown>>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id)
+        reject(new Error(`vbt worker: request ${id} timed out after ${this.requestTimeoutMs}ms`))
+        child.kill('SIGKILL')
+      }, this.requestTimeoutMs)
+      this.pending.set(id, {
+        resolve: (value) => {
+          clearTimeout(timer)
+          resolve(value)
+        },
+        reject: (error) => {
+          clearTimeout(timer)
+          reject(error)
+        },
+      })
+      try {
+        child.stdin.write(JSON.stringify({ id, ...payload }) + '\n')
+      } catch {
+        // A destroyed stdin means the worker is dying; the exit handler
+        // settles this pending request with VbtWorkerCrashError.
+      }
+    })
+    if (reply.ok !== true) {
+      throw new Error(`vbt worker: request failed: ${String(reply.error ?? 'unknown error')}`)
+    }
+    return reply
+  }
+
+  async ping(): Promise<string> {
+    const reply = await this.request({ op: 'ping' })
+    return String(reply.vbt)
+  }
+
+  async score(req: VbtScoreRequest): Promise<VbtScore> {
+    const payload = {
+      op: 'score',
+      open_prices: req.openPrices,
+      close_prices: req.closePrices,
+      weights: req.weights,
+      cost_bps: req.costBps,
+      slippage_bps: req.slippageBps,
+      windows: req.windows,
+    }
+    let reply: Record<string, unknown>
+    try {
+      reply = await this.request(payload)
+    } catch (cause) {
+      if (!(cause instanceof VbtWorkerCrashError)) throw cause
+      // One retry after a crash-respawn; a second crash propagates.
+      reply = await this.request(payload)
+    }
+    return {
+      windows: reply.windows as VbtWindowStats[],
+      full: reply.full as VbtWindowStats,
+      equity: reply.equity as number[],
+    }
+  }
+
+  async close(): Promise<void> {
+    this.closed = true
+    const child = this.child
+    this.child = null
+    this.ready = null
+    this.settleAllPending(new Error('vbt worker: client closed'))
+    if (child !== null) {
+      child.stdin.end()
+      await new Promise<void>((resolve) => {
+        const killTimer = setTimeout(() => {
+          child.kill('SIGKILL')
+          resolve()
+        }, 3_000)
+        child.on('exit', () => {
+          clearTimeout(killTimer)
+          resolve()
+        })
+      })
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Adapters: aligned bars + Signal[] -> the worker's matrix protocol.
+// ---------------------------------------------------------------------------
+
+/** Row-major T x N open/close price matrices from aligned bar series. */
+export function barsToPriceMatrices(bars: Bar[][]): { open: number[][]; close: number[][] } {
+  const N = bars.length
+  const T = bars[0]?.length ?? 0
+  const open: number[][] = new Array(T)
+  const close: number[][] = new Array(T)
+  for (let t = 0; t < T; t++) {
+    const openRow = new Array<number>(N)
+    const closeRow = new Array<number>(N)
+    for (let k = 0; k < N; k++) {
+      openRow[k] = bars[k]![t]!.open
+      closeRow[k] = bars[k]![t]!.close
+    }
+    open[t] = openRow
+    close[t] = closeRow
+  }
+  return { open, close }
+}
+
+/** Decision rows for the worker: weights at each decision day, null rows on
+ *  no-rebalance days (JSON's stand-in for the NaN convention). */
+export function signalsToWeightRows(signals: Signal[], totalDays: number): Array<number[] | null> {
+  const rows: Array<number[] | null> = new Array(totalDays).fill(null)
+  for (const signal of signals) {
+    if (!Number.isInteger(signal.t) || signal.t < 0 || signal.t >= totalDays) {
+      throw new Error(`signalsToWeightRows: signal.t=${signal.t} outside [0, ${totalDays})`)
+    }
+    if (rows[signal.t] !== null) throw new Error(`signalsToWeightRows: two signals share t=${signal.t}`)
+    rows[signal.t] = signal.weights
+  }
+  return rows
+}
+
+/** Score a decision record on the official vectorbt engine. */
+export async function scoreSignals(
+  worker: VbtWorker,
+  bars: Bar[][],
+  signals: Signal[],
+  config: { costBps: number; slippageBps: number },
+  windows: Array<[number, number]>,
+): Promise<VbtScore> {
+  const { open, close } = barsToPriceMatrices(bars)
+  return worker.score({
+    openPrices: open,
+    closePrices: close,
+    weights: signalsToWeightRows(signals, open.length),
+    costBps: config.costBps,
+    slippageBps: config.slippageBps,
+    windows,
+  })
+}