outcometick 1.5.2 → 1.6.2

package/runner/engine/replay.mjs

@@ -15,25 +15,50 @@
 import { Book } from './book.mjs';
 import { Portfolio } from './portfolio.mjs';
 
+
+// Mirrors LIMITS.logLineChars / LIMITS.logBytesPerRun, and does NOT import
+// them. This file is baked into the sandbox image, which contains runner/ and
+// nothing else — an import of api/ resolves fine on a developer's machine and
+// on the worker, and then fails inside every container with
+// ERR_MODULE_NOT_FOUND. otreplay.py mirrors the same two numbers for the same
+// reason. replay-limits.test.mjs asserts both copies still equal the contract.
+const LOG_LINE_CHARS = 512;
+const LOG_BYTES_PER_RUN = 2 * 1024 * 1024;
+
+// A settlement recompute is a once-per-market claim, so these are generous.
+// They exist because `crosschecks` rides the same result line as everything
+// else: unbounded, it is an output channel with no budget. Mirrored in
+// otreplay.py.
+const MAX_CROSSCHECKS_PER_MARKET = 16;
+const CROSSCHECK_CLAIMED_CHARS = 32;
+
+
 /** Event kinds the loop understands, in the order they dispatch. */
 export const EVENT_KINDS = Object.freeze(['tick', 'book', 'trade']);
 
 /**
  * Per-event budget.
  *
- * Measured over the strategy's own hook, not the loop around it. A p99 breach
- * kills the shard rather than the run: one pathological market must not cost
- * the customer the other 719.
+ * Measured over the strategy's own hook, not the loop around it. Sustained cost
+ * over budget kills the shard rather than the run: one pathological market must
+ * not cost the customer the other 719.
  */
 export class BudgetMonitor {
-  constructor({ limitMicros = 400, sampleFloor = 200, tolerance = 0.01 } = {}) {
+  constructor({ limitMicros = 400, sampleFloor = 2000 } = {}) {
     this.limitMicros = limitMicros;
     this.sampleFloor = sampleFloor;
-    this.tolerance = tolerance;
     this.count = 0;
     this.breaches = 0;
     this.maxMicros = 0;
     this.totalMicros = 0;
+    // The most recent `sampleFloor` events, as a ring. One monitor covers the
+    // WHOLE run, so a lifetime mean is diluted by however much came before: a
+    // strategy that runs 18,000 events at 8us and then 2,000 at 2,000us has a
+    // lifetime mean of 207us and passes, while its last 2,000 events are
+    // continuously 5x over budget. The window is what makes "sustained" local.
+    this.window = new Float64Array(sampleFloor);
+    this.windowSum = 0;
+    this.windowAt = 0;
   }
 
   record(micros) {
@@ -41,14 +66,64 @@ export class BudgetMonitor {
     this.totalMicros += micros;
     if (micros > this.maxMicros) this.maxMicros = micros;
     if (micros > this.limitMicros) this.breaches += 1;
+    this.windowSum += micros - this.window[this.windowAt];
+    this.window[this.windowAt] = micros;
+    this.windowAt = (this.windowAt + 1) % this.window.length;
+  }
+
+  /** Mean of the most recent `sampleFloor` events. Zero until the window fills. */
+  get windowMicros() {
+    return this.count >= this.sampleFloor ? this.windowSum / this.window.length : 0;
   }
 
   /**
-   * A single slow event is not a breach — a JIT warm-up or a GC pause is not
-   * the strategy's fault. Sustained breach past the p99 tolerance is.
+   * SUSTAINED cost, which is what the limit is for and what customers are told
+   * it means ("sustained breach kills the shard"): the mean of the most recent
+   * `sampleFloor` events, not the tail and not the lifetime.
+   *
+   * This judged the breach RATE against a 1% tolerance, and it was measuring
+   * the wrong machine. The budget brackets each hook with two wall-clock reads,
+   * on a 2-core box where the worker is decompressing and feeding stdin the
+   * whole time and the sandbox holds one vCPU — so an event that gets
+   * descheduled is recorded as an event the strategy spent 4ms in. Measured
+   * inside the real image: the same strategy on an idle host averages 7.8us
+   * with a 766us worst case, and under contention averages 20.9us with a 4090us
+   * worst case. Nothing about the strategy changed.
+   *
+   * The page's own sample was rejected in production at avg 72us — a fifth of
+   * its 400us budget — because 1.1% of its events had been interrupted. The
+   * breaches were not even front-loaded, so a longer warm-up floor could never
+   * have fixed it: measured in-image they land at events 171, 2368, 3201 and so
+   * on, which is the shape of GC and scheduling, not of a slow strategy.
+   *
+   * The mean is what actually predicts the thing this protects — the 20-minute
+   * wall clock is mean times event count — and it is not fooled by a machine
+   * that takes the CPU away. A strategy that really is slow raises the mean; an
+   * interrupted one does not. A single hook that never returns is still caught,
+   * by the run deadline, which is where that belongs.
+   *
+   * WINDOWED, NOT LIFETIME. One monitor covers the whole run, so a lifetime
+   * mean lets a cheap prefix pay for an expensive phase: 18,000 events at 8us
+   * followed by 2,000 at 2,000us averages 207us and passes, while the strategy
+   * has been 5x over budget for its last two thousand events. The window is
+   * what makes "sustained" mean sustained rather than "on average, eventually".
+   *
+   * `breaches` and `max_micros` stay in the summary. They are good diagnostics.
+   * They are not a verdict.
+   *
+   * KNOWN AND DELIBERATE GAP: a low-frequency, very heavy event slips through.
+   * One 500ms hook among 2,000 events at 8us is a window mean of 258us, inside
+   * budget; it would take 800ms to trip on its own. A per-hook hard cap would
+   * close it, and it is not being added, because this limit has now been wrong
+   * twice and BOTH times the same way — a one-off cost judged as a sustained
+   * one, killing a strategy that was fine. Building an index on the first tick
+   * is a legitimate 300ms hook. The wall clock already bounds total resource
+   * use, so the cap would buy protection against a failure nobody has seen at
+   * the cost of the exact mistake already made twice. Add it when a real run
+   * demonstrates the need, not before.
    */
   get breached() {
-    return this.count >= this.sampleFloor && this.breaches / this.count > this.tolerance;
+    return this.count >= this.sampleFloor && this.windowMicros > this.limitMicros;
   }
 
   get avgMicros() { return this.count ? this.totalMicros / this.count : 0; }
@@ -59,6 +134,10 @@ export class BudgetMonitor {
       breaches: this.breaches,
       breach_rate: this.count ? this.breaches / this.count : 0,
       avg_micros: this.avgMicros,
+      // The number the verdict is actually made on. Without it a rejection
+      // shows a lifetime average comfortably inside budget and reads as a lie.
+      window_micros: this.windowMicros,
+      window_events: this.sampleFloor,
       max_micros: this.maxMicros,
       limit_micros: this.limitMicros,
     };
@@ -124,7 +203,7 @@ function bookView(book) {
   });
 }
 
-function createCtx({ params, portfolio, marketId, market, logLimit, references, series, rng }) {
+function createCtx({ params, portfolio, marketId, market, logBudget, references, series, rng }) {
   const history = [];
   const logs = [];
   const crosschecks = [];
@@ -171,8 +250,20 @@ function createCtx({ params, portfolio, marketId, market, logLimit, references,
     },
 
     log(msg) {
-      if (logs.length >= logLimit) { logTruncated = true; return; }
-      logs.push(`${now} ${String(msg)}`);
+      // BYTES FOR THE WHOLE RUN, not lines per market. The old shape — 10,000
+      // lines per market, no length cap — let a run emit the archive it had
+      // just paid a market-day for into a file the customer downloads. See
+      // LIMITS.logBytesPerRun for the arithmetic that picks these numbers.
+      if (logBudget.spent >= logBudget.bytes) { logTruncated = true; return; }
+      const line = `${now} ${String(msg)}`.slice(0, logBudget.lineChars);
+      // BYTES, not string length. logs.txt is UTF-8, and `.length` counts
+      // UTF-16 units — so a run logging Chinese spent a third of what it
+      // wrote, and the archive could reach three times the 2 MB this budget
+      // advertises. The line cap stays in characters because it is about
+      // being readable; the run cap is about how much data leaves with the
+      // customer, and that is measured in bytes.
+      logBudget.spent += Buffer.byteLength(line, 'utf8') + 1;
+      logs.push(line);
     },
 
     /** Seeded generator — the only randomness available, and it is recorded. */
@@ -240,9 +331,19 @@ function createCtx({ params, portfolio, marketId, market, logLimit, references,
       // recompute match that never happened. The cross-check panel's whole
       // value is that it is the ARCHIVE's answer, not the strategy's.
       const official = market?.outcome ?? null;
+      // BOUNDED, for the same reason ctx.log is. `outcome` is whatever the
+      // strategy passed and this can be called on every event, so an unbounded
+      // push here is an unmetered output channel wearing a different name: the
+      // whole array is serialised onto the authenticated result line, sent,
+      // and parsed by the worker before anything downstream gets to ignore it.
+      // The panel only ever shows an aggregate, so nothing of value is lost by
+      // capping — a settlement recompute is a once-per-market claim.
+      if (crosschecks.length >= MAX_CROSSCHECKS_PER_MARKET) return;
       crosschecks.push({
         market_id: marketId,
-        claimed: outcome,
+        claimed: typeof outcome === 'string'
+          ? outcome.slice(0, CROSSCHECK_CLAIMED_CHARS)
+          : String(outcome).slice(0, CROSSCHECK_CLAIMED_CHARS),
         official,
         match: official === outcome,
       });
@@ -293,11 +394,32 @@ const HOOK_FOR = { tick: 'on_tick', book: 'on_book', trade: 'on_trade' };
  *   decision. Zero is "as captured"; the latency panel is this same replay at
  *   100ms, 250ms, 500ms, 1s and 2s.
  */
+/**
+ * A log allowance for one run.
+ *
+ * Bytes, not lines, and shared by every market in the run — see
+ * LIMITS.logBytesPerRun for why those two choices are the whole fix.
+ */
+export function makeLogBudget({
+  bytes = LOG_BYTES_PER_RUN,
+  lineChars = LOG_LINE_CHARS,
+} = {}) {
+  return { bytes, lineChars, spent: 0 };
+}
+
 export function replayMarket({
   market, events, strategy, hooks,
   portfolio = null,
   fillDelayMs = 0,
-  logLimit = 10_000,
+  /**
+   * The run's remaining log allowance, SHARED ACROSS MARKETS.
+   *
+   * Passed in rather than created here, because a per-market allowance is what
+   * the old limit was and what made the log channel an export route: 386
+   * markets a day each got their own budget. One object for the whole run is
+   * the fix — the harness makes it once and hands the same one to every market.
+   */
+  logBudget = null,
   budget = null,
   references = null,
   series = null,
@@ -338,7 +460,7 @@ export function replayMarket({
     portfolio: pf,
     marketId,
     market: engineMarket,
-    logLimit,
+    logBudget: logBudget ?? makeLogBudget(),
     references,
     series,
     rng: makeRng(seed),

package/runner/engine/report.mjs

@@ -8,16 +8,6 @@
 // Nothing in this module can see the strategy. It reads the trade and fill logs
 // the engine produced, so a report cannot be tuned by the thing it is judging.
 
-/** Delays the latency panel re-prices every fill at. */
-export const LATENCY_STEPS = Object.freeze([
-  { label: 'as captured (0 ms)', ms: 0 },
-  { label: '+100 ms', ms: 100 },
-  { label: '+250 ms', ms: 250 },
-  { label: '+500 ms', ms: 500 },
-  { label: '+1 s', ms: 1000 },
-  { label: '+2 s', ms: 2000 },
-]);
-
 /** Entry-price buckets for the calibration panel. */
 export const CALIBRATION_BUCKETS = Object.freeze([
   [0.0, 0.1], [0.1, 0.2], [0.2, 0.3], [0.3, 0.4], [0.4, 0.5],
@@ -46,6 +36,64 @@ const r4 = (x) => (Number.isFinite(x) ? Number(x.toFixed(4)) : null);
  */
 const collateralOf = (t) => (t.entry_px ?? 0) * (t.size ?? 0);
 
+/**
+ * The most money this strategy had at risk AT ONE TIME.
+ *
+ * THE NUMBER THAT ANSWERS "how much do I need to run this", and the one this
+ * report was missing. Summing every entry answers a different question: the
+ * sample strategy opened 1,676 positions over fifteen days and never held more
+ * than one, so its entries total $72,175 while it never needed more than $80.
+ * Dividing a loss by the sum therefore reported −4.35% for a strategy that had
+ * burned through its stake thirty-nine times over.
+ *
+ * Computed by sweeping the open/close events, so overlapping positions add up
+ * and sequential ones do not. Trades with no timestamps are skipped rather
+ * than assumed concurrent — an unknown that inflates the peak would make the
+ * strategy look safer to fund than it is.
+ */
+function peakCapital(trades) {
+  const events = [];
+  for (const t of trades) {
+    if (t.opened_ms == null || t.closed_ms == null) continue;
+    const amt = collateralOf(t);
+    if (!(amt > 0)) continue;
+    events.push([t.opened_ms, amt]);
+    events.push([t.closed_ms, -amt]);
+  }
+  // Closes before opens at the same instant: a position that ends exactly when
+  // the next begins did not need both stakes at once.
+  events.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+  let cur = 0;
+  let peak = 0;
+  for (const [, delta] of events) {
+    cur += delta;
+    if (cur > peak) peak = cur;
+  }
+  return peak;
+}
+
+/** Share of the run's span with a position open. Money idle is money wasted. */
+function holdingRatio(trades) {
+  const withTimes = trades.filter((t) => t.opened_ms != null && t.closed_ms != null);
+  if (withTimes.length === 0) return null;
+  const first = Math.min(...withTimes.map((t) => t.opened_ms));
+  const last = Math.max(...withTimes.map((t) => t.closed_ms));
+  const span = last - first;
+  if (!(span > 0)) return null;
+  // Union of the intervals, not their sum: two overlapping positions are one
+  // stretch of being in the market, and summing them can exceed the span.
+  const spans = withTimes
+    .map((t) => [t.opened_ms, t.closed_ms])
+    .sort((a, b) => a[0] - b[0]);
+  let held = 0;
+  let [s, e] = spans[0];
+  for (const [a, b] of spans.slice(1)) {
+    if (a > e) { held += e - s; [s, e] = [a, b]; } else if (b > e) e = b;
+  }
+  held += e - s;
+  return held / span;
+}
+
 /**
  * Headline metrics — the twelve cells at the top of the report.
  */
@@ -62,6 +110,8 @@ export function metrics(trades, { feesPaid = 0, days = 1 } = {}) {
   const wins = pnls.filter((p) => p > 0);
   const losses = pnls.filter((p) => p < 0);
   const collateral = sum(closed.map(collateralOf));
+  const peak = peakCapital(closed);
+  const hold = holdingRatio(closed);
 
   const equity = equityCurve(closed);
   const dd = maxDrawdown(equity.map((p) => p.equity));
@@ -96,7 +146,25 @@ export function metrics(trades, { feesPaid = 0, days = 1 } = {}) {
     max_drawdown_abs: r2(dd.abs),
     sharpe: sharpe == null ? null : r2(sharpe),
     trades: closed.length,
+    // Distinct markets the strategy actually took a position in. The engine
+    // does not stop a strategy trading a market twice, so `trades / markets`
+    // is only an entry rate for strategies that enter once — this one is an
+    // entry rate for all of them, and equals `trades` in the common case.
+    markets_traded: new Set(closed.map((t) => t.market_id)).size,
+    // Net P&L over the SUM of every entry — "for each dollar traded, how much
+    // was made". Renamed on the page to say that, because "return on capital"
+    // reads as an account return and is not one: the same stake recycled a
+    // thousand times makes this number a thousand times smaller than what
+    // happened to the money.
     return_on_collateral: collateral > 0 ? r4(netPnl / collateral) : null,
+    // THE ACCOUNT NUMBER. Net P&L over the most that was ever at risk at once,
+    // which is what someone funding this strategy actually has to put up.
+    peak_capital: r2(peak),
+    return_on_peak: peak > 0 ? r4(netPnl / peak) : null,
+    // How much of the run had a position open. The sample strategy is in the
+    // market 13% of the time, which is the other half of why the two return
+    // figures differ by three orders of magnitude.
+    holding_ratio: hold == null ? null : r4(hold),
     // Cents of edge per contract: what the outcome was worth minus what was
     // paid, averaged. This is the number that says whether there was an edge
     // at all, as opposed to a lucky run of variance.
@@ -202,7 +270,7 @@ export function calibration(trades) {
     const implied = mean(inBucket.map((t) => t.entry_px));
     const realized = mean(inBucket.map((t) => (t.outcome === t.side ? 1 : 0)));
     return {
-      bucket: `${lo.toFixed(2)} – ${hi.toFixed(2)}`,
+      bucket: `${lo.toFixed(2)}-${hi.toFixed(2)}`,
       lo,
       hi,
       implied: r4(implied),
@@ -308,26 +376,6 @@ export function splitByMarket(trades, marketMeta = new Map()) {
   return rows;
 }
 
-/**
- * The latency panel: net PnL if every fill had landed later.
- *
- * The rows come from re-running the replay at each delay, which the caller
- * does — this only shapes the result. We keep event time, upstream server time
- * and our receive time separate on every row, which is what makes re-pricing at
- * an arbitrary delay meaningful rather than a guess.
- */
-export function latencyPanel(resultsByDelay) {
-  const base = resultsByDelay.find((r) => r.delayMs === 0)?.netPnl ?? 0;
-  return resultsByDelay.map((r) => ({
-    label: LATENCY_STEPS.find((s) => s.ms === r.delayMs)?.label ?? `+${r.delayMs} ms`,
-    delay_ms: r.delayMs,
-    net_pnl: r2(r.netPnl),
-    // Relative to the as-captured run, so the shape of the decay is readable
-    // without dividing in your head.
-    ratio: base === 0 ? null : r4(r.netPnl / base),
-    unprofitable: r.netPnl < 0,
-  }));
-}
 
 /**
  * The parameter sweep grid.
@@ -365,9 +413,9 @@ export function sweepPanel(cells, { xParam, yParam, metric = 'sharpe' }) {
  * makes the rest of it checkable, so it is never summarised away.
  */
 export function buildReport({
-  runId, submittedAt, manifest, scope,
+  runId, submittedAt, manifest, scope, sourceSha256 = null,
   trades, fills, marketSummaries, marketMeta,
-  feesPaid = 0, latency = [], sweep = null, coverage = null,
+  feesPaid = 0, fillDelayMs = 0, sweep = null, coverage = null,
   crosschecks = [], budget = null, seed = null, scanned = {},
 }) {
   const closed = trades.filter((t) => Number.isFinite(t.pnl));
@@ -376,6 +424,10 @@ export function buildReport({
 
   return {
     run_id: runId,
+    // WHICH CODE PRODUCED THIS. The source itself is no longer in the archive
+    // — a report is a thing you forward to someone and the strategy is not —
+    // so this is what answers "which version of my strategy was this?".
+    source_sha256: sourceSha256,
     generated_ms: submittedAt,
     sdk_schema: manifest?.schema ?? null,
     language: manifest?.language ?? null,
@@ -409,7 +461,17 @@ export function buildReport({
     }),
     split: splitByMarket(closed, marketMeta ?? new Map()),
     slippage: slippage(fills ?? []),
-    latency: latencyPanel(latency),
+    // THE DELAY THIS RUN WAS PRICED AT, not a comparison table.
+    //
+    // There used to be five extra replays at 100ms..2s, then one, and the
+    // panel that compared them. It is gone: a run now replays ONCE, at
+    // whatever delay the submitter asked for, which is both the fastest answer
+    // and the only one that is a measurement rather than an extrapolation.
+    //
+    // It has to be IN the report, because it changes every number in it and
+    // nothing else in here would tell a reader whether they are looking at a
+    // zero-latency run or a 250ms one.
+    fill_delay_ms: fillDelayMs,
     sweep,
     coverage,
     budget,