outcometick 1.5.2 → 1.6.2

package/README.md

@@ -5,7 +5,7 @@
   scripts/publish-sdk-repos.mjs and overwritten wholesale on each publish.
   An edit made here survives until the next publish and then disappears.
 
-  Generated from monorepo revision 6044c75a16de3ee60870d59cb44a25096bcc7258.
+  Generated from monorepo revision 5c26d7ed75e2172fff108af183089aa1460652ba.
 -->
 
 # outcometick
@@ -117,3 +117,10 @@ Writing your strategy in Python instead? The SDK for it is
 those too — it is the one CLI for both languages.
 
 Full reference: https://outcometick.com/docs/sdk
+
+## Links
+
+- [outcometick.com](https://outcometick.com) — what this is, and what the data covers
+- [Run a backtest](https://outcometick.com/backtest) — paste a strategy, watch it run
+- [SDK reference](https://outcometick.com/docs/sdk) — manifest, hooks, `ctx`, limits
+- [Data API](https://outcometick.com/docs) — the archive these strategies read

package/api/lib/backtest-contract.mjs

@@ -15,7 +15,52 @@ import { FIRST_COMPLETE_DAY } from './coverage-window.mjs';
 export const SCHEMA_VERSION = 1;
 
 /** SDK version reported by the docs page and stamped into every report. */
-export const SDK_VERSION = '1.5.2';
+export const SDK_VERSION = '1.6.2';
+
+/**
+ * The tag of the sandbox images, and the ONLY place it is written down.
+ *
+ * The harness is baked into the image, so this tag is really a protocol
+ * version: worker and harness have to agree on how results come back. Bump it
+ * whenever that agreement changes, or a host still holding the previous images
+ * runs the old harness under the new worker — and the failure is silent. The
+ * run completes, produces no result line, and every job is refunded while
+ * looking like a strategy problem.
+ *
+ * 1.9.0: the per-event budget judges SUSTAINED cost — the mean — instead of the
+ * rate of events over the limit. The old rule measured the machine, not the
+ * strategy: the budget brackets each hook with two wall-clock reads, and the
+ * sandbox holds one vCPU on a two-core box while the worker decompresses and
+ * feeds stdin, so an interrupted event was recorded as an event the strategy
+ * spent milliseconds in. The page's own sample was rejected in production at an
+ * average of 72us against this 400us budget because 1.1% of its events had been
+ * descheduled. Measured in-image: the same strategy averages 7.8us idle and
+ * 20.9us under contention, worst case 766us and 4090us respectively — and the
+ * slow events are scattered, not front-loaded, so 1.8.0's higher floor could
+ * never have fixed it. The mean is what the wall clock is made of, and it is
+ * the thing customers were already told the limit means ("sustained breach").
+ * 1.8.0: the per-event budget no longer judges a strategy on its first 200
+ * events. That window is where lazy imports and every first call land, so a
+ * strategy that breached 7 times in 151,606 events — 0.005%, against a 1%
+ * tolerance — was killed because 3 of those 7 fell inside the sample. Both
+ * engines now need 2,000 events before the ratio means anything, and a
+ * conformance test pins the two defaults to the same number: this value decides
+ * whether a run is rejected, so a divergence is the same strategy passing in
+ * one language and failing in the other. THE FLOOR LIVES IN THE IMAGE, which is
+ * why this tag moves — a worker on the old image keeps judging at 200.
+ * 1.7.0: a book snapshot now carries only the side it is about, because
+ * Polymarket publishes one side per row and an empty ladder is a side the
+ * engine resets. Both harnesses track the last ask ladder per side and take the
+ * LOWEST ask rather than element zero — the archive sorts descending on one
+ * venue and ascending on the other, so an index read the worst offer on one of
+ * them.
+ * 1.6.0: the harness acknowledges each market-day it finishes replaying, on a
+ * new channel, so a watching page counts finished work instead of queued bytes.
+ * 1.5.0: results moved from fd 3 to the container's stdout. Docker never
+ * forwarded a fourth descriptor, so fd 3 was closed inside the container and no
+ * containerised run had ever returned anything.
+ */
+export const SANDBOX_IMAGE_TAG = '1.14.0';
 
 // ---------------------------------------------------------------------------
 // Languages
@@ -162,6 +207,81 @@ export const CAPTURE_WINDOWS = Object.freeze({
   }),
 });
 
+/**
+ * How densely the venue's book stream was actually captured, over time.
+ *
+ * NOT PART OF THE CONTRACT DOCUMENT, and deliberately absent from every
+ * customer-facing surface of the backtest. The data product discloses its own
+ * capture cadence — that is the archive's business, and chainlink-data's guide
+ * states it. A backtest is a different promise: it says "here is what your
+ * strategy would have done", and a reader should not have to hold a table of
+ * sampling rates in their head to know whether the first half of their report
+ * is comparable to the second.
+ *
+ * So this exists to REMOVE the difference rather than to report it. See
+ * bookThrottleMs: EACH ASSET in a run is replayed at the coarsest cadence its
+ * own date range contains, so no asset changes density partway through and no
+ * boundary is ever visible. Per asset, not per run — see makeBookThrottle for
+ * why levelling a whole basket to its coarsest member was rejected.
+ *
+ * MIRRORS PRICE_CHANGE_THROTTLE_HISTORY and PREDICT_BOOK_THROTTLE_HISTORY in
+ * chainlink-data (scripts/lib/delivery.mjs). Two copies of a fact drift — this
+ * repo has the scars — so if this is ever wrong the symptom is a backtest
+ * quietly replaying at the wrong density, which nothing else would catch. The
+ * dates are settled history and do not move; a NEW entry is the only edit this
+ * should ever need, and it has to be made in both places on the same day.
+ *
+ * `from` is the first day the entry applies to. The collector dates a change to
+ * the day AFTER it was deployed, because the deploy day is mixed and claiming
+ * the finer cadence for it would promise more than the archive holds.
+ */
+const BOOK_CAPTURE = Object.freeze({
+  polymarket: Object.freeze([
+    Object.freeze({ from: '2026-06-06', defaultMs: 500, perAsset: Object.freeze({}) }),
+    Object.freeze({ from: '2026-08-25', defaultMs: 500, perAsset: Object.freeze({ BTC: 20, ETH: 100 }) }),
+  ]),
+  predict: Object.freeze([
+    Object.freeze({ from: '2026-06-12', defaultMs: 1000, perAsset: Object.freeze({}) }),
+    // 0 means every upstream frame was archived.
+    Object.freeze({ from: '2026-08-25', defaultMs: 0, perAsset: Object.freeze({}) }),
+  ]),
+});
+
+/**
+ * The cadence ONE ASSET replays at within a run: the COARSEST its date range
+ * contains. Ask per asset; there is no run-wide answer by design.
+ *
+ * A range that crosses a change gets the older, sparser setting for all of it.
+ * That is the only choice that makes a single report self-consistent without
+ * making the reader aware of anything: fine days are thinned to match coarse
+ * ones, rather than a report where the same strategy fills differently in its
+ * first month than its last and nothing says why.
+ *
+ * Returns 0 when nothing in the range was throttled, which means "replay every
+ * row" and is the case the whole mechanism disappears in.
+ */
+export function bookThrottleMs({ venue, asset, from, to }) {
+  const history = BOOK_CAPTURE[venue];
+  if (!history) return 0;
+  const a = String(asset ?? '').toUpperCase();
+  let coarsest = 0;
+  for (let i = 0; i < history.length; i += 1) {
+    const entry = history[i];
+    const next = history[i + 1];
+    // Does [from, to] overlap the window this entry governs?
+    if (to < entry.from) continue;
+    if (next && from >= next.from) continue;
+    // `'*'` asks for the coarsest window ANY asset had — the answer for a row
+    // whose asset could not be determined, which must never be thinned less
+    // than the rows that could be.
+    const ms = a === '*'
+      ? Math.max(entry.defaultMs, ...Object.values(entry.perAsset))
+      : (entry.perAsset[a] ?? entry.defaultMs);
+    if (ms > coarsest) coarsest = ms;
+  }
+  return coarsest;
+}
+
 // ---------------------------------------------------------------------------
 // Reference feeds (external data, resolved before the run)
 // ---------------------------------------------------------------------------
@@ -172,18 +292,34 @@ export const CAPTURE_WINDOWS = Object.freeze({
  * reports on different days. Feeds are resolved into a dataset ahead of the run
  * and replayed on the same clock as everything else.
  *
- * Binance klines are pre-downloaded from data.binance.vision, which publishes
- * one zip per symbol per day; see scripts/fetch-binance-reference.mjs.
+ * Binance publishes these at data.binance.vision; scripts/fetch-binance.mjs
+ * pulls them onto the worker ahead of time.
+ *
+ * EVERY ENTRY HERE IS ONE BINANCE ACTUALLY PUBLISHES. That is not a truism: the
+ * first version of this table offered `spot:100ms` and `perp:1s`, and Binance
+ * publishes neither — no 100ms klines at all, and futures klines stop at 1m. A
+ * strategy declaring one would have passed validation, queued, been billed, and
+ * received an empty feed with no error. Checked against the archive, not
+ * assumed; `scripts/fetch-binance.mjs --check` re-checks.
  */
 export const REFERENCE_FEEDS = Object.freeze({
+  // spot/daily/klines/<SYM>/1s/ and /1m/.
   'binance:{symbol}:spot:1s': Object.freeze({ kind: 'klines', market: 'spot', interval: '1s' }),
-  'binance:{symbol}:spot:100ms': Object.freeze({ kind: 'klines', market: 'spot', interval: '100ms', assets: Object.freeze(['BTC', 'ETH']) }),
-  'binance:{symbol}:perp:1s': Object.freeze({ kind: 'klines', market: 'perp', interval: '1s' }),
-  'binance:{symbol}:funding': Object.freeze({ kind: 'funding', market: 'perp', interval: null }),
+  'binance:{symbol}:spot:1m': Object.freeze({ kind: 'klines', market: 'spot', interval: '1m' }),
 });
 
-/** Symbols we carry a reference feed for. */
-export const REFERENCE_SYMBOLS = Object.freeze(['btcusdt', 'ethusdt', 'solusdt', 'xrpusdt']);
+/**
+ * Symbols we carry a reference feed for — the assets we sell backtests on that
+ * Binance also lists ON SPOT.
+ *
+ * No HYPE: Binance has no HYPEUSDT spot pair. It has a perp one, and an earlier
+ * version of this table offered perp feeds for that reason — dropped, because
+ * spot is what these strategies are pricing against and a perp mark is a
+ * different number wearing the same name.
+ */
+export const REFERENCE_SYMBOLS = Object.freeze([
+  'btcusdt', 'ethusdt', 'solusdt', 'xrpusdt', 'bnbusdt', 'dogeusdt',
+]);
 
 /** `binance:btcusdt:spot:1s` -> {feed, symbol} or null if it is not a feed we carry. */
 export function parseReferenceFeed(name) {
@@ -209,9 +345,52 @@ export function parseReferenceFeed(name) {
  * single ordered stream — it cannot be sharded, so it runs slower and bills at
  * a multiple.
  */
+/**
+ * The two ways a strategy sees the range. NEITHER COSTS MORE THAN THE OTHER.
+ *
+ * `session` used to bill at 3x, on the stated grounds that it "cannot be
+ * sharded, so it runs slower". That reason was not true: `shardable` is read
+ * nowhere outside this table and its tests — `market` mode has never actually
+ * been sharded, so both modes occupy one worker for the same time. Charging
+ * three times for a cost difference that does not exist is the one thing a
+ * product sold on honest reporting cannot do. Owner's call, 2026-08-25: the
+ * multiplier is gone. A market-day costs a credit, whichever mode reads it.
+ *
+ * If sharding is ever built, price it then — from the difference it actually
+ * makes, not from the difference it was supposed to make.
+ */
+/**
+ * The market intervals a backtest can ask for.
+ *
+ * A CLOSED set, and short on purpose: the archive contains prediction markets
+ * at 5m and 15m and at no other length. Everything else the catalog carries at
+ * other intervals -- 1s through 1mo -- is klines, which is Binance spot price
+ * data, not a market anyone can take a position in.
+ *
+ * The default is 5m alone rather than both, because the two settle on the same
+ * stream but behave nothing alike, and a run that quietly mixed them was
+ * answering a question nobody asked.
+ */
+/**
+ * The fill delay a run may ask the latency panel to re-price at.
+ *
+ * ONE delay, and OFF BY DEFAULT. Each one is another full replay of the range:
+ * with the five that used to be built in, five sixths of a run's wall clock
+ * went on a six-row table nobody had asked for, and a sixteen market-day run
+ * could not finish inside its twenty-minute budget at all.
+ *
+ * The panel is worth having -- a strategy that only makes money at zero
+ * latency loses it in production -- but it is a question the submitter asks,
+ * one delay at a time, rather than one every run answers by default.
+ */
+export const MAX_LATENCY_MS = 10_000;
+
+export const MARKET_INTERVALS = Object.freeze(['5m', '15m']);
+export const DEFAULT_INTERVALS = Object.freeze(['5m']);
+
 export const MODES = Object.freeze({
-  market: Object.freeze({ shardable: true, rateMultiplier: 1 }),
-  session: Object.freeze({ shardable: false, rateMultiplier: 3 }),
+  market: Object.freeze({ shardable: true }),
+  session: Object.freeze({ shardable: false }),
 });
 
 export const KNOWN_MODES = Object.freeze(Object.keys(MODES));
@@ -229,18 +408,152 @@ export const LIMITS = Object.freeze({
   maxFiles: 6,
   maxTotalSourceBytes: 256 * 1024,
   maxFileNameLength: 96,
-  maxSeriesBytes: 32 * 1024 * 1024,
+  // A series is the submitter's own CSV, and it rides in with the source — so
+  // its budget has to be its own. It used to be 32MB while the whole submission
+  // was capped at 256KB, which meant the advertised number was thirty times
+  // what could actually be sent: a 90-day minute series is ~2.7MB and 256KB is
+  // eight days of one.
+  //
+  // 8MB is 90 days of minute data on four series with room over. Kept well
+  // under a body a JSON parse can hold comfortably, because these arrive as
+  // strings in the submit payload.
+  // 4MB, sized on the longest range we actually sell: 90 days of minute data is
+  // ~2.7MB. 8MB was room for nothing anyone can ask for, and it doubled the
+  // request body the API has to hold — on a box with 800MB free that also
+  // carries live Stripe keys and the production database tunnel.
+  //
+  // This is the interim number. A series belongs in R2 by a presigned PUT from
+  // the browser, not in a JSON body routed through this machine; when that
+  // lands, the body limit goes back to 16kb and this cap stops mattering.
+  maxSeriesBytes: 4 * 1024 * 1024,
   maxSeriesCount: 4,
   perEventBudgetMicros: 400,
-  memoryBytes: 8 * 1024 * 1024 * 1024,
-  vcpu: 4,
-  wallClockMs: 20 * 60 * 1000,
-  logLinesPerMarketDay: 10_000,
+  /**
+   * What `ctx.log` may emit, per line and per run.
+   *
+   * ctx.log is for reading, not for exporting. The limit used to be 10,000
+   * lines PER MARKET with no length cap at all — and polymarket has ~386
+   * markets a day, so a run could emit millions of arbitrarily long lines into
+   * logs.txt, which the customer then downloads. That is the archive itself
+   * leaving through a channel priced at nothing.
+   *
+   * The numbers are chosen against the subscription, which is what the same
+   * bytes cost through the front door: $5/month for 30 rolling days across
+   * seven assets is $0.0238 per market-day. A decoded market-day is 573 MB, so
+   * at a 2 MB budget, exporting one through the log channel takes 287 runs —
+   * about $2.87 of credits, or 121x what it costs to simply buy it. And 287
+   * repeat purchases of the same market-day by one subject is the loudest
+   * pattern in the ledger.
+   *
+   * Legitimate use is nowhere near it: sixteen market-days logging one line per
+   * market is 0.59 MB, which leaves more than triple the headroom.
+   */
+  logLineChars: 512,
+  logBytesPerRun: 2 * 1024 * 1024,
+  // What the worker box actually has, not what sounds generous. It is a 2 vCPU
+  // / 4 GB VPS: `--cpus=4` is refused outright by the daemon ("range of CPUs is
+  // from 0.01 to 2.00"), so the advertised 4 vCPU / 8 GB could never have run a
+  // single container. Nobody bought under those numbers — credits have no
+  // Stripe price yet — but /docs/sdk was printing them.
+  //
+  // 1 vCPU is not a cut, it is the truth: a strategy is single-threaded by
+  // construction (the analysers reject threading and Worker, and the images
+  // pin OMP_NUM_THREADS and friends to 1), so the second core was never
+  // reachable from inside. It stays with the worker, which has to keep feeding
+  // events down stdin while the sandbox runs. 2 GB leaves room for the worker,
+  // the docker daemon and gVisor's own footprint.
+  memoryBytes: 2 * 1024 * 1024 * 1024,
+  vcpu: 1,
+  // The REPLAY budget: it starts when the strategy does, not when the run is
+  // leased. Downloading the archive is our pipe being slow, not the customer's
+  // strategy being slow, and charging their execution budget for our network
+  // is backwards — a 30-day run spent all twenty minutes fetching and was
+  // killed without replaying an event.
+  //
+  // SIZED PER RUN, not a constant — see wallClockMsFor below. A flat twenty
+  // minutes was a limit on the SMALLEST run that could not finish: replay costs
+  // ~24s per market-day (measured, warm cache, 289 markets and ~1.04M events in
+  // a polymarket BTC day), so twenty minutes covers about fifty of them while
+  // the page was selling a ninety-day chip. The customer paid, watched it work
+  // for twenty minutes, and got a refund and no report.
+  //
+  // These two are the inputs to that function and the only numbers to tune.
+  replayBaseMs: 3 * 60 * 1000,
+  replayPerMarketDayMs: 35 * 1000,
+  // The FETCH budget, separate and bounded. Not unbounded, because there is one
+  // worker and one slot: a stalled R2 read used to sit inside the fetch while
+  // the heartbeat kept renewing the lease, so nobody could reclaim the run and
+  // the customer's credits stayed held on a wedged machine. That incident is
+  // why the clock covers the fetch at all; this keeps the bound and stops it
+  // being taken out of the strategy's time.
+  //
+  // 60, MEASURED, and sized for the DEFAULT range rather than the longest one.
+  // A polymarket BTC market-day is ~112 MB and R2 to the worker runs 1.9–3.5
+  // MB/s, so cold: 30 days is 27–50 minutes, 60 days is 54–98, 90 days is
+  // 81–147. No budget covers 90 days cold without letting one run hold the
+  // only worker slot for over two hours, so the honest position is that long
+  // ranges depend on the cache being warm — which is what the prewarm is for.
+  // A cold long run fails inside its budget and is refunded in full, rather
+  // than being allowed to monopolise the queue.
+  fetchClockMs: 60 * 60 * 1000,
   maxParams: 64,
   maxSweepCells: 256,
-  archiveRetentionDays: 90,
+  archiveRetentionDays: 7,
 });
 
+/**
+ * The longest range one run may cover, in CALENDAR DAYS.
+ *
+ * The product limit. Checked against the days that actually exist in the
+ * archive — the range AFTER it is intersected — not against what was asked
+ * for: requesting more days than exist has always been fine and is billed for
+ * what was there, and moving the check earlier would hard-fail a page whose
+ * capacity figure is a few minutes stale.
+ *
+ * Enforced in the API and mirrored in the editor, so nobody can build a
+ * submission the queue will refuse.
+ *
+ * Raising it is a hardware decision, not a config one: there is one worker
+ * slot and a run holds it for its whole life.
+ */
+export const MAX_BACKTEST_DAYS = 90;
+
+/**
+ * The clamp on the REPLAY BUDGET's input — not a limit on what may be run.
+ *
+ * The budget below grows with market-days because that is what the machine
+ * spends time on, and market-days are days × assets × intervals: ninety days
+ * of one asset is 90, ninety days of seven assets over two intervals is 1,260.
+ * Without a clamp the second would be handed an eight-hour budget and would
+ * hold the only worker slot for a working day.
+ *
+ * So a run larger than this still RUNS — it simply is not given proportionally
+ * more time, and if it cannot finish it is refunded in full like any other
+ * overrun. That is the honest failure: bounded queue damage, money back.
+ */
+export const BUDGET_CLAMP_MARKET_DAYS = 180;
+
+/**
+ * How long a run's REPLAY may take, given its size.
+ *
+ * Derived rather than declared so the limit and the thing it limits cannot
+ * drift: `MAX_BACKTEST_DAYS` decides how long a range can be, this decides how long
+ * that size is allowed to take, and both come from the two constants in LIMITS.
+ *
+ * Sized on the WARM path (~24s/market-day measured) plus margin, because the
+ * fetch has its own budget — `fetchClockMs` — and a slow archive read is our
+ * pipe being slow, not the strategy. A run whose days are cold spends that
+ * time under the fetch clock and arrives here with the same work to do.
+ */
+export function wallClockMsFor(marketDays) {
+  const n = Number.isFinite(marketDays) && marketDays > 0 ? Math.ceil(marketDays) : 1;
+  return LIMITS.replayBaseMs
+    + LIMITS.replayPerMarketDayMs * Math.min(n, BUDGET_CLAMP_MARKET_DAYS);
+}
+
+/** The ceiling that follows from the numbers above. For copy and for docs. */
+export const MAX_WALL_CLOCK_MS = wallClockMsFor(BUDGET_CLAMP_MARKET_DAYS);
+
 // ---------------------------------------------------------------------------
 // Rejection codes
 // ---------------------------------------------------------------------------
@@ -313,6 +626,12 @@ export function contractDocument() {
     referenceSymbols: [...REFERENCE_SYMBOLS],
     modes: MODES,
     limits: LIMITS,
+    // THE CEILINGS A CLIENT HAS TO KNOW BEFORE IT BUILDS A REQUEST. They are
+    // not in LIMITS because LIMITS describes the sandbox — what one strategy
+    // gets — and these describe what one RUN may ask for. A client that cannot
+    // read them discovers them as a 422 on the paid path.
+    maxBacktestDays: MAX_BACKTEST_DAYS,
+    maxMarketDays: BUDGET_CLAMP_MARKET_DAYS,
     rejectionCodes: REJECTION_CODES,
   };
 }

package/api/lib/backtest-datasets.mjs

@@ -11,6 +11,7 @@
 import { classifyPath } from './data-taxonomy.mjs';
 import {
   CAPTURE_WINDOWS, DERIVED_DATASETS, KNOWN_DATASETS, BacktestRejection,
+  MARKET_INTERVALS, DEFAULT_INTERVALS, MAX_LATENCY_MS,
 } from './backtest-contract.mjs';
 
 /**
@@ -190,6 +191,56 @@ export function archiveDatasetsFor({ datasets, venue, from, to }) {
   return [...out].sort();
 }
 
+/**
+ * The settlement files a day's markets need, beyond what the strategy asked for.
+ *
+ * ALWAYS FED, like `markets`, and for the same reason: which stream a market
+ * settled on is a property of that market, so it cannot be known before the
+ * markets are read — and a run without it has no ticks at all. A strategy
+ * declaring `datasets: ["prices"]` was handed the 1 Hz feed while every market
+ * settled on twap60s, so nothing reached the engine and the day came back
+ * empty.
+ *
+ * SHARED, because the worker and `ot run` must select the same files. They
+ * already share the decoder; a feed list computed twice is the same drift in a
+ * different place, and it shows up as "it works locally but not in the queue"
+ * — or worse, the reverse.
+ *
+ * @param {Iterable} markets  normalised records, from indexMarkets
+ * @param {string[]} paths    every archive path available for the day
+ * @param {object} opts       venue, the assets in scope, and the paths already chosen
+ */
+export function settlementPathsFor(markets, paths, { venue, assets, already = [] }) {
+  const need = new Set();
+  for (const m of markets) if (m?.stream) need.add(m.stream);
+  if (need.size === 0) return [];
+  const have = new Set(already);
+  const inScope = new Set((assets ?? []).map((a) => String(a).toUpperCase()));
+  const out = [];
+  for (const p of paths) {
+    if (have.has(p)) continue;
+    const c = classifyPath(p);
+    if (c.venue !== venue || !need.has(c.dataset)) continue;
+    if (c.asset && inScope.size && !inScope.has(String(c.asset).toUpperCase())) continue;
+    out.push(p);
+  }
+  return out.sort();
+}
+
+/**
+ * The order a day's archive files are read in, fixed by path.
+ *
+ * NOT cosmetic. Events stamped the same millisecond are ordered by kind, and
+ * ties beyond that fall back to the order they were read — which was the
+ * catalog's order for the queue and the directory walk's for `ot run`. Two
+ * readers, two orders, one archive, two different reports. Sorting by path
+ * makes the input order a property of the archive rather than of whoever is
+ * reading it.
+ */
+export function orderedFeed(paths) {
+  return [...paths].sort();
+}
+
 /**
  * Does an archived file belong to this run's scope?
  *
@@ -198,15 +249,72 @@ export function archiveDatasetsFor({ datasets, venue, from, to }) {
  * A backtest reads files the submitter has not bought, which is the product —
  * so this filter must never be mistaken for an authorisation check.
  */
-export function fileMatchesRun(filePath, { venue, assets, archiveDatasets }) {
+export function fileMatchesRun(filePath, { venue, assets, archiveDatasets, intervals }) {
   const meta = classifyPath(filePath);
   if (meta.venue !== venue) return false;
   if (!archiveDatasets.includes(meta.dataset)) return false;
   // Venue-wide datasets (markets) carry no asset and are always in scope.
   if (meta.asset && assets?.length && !assets.includes(meta.asset)) return false;
+  // The interval narrowing applies ONLY to files that have an interval.
+  //
+  // The settlement streams -- prices, twap30s, twap60s -- are interval:null,
+  // and they are not a choice: without one, every market in the day fails to
+  // settle, every market is unusable, and the whole run is dropped and
+  // refunded. Silently, because "no usable market" is indistinguishable from
+  // "the archive had nothing". So a null interval is always in scope, and only
+  // a file that actually declares one has to match.
+  if (meta.interval && intervals?.length && !intervals.includes(meta.interval)) return false;
   return true;
 }
 
+/**
+ * Validate the declared latency comparison, returning it normalised.
+ *
+ * ONE delay in milliseconds, or null for none — which is the default. Each
+ * delay replays the whole range again, so this is deliberately not a list:
+ * see MAX_LATENCY_MS.
+ */
+export function normalizeLatency(value) {
+  if (value == null) return null;
+  if (Array.isArray(value)) {
+    // Said plainly rather than by silently taking the first: a manifest that
+    // asked for three delays and got one would be a run priced and timed for
+    // something the submitter did not write.
+    throw new BacktestRejection('E_MANIFEST',
+      'latency is a single delay in milliseconds, not a list — each delay replays the whole range again');
+  }
+  const ms = Number(value);
+  if (!Number.isInteger(ms) || ms <= 0 || ms > MAX_LATENCY_MS) {
+    throw new BacktestRejection('E_MANIFEST',
+      `latency must be a whole number of milliseconds between 1 and ${MAX_LATENCY_MS}, got ${JSON.stringify(value)}`);
+  }
+  return ms;
+}
+
+/**
+ * Validate a declared interval list, returning it normalised.
+ *
+ * Absent means the default (5m), not "everything": see MARKET_INTERVALS.
+ */
+export function normalizeIntervals(list) {
+  if (list == null) return [...DEFAULT_INTERVALS];
+  if (!Array.isArray(list) || list.length === 0) {
+    throw new BacktestRejection('E_MANIFEST',
+      `intervals must be a non-empty array; known: ${MARKET_INTERVALS.join(', ')}`);
+  }
+  const out = [];
+  for (const raw of list) {
+    const iv = String(raw ?? '').trim();
+    if (!MARKET_INTERVALS.includes(iv)) {
+      throw new BacktestRejection('E_MANIFEST',
+        `unknown interval ${JSON.stringify(iv)}; known: ${MARKET_INTERVALS.join(', ')}`);
+    }
+    if (!out.includes(iv)) out.push(iv);
+  }
+  // Sorted so the same request always produces the same decoded-cache identity.
+  return out.sort();
+}
+
 /** Validate a declared dataset list, returning it normalised. */
 export function normalizeDatasets(list) {
   if (!Array.isArray(list) || list.length === 0) {
@@ -223,3 +331,19 @@ export function normalizeDatasets(list) {
   }
   return out;
 }
+
+/**
+ * The dataset list the prewarm decodes for.
+ *
+ * A decoded day is cached per dataset SHAPE, so warming a shape nobody submits
+ * warms nothing at all — the paid run still pays the full decode. This is the
+ * shape the editor submits: every dataset it offers as a chip, which is what
+ * almost every run declares.
+ *
+ * Derived from the contract, not written out, so a dataset added to the
+ * product is warmed without anyone remembering to come here.
+ */
+const SETTLEMENT_STREAMS = new Set(['prices', 'twap30s', 'twap60s']);
+export const PREWARM_DATASETS = Object.freeze(
+  KNOWN_DATASETS.filter((d) => !SETTLEMENT_STREAMS.has(d)),
+);

package/api/lib/backtest-manifest.mjs

@@ -13,7 +13,13 @@ import {
   LANGUAGES, KNOWN_LANGUAGES, HOOKS, KNOWN_HOOKS, HOOK_NAMES, LIMITS, MODES,
   KNOWN_MODES, SCHEMA_VERSION, BacktestRejection, parseReferenceFeed,
 } from './backtest-contract.mjs';
-import { normalizeDatasets, assertCoverage } from './backtest-datasets.mjs';
+import {
+  normalizeDatasets, normalizeIntervals, normalizeLatency, assertCoverage,
+} from './backtest-datasets.mjs';
+// The series parser, so the FREE check rejects a malformed CSV instead of the
+// worker rejecting it after credits are held. One implementation, shared: the
+// docs promise a local pass is not rejected on submit.
+import { parseSeries } from '../../runner/series-data.mjs';
 
 /** The one file name that is not the submitter's to choose. */
 export const MANIFEST_NAME = 'outcometick.json';
@@ -116,6 +122,15 @@ export function validateManifest(doc) {
 
   const datasets = normalizeDatasets(doc.datasets);
 
+  // Which market lengths to replay. Absent means 5m — see MARKET_INTERVALS for
+  // why the set is closed and why the default is not "both".
+  const intervals = normalizeIntervals(doc.intervals);
+
+  // Which fill delays to re-price at, for the latency panel. Empty by default:
+  // every delay is another replay of the range, and it used to be five of them
+  // whether or not anyone wanted the table.
+  const latency = normalizeLatency(doc.latency);
+
   // A hook that needs a dataset it was not given would simply never fire, and
   // a strategy that silently never trades looks like a bad strategy rather
   // than a bad manifest. Say so instead.
@@ -231,6 +246,8 @@ export function validateManifest(doc) {
     entry: { file: entryFile, className: entryClass },
     hooks,
     datasets,
+    intervals,
+    latency,
     mode,
     deps,
     reference,
@@ -272,13 +289,45 @@ export function validateFiles(files, manifest) {
         `file ${name} contains a NUL byte — submissions are text only, no archives`);
     }
     const bytes = Buffer.byteLength(content, 'utf8');
-    total += bytes;
     seen.set(name, { name, bytes, content });
   }
 
+  // SOURCE and DATA have separate budgets.
+  //
+  // One shared 256KB cap meant a declared series ate the code's allowance, and
+  // the advertised 32MB series could never be sent at all — the two limits
+  // contradicted each other and the smaller one won silently. Code is small and
+  // is read by a human during an abuse review; a series is bulk and is read by
+  // nothing.
+  const seriesFiles = new Set(manifest.series.map((s) => s.file));
+  for (const f of seen.values()) {
+    if (seriesFiles.has(f.name)) continue;
+    total += f.bytes;
+  }
   if (total > LIMITS.maxTotalSourceBytes) {
     throw new BacktestRejection('E_LIMIT',
-      `total source is ${total} bytes, over the ${LIMITS.maxTotalSourceBytes} byte limit`);
+      `total source is ${total} bytes, over the ${LIMITS.maxTotalSourceBytes} byte limit`
+      + ' (series files have their own budget and are not counted here)');
+  }
+  for (const s of manifest.series) {
+    const f = seen.get(s.file);
+    if (f && f.bytes > LIMITS.maxSeriesBytes) {
+      throw new BacktestRejection('E_LIMIT',
+        `series ${s.name} is ${f.bytes} bytes, over the ${LIMITS.maxSeriesBytes} byte limit`);
+    }
+    // PARSED HERE, in the free check — not later, in the worker.
+    //
+    // The docs promise that a local pass is not rejected on submit, and this is
+    // the same validator `ot check` runs. Leaving it to the worker meant a CSV
+    // with no readable timestamp got as far as holding credits and queueing,
+    // then failed and refunded: technically correct, and a terrible way to
+    // learn your header was wrong.
+    if (f) {
+      const out = parseSeries(f.content);
+      if (out.rows.length === 0) {
+        throw new BacktestRejection('E_MANIFEST', `series ${s.name} (${s.file}): ${out.problem}`);
+      }
+    }
   }
   if (!seen.has(MANIFEST_NAME)) {
     throw new BacktestRejection('E_MANIFEST', `${MANIFEST_NAME} is required`);
@@ -340,6 +389,5 @@ export function checkSubmission({ manifestText, manifest: manifestDoc, files, sc
     // so neither the runner nor the CLI re-implements the parity table.
     hookNames: Object.fromEntries(manifest.hooks.map((h) => [h, HOOK_NAMES[manifest.languageId][h]])),
     shardable: MODES[manifest.mode].shardable,
-    rateMultiplier: MODES[manifest.mode].rateMultiplier,
   };
 }