outcometick 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +88 -0
  3. package/api/lib/backtest-contract.mjs +318 -0
  4. package/api/lib/backtest-datasets.mjs +225 -0
  5. package/api/lib/backtest-manifest.mjs +345 -0
  6. package/api/lib/coverage-window.mjs +42 -0
  7. package/api/lib/data-taxonomy.mjs +175 -0
  8. package/api/lib/venue-path.mjs +16 -0
  9. package/bin/ot.mjs +4 -0
  10. package/cli/api-client.mjs +71 -0
  11. package/cli/commands/fetch.mjs +43 -0
  12. package/cli/commands/run.mjs +269 -0
  13. package/cli/commands/status.mjs +102 -0
  14. package/cli/commands/submit.mjs +77 -0
  15. package/cli/local-data.mjs +177 -0
  16. package/cli/ot.mjs +223 -0
  17. package/index.d.ts +195 -0
  18. package/index.mjs +2 -0
  19. package/package.json +58 -0
  20. package/runner/analyze/index.mjs +40 -0
  21. package/runner/analyze/javascript.mjs +380 -0
  22. package/runner/analyze/python.mjs +85 -0
  23. package/runner/analyze/python_analyze.py +320 -0
  24. package/runner/archive.mjs +185 -0
  25. package/runner/engine/book.mjs +226 -0
  26. package/runner/engine/portfolio.mjs +292 -0
  27. package/runner/engine/replay.mjs +496 -0
  28. package/runner/engine/report.mjs +417 -0
  29. package/runner/events.mjs +190 -0
  30. package/runner/harness/node/harness.mjs +467 -0
  31. package/runner/harness/node/sdk/index.d.ts +195 -0
  32. package/runner/harness/node/sdk/index.mjs +71 -0
  33. package/runner/harness/node/sdk/package.json +8 -0
  34. package/runner/harness/protocol.mjs +255 -0
  35. package/runner/harness/python/harness.py +374 -0
  36. package/runner/harness/python/otengine.py +523 -0
  37. package/runner/harness/python/otreplay.py +409 -0
  38. package/runner/harness/python/outcometick.py +67 -0
@@ -0,0 +1,417 @@
1
+ // Turning a run's trades and fills into the report.
2
+ //
3
+ // Every panel here exists to make the data's value visible, not the strategy's.
4
+ // Calibration, latency and slippage are the three that most often kill a
5
+ // promising curve, and they are on by default for exactly that reason — a
6
+ // backtest that only showed an equity line would be flattering by omission.
7
+ //
8
+ // Nothing in this module can see the strategy. It reads the trade and fill logs
9
+ // the engine produced, so a report cannot be tuned by the thing it is judging.
10
+
11
+ /** Delays the latency panel re-prices every fill at. */
12
+ export const LATENCY_STEPS = Object.freeze([
13
+ { label: 'as captured (0 ms)', ms: 0 },
14
+ { label: '+100 ms', ms: 100 },
15
+ { label: '+250 ms', ms: 250 },
16
+ { label: '+500 ms', ms: 500 },
17
+ { label: '+1 s', ms: 1000 },
18
+ { label: '+2 s', ms: 2000 },
19
+ ]);
20
+
21
+ /** Entry-price buckets for the calibration panel. */
22
+ export const CALIBRATION_BUCKETS = Object.freeze([
23
+ [0.0, 0.1], [0.1, 0.2], [0.2, 0.3], [0.3, 0.4], [0.4, 0.5],
24
+ [0.5, 0.6], [0.6, 0.7], [0.7, 0.8], [0.8, 0.9], [0.9, 1.0],
25
+ ]);
26
+
27
+ const sum = (xs) => xs.reduce((a, b) => a + b, 0);
28
+ const mean = (xs) => (xs.length ? sum(xs) / xs.length : 0);
29
+
30
+ function stdev(xs) {
31
+ if (xs.length < 2) return 0;
32
+ const m = mean(xs);
33
+ return Math.sqrt(sum(xs.map((x) => (x - m) ** 2)) / (xs.length - 1));
34
+ }
35
+
36
+ /** Round for reporting only — never for arithmetic that feeds another number. */
37
+ const r2 = (x) => (Number.isFinite(x) ? Number(x.toFixed(2)) : null);
38
+ const r4 = (x) => (Number.isFinite(x) ? Number(x.toFixed(4)) : null);
39
+
40
+ /**
41
+ * Collateral a trade tied up: contracts times the price paid.
42
+ *
43
+ * The denominator for return-on-collateral. Using notional-at-settlement
44
+ * instead would flatter cheap entries, which is the opposite of what this
45
+ * report is for.
46
+ */
47
+ const collateralOf = (t) => (t.entry_px ?? 0) * (t.size ?? 0);
48
+
49
+ /**
50
+ * Headline metrics — the twelve cells at the top of the report.
51
+ */
52
+ export function metrics(trades, { feesPaid = 0, days = 1 } = {}) {
53
+ const closed = trades.filter((t) => Number.isFinite(t.pnl));
54
+ // In CLOSE order. "Longest losing streak" is a statement about a sequence in
55
+ // time, and `trades` arrives in whatever order the worker flushed markets —
56
+ // roughly chronological, but not guaranteed, and a run that sharded would
57
+ // report a streak that never happened.
58
+ const pnls = [...closed]
59
+ .sort((a, b) => (a.closed_ms ?? 0) - (b.closed_ms ?? 0))
60
+ .map((t) => t.pnl);
61
+ const netPnl = sum(pnls);
62
+ const wins = pnls.filter((p) => p > 0);
63
+ const losses = pnls.filter((p) => p < 0);
64
+ const collateral = sum(closed.map(collateralOf));
65
+
66
+ const equity = equityCurve(closed);
67
+ const dd = maxDrawdown(equity.map((p) => p.equity));
68
+
69
+ // Sharpe over per-day PnL, annualised at 365 — these markets settle every
70
+ // day of the week, so a 252-day year would overstate it.
71
+ const byDay = new Map();
72
+ for (const t of closed) {
73
+ const day = t.closed_ms ? new Date(t.closed_ms).toISOString().slice(0, 10) : 'unknown';
74
+ byDay.set(day, (byDay.get(day) ?? 0) + t.pnl);
75
+ }
76
+ const daily = [...byDay.values()];
77
+ const sd = stdev(daily);
78
+ const sharpe = sd === 0 ? null : (mean(daily) / sd) * Math.sqrt(365);
79
+
80
+ const holds = closed
81
+ .filter((t) => t.opened_ms != null && t.closed_ms != null)
82
+ .map((t) => t.closed_ms - t.opened_ms);
83
+
84
+ return {
85
+ net_pnl: r2(netPnl),
86
+ win_rate: closed.length ? r4(wins.length / closed.length) : null,
87
+ // Gross profit over gross loss. Undefined rather than Infinity when there
88
+ // were no losses — a number that cannot be compared is worse than a blank.
89
+ profit_factor: losses.length ? r2(sum(wins) / Math.abs(sum(losses))) : null,
90
+ // Absolute dollars is the headline figure, because a PERCENTAGE needs a
91
+ // capital base we were never told. Measuring against the running peak — the
92
+ // textbook definition — reports -171% for a curve that went +50 then -35,
93
+ // which is arithmetically true and useless. The percentage is still
94
+ // published, defined against the running peak, for anyone who wants it.
95
+ max_drawdown: dd.pct == null ? null : r4(dd.pct),
96
+ max_drawdown_abs: r2(dd.abs),
97
+ sharpe: sharpe == null ? null : r2(sharpe),
98
+ trades: closed.length,
99
+ return_on_collateral: collateral > 0 ? r4(netPnl / collateral) : null,
100
+ // Cents of edge per contract: what the outcome was worth minus what was
101
+ // paid, averaged. This is the number that says whether there was an edge
102
+ // at all, as opposed to a lucky run of variance.
103
+ edge_per_contract: closed.length ? r4(edgePerContract(closed)) : null,
104
+ brier_score: r4(brier(closed)),
105
+ fees: r2(-Math.abs(feesPaid)),
106
+ avg_hold_ms: holds.length ? Math.round(mean(holds)) : null,
107
+ worst_losing_run: worstLosingRun(pnls),
108
+ collateral_deployed: r2(collateral),
109
+ market_days: days,
110
+ };
111
+ }
112
+
113
+ /**
114
+ * Mean realised edge per contract, in dollars.
115
+ *
116
+ * A binary token bought at p is worth 1 if its side settles and 0 otherwise, so
117
+ * the edge on one contract is (outcome - p). Only settled trades carry an
118
+ * outcome; a trade closed early is edge against the market, not against the
119
+ * truth, and is excluded rather than scored as if it had settled.
120
+ */
121
+ export function edgePerContract(trades) {
122
+ const settled = trades.filter((t) => t.how === 'settled' && t.entry_px != null && t.outcome);
123
+ if (!settled.length) return 0;
124
+ let contracts = 0;
125
+ let edge = 0;
126
+ for (const t of settled) {
127
+ const won = t.outcome === t.side ? 1 : 0;
128
+ edge += (won - t.entry_px) * t.size;
129
+ contracts += t.size;
130
+ }
131
+ return contracts > 0 ? edge / contracts : 0;
132
+ }
133
+
134
+ /**
135
+ * Brier score over settled trades: mean squared error of the price as a
136
+ * forecast. Lower is better; 0.25 is what you get by always saying 50%.
137
+ */
138
+ export function brier(trades) {
139
+ const settled = trades.filter((t) => t.how === 'settled' && t.entry_px != null && t.outcome);
140
+ if (!settled.length) return null;
141
+ return mean(settled.map((t) => {
142
+ const won = t.outcome === t.side ? 1 : 0;
143
+ return (t.entry_px - won) ** 2;
144
+ }));
145
+ }
146
+
147
+ /** Cumulative realised PnL, one point per closed trade. */
148
+ export function equityCurve(trades) {
149
+ const ordered = [...trades].sort((a, b) => (a.closed_ms ?? 0) - (b.closed_ms ?? 0));
150
+ let acc = 0;
151
+ return ordered.map((t) => {
152
+ acc += t.pnl;
153
+ return { ts_ms: t.closed_ms, equity: acc };
154
+ });
155
+ }
156
+
157
+ /**
158
+ * Peak-to-trough decline.
159
+ *
160
+ * Expressed against the running peak, so a drawdown early in a run is not
161
+ * diluted by profits that had not happened yet. A curve that never reaches a
162
+ * positive peak reports the absolute decline and a null percentage rather than
163
+ * dividing by something near zero and printing -4000%.
164
+ */
165
+ export function maxDrawdown(series) {
166
+ let peak = 0;
167
+ let worstAbs = 0;
168
+ let worstPct = null;
169
+ for (const v of series) {
170
+ if (v > peak) peak = v;
171
+ const decline = peak - v;
172
+ if (decline > worstAbs) {
173
+ worstAbs = decline;
174
+ worstPct = peak > 0 ? -(decline / peak) : null;
175
+ }
176
+ }
177
+ return { abs: worstAbs, pct: worstPct };
178
+ }
179
+
180
+ /** Longest consecutive run of losing trades. */
181
+ export function worstLosingRun(pnls) {
182
+ let worst = 0;
183
+ let current = 0;
184
+ for (const p of pnls) {
185
+ if (p < 0) { current += 1; if (current > worst) worst = current; } else current = 0;
186
+ }
187
+ return worst;
188
+ }
189
+
190
+ /**
191
+ * Calibration: what you paid, against what actually settled.
192
+ *
193
+ * The panel this feeds is the one that separates edge from variance. Buckets
194
+ * where the realised rate sits above the price paid are where the money came
195
+ * from; everywhere else is a good equity curve wearing variance.
196
+ */
197
+ export function calibration(trades) {
198
+ const settled = trades.filter((t) => t.how === 'settled' && t.entry_px != null && t.outcome);
199
+ return CALIBRATION_BUCKETS.map(([lo, hi]) => {
200
+ const inBucket = settled.filter((t) => t.entry_px >= lo && t.entry_px < hi);
201
+ if (!inBucket.length) return null;
202
+ const implied = mean(inBucket.map((t) => t.entry_px));
203
+ const realized = mean(inBucket.map((t) => (t.outcome === t.side ? 1 : 0)));
204
+ return {
205
+ bucket: `${lo.toFixed(2)} – ${hi.toFixed(2)}`,
206
+ lo,
207
+ hi,
208
+ implied: r4(implied),
209
+ realized: r4(realized),
210
+ // In cents, the unit the panel labels it in.
211
+ edge_cents: r2((realized - implied) * 100),
212
+ trades: inBucket.length,
213
+ };
214
+ }).filter(Boolean);
215
+ }
216
+
217
+ /**
218
+ * Naive baselines over the same markets, at the SAME SIZE the strategy traded.
219
+ *
220
+ * The size matters or the panel is meaningless: a baseline priced at one
221
+ * contract sitting next to a strategy that traded five hundred compares two
222
+ * different quantities and makes the strategy look several hundred times
223
+ * better than it is. The caller passes the strategy's average trade size.
224
+ *
225
+ * Not decoration: if a strategy cannot beat buying the favourite, no parameter
226
+ * sweep is going to save it, and the customer should find that out here rather
227
+ * than after a month of subscription.
228
+ */
229
+ export function baselines(marketSummaries, { size = 1 } = {}) {
230
+ const out = { always_up: 0, always_down: 0, always_favourite: 0 };
231
+ for (const m of marketSummaries) {
232
+ if (!m.outcome || m.up_px == null || m.down_px == null) continue;
233
+ out.always_up += ((m.outcome === 'UP' ? 1 : 0) - m.up_px) * size;
234
+ out.always_down += ((m.outcome === 'DOWN' ? 1 : 0) - m.down_px) * size;
235
+ // The favourite is the side the market thinks is MORE likely, and on a
236
+ // binary market the price IS the implied probability — so it is the DEARER
237
+ // side, not the cheaper one. This was inverted: the panel labelled "always
238
+ // buy the favourite" was actually buying the underdog every time, which
239
+ // handed customers a backwards comparison to judge their strategy against.
240
+ const favSide = m.up_px >= m.down_px ? 'UP' : 'DOWN';
241
+ const favPx = Math.max(m.up_px, m.down_px);
242
+ out.always_favourite += ((m.outcome === favSide ? 1 : 0) - favPx) * size;
243
+ }
244
+ return {
245
+ always_up: r2(out.always_up),
246
+ always_down: r2(out.always_down),
247
+ always_favourite: r2(out.always_favourite),
248
+ };
249
+ }
250
+
251
+ /**
252
+ * Fill quality against the depth that was resting.
253
+ *
254
+ * `quoted_px` is the price on the screen when the order was sent and `avg_px`
255
+ * is what it actually cost. The gap is the slippage, and the unfilled remainder
256
+ * is the size the book never had.
257
+ */
258
+ export function slippage(fills) {
259
+ const attempted = fills.filter((f) => f.action === 'open');
260
+ if (!attempted.length) {
261
+ return {
262
+ fills_at_quote: null, partial_fills: null, unfilled: null,
263
+ median_slippage_cents: null, worst_1pct_slippage_cents: null, pnl_lost_to_slippage: null,
264
+ orders: 0,
265
+ };
266
+ }
267
+ const atQuote = attempted.filter((f) => f.filled > 0 && f.levels_walked === 1);
268
+ const walked = attempted.filter((f) => f.filled > 0 && f.levels_walked > 1);
269
+ const nothing = attempted.filter((f) => f.filled === 0);
270
+
271
+ const slips = attempted
272
+ .filter((f) => f.filled > 0 && f.quoted_px != null && f.avg_px != null)
273
+ .map((f) => (f.avg_px - f.quoted_px) * 100);
274
+ const sorted = [...slips].sort((a, b) => a - b);
275
+ const at = (q) => (sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] : null);
276
+
277
+ const cost = sum(attempted
278
+ .filter((f) => f.filled > 0 && f.quoted_px != null && f.avg_px != null)
279
+ .map((f) => (f.avg_px - f.quoted_px) * f.filled));
280
+
281
+ return {
282
+ orders: attempted.length,
283
+ fills_at_quote: r4(atQuote.length / attempted.length),
284
+ partial_fills: r4(walked.length / attempted.length),
285
+ unfilled: r4(nothing.length / attempted.length),
286
+ median_slippage_cents: r2(at(0.5)),
287
+ worst_1pct_slippage_cents: r2(at(0.99)),
288
+ // Negative: this is money the strategy did not keep.
289
+ pnl_lost_to_slippage: r2(-cost),
290
+ // Size the book never had, as a fraction of what was asked for.
291
+ unfilled_size_ratio: r4(sum(attempted.map((f) => f.unfilled)) / sum(attempted.map((f) => f.requested))),
292
+ };
293
+ }
294
+
295
+ /** PnL split by asset and market period, for the by-market panel. */
296
+ export function splitByMarket(trades, marketMeta = new Map()) {
297
+ const groups = new Map();
298
+ for (const t of trades) {
299
+ const meta = marketMeta.get(t.market_id) ?? {};
300
+ const key = `${meta.asset ?? 'unknown'} ${meta.interval ?? ''}`.trim();
301
+ const g = groups.get(key) ?? { name: key, pnl: 0, trades: 0 };
302
+ g.pnl += t.pnl;
303
+ g.trades += 1;
304
+ groups.set(key, g);
305
+ }
306
+ const rows = [...groups.values()].map((g) => ({ ...g, pnl: r2(g.pnl) }));
307
+ rows.sort((a, b) => Math.abs(b.pnl) - Math.abs(a.pnl));
308
+ return rows;
309
+ }
310
+
311
+ /**
312
+ * The latency panel: net PnL if every fill had landed later.
313
+ *
314
+ * The rows come from re-running the replay at each delay, which the caller
315
+ * does — this only shapes the result. We keep event time, upstream server time
316
+ * and our receive time separate on every row, which is what makes re-pricing at
317
+ * an arbitrary delay meaningful rather than a guess.
318
+ */
319
+ export function latencyPanel(resultsByDelay) {
320
+ const base = resultsByDelay.find((r) => r.delayMs === 0)?.netPnl ?? 0;
321
+ return resultsByDelay.map((r) => ({
322
+ label: LATENCY_STEPS.find((s) => s.ms === r.delayMs)?.label ?? `+${r.delayMs} ms`,
323
+ delay_ms: r.delayMs,
324
+ net_pnl: r2(r.netPnl),
325
+ // Relative to the as-captured run, so the shape of the decay is readable
326
+ // without dividing in your head.
327
+ ratio: base === 0 ? null : r4(r.netPnl / base),
328
+ unprofitable: r.netPnl < 0,
329
+ }));
330
+ }
331
+
332
+ /**
333
+ * The parameter sweep grid.
334
+ *
335
+ * Billed once per market-day, not per cell: the archive is decoded once and
336
+ * every cell is evaluated against the same decoded stream. Charging per cell
337
+ * would be charging for our CPU rather than for data scanned.
338
+ */
339
+ export function sweepPanel(cells, { xParam, yParam, metric = 'sharpe' }) {
340
+ const xs = [...new Set(cells.map((c) => c.params[xParam]))].sort((a, b) => a - b);
341
+ const ys = [...new Set(cells.map((c) => c.params[yParam]))].sort((a, b) => a - b);
342
+ const grid = ys.map((y) => xs.map((x) => {
343
+ const cell = cells.find((c) => c.params[xParam] === x && c.params[yParam] === y);
344
+ return cell ? r2(cell.metrics[metric]) : null;
345
+ }));
346
+ const flat = grid.flat().filter((v) => v != null);
347
+ return {
348
+ metric,
349
+ x_param: xParam,
350
+ y_param: yParam,
351
+ x_labels: xs,
352
+ y_labels: ys,
353
+ grid,
354
+ max: flat.length ? Math.max(...flat) : null,
355
+ min: flat.length ? Math.min(...flat) : null,
356
+ cells: cells.length,
357
+ };
358
+ }
359
+
360
+ /**
361
+ * Assemble the whole report.
362
+ *
363
+ * `coverage` is carried through untouched from the archive: which stream backed
364
+ * each market-day, and where the gaps were. It is the part of the report that
365
+ * makes the rest of it checkable, so it is never summarised away.
366
+ */
367
+ export function buildReport({
368
+ runId, submittedAt, manifest, scope,
369
+ trades, fills, marketSummaries, marketMeta,
370
+ feesPaid = 0, latency = [], sweep = null, coverage = null,
371
+ crosschecks = [], budget = null, seed = null, scanned = {},
372
+ }) {
373
+ const closed = trades.filter((t) => Number.isFinite(t.pnl));
374
+ const equity = equityCurve(closed);
375
+ const matched = crosschecks.filter((c) => c.match).length;
376
+
377
+ return {
378
+ run_id: runId,
379
+ generated_ms: submittedAt,
380
+ sdk_schema: manifest?.schema ?? null,
381
+ language: manifest?.language ?? null,
382
+ mode: manifest?.mode ?? 'market',
383
+ seed,
384
+ scope: {
385
+ venue: scope?.venue ?? null,
386
+ assets: scope?.assets ?? [],
387
+ from: scope?.from ?? null,
388
+ to: scope?.to ?? null,
389
+ market_days: scope?.marketDays ?? null,
390
+ },
391
+ scanned,
392
+ metrics: metrics(closed, { feesPaid, days: scope?.archivedDayCount ?? 1 }),
393
+ equity: equity.map((p) => ({ ts_ms: p.ts_ms, equity: r2(p.equity) })),
394
+ crosscheck: {
395
+ markets_touched: new Set(closed.map((t) => t.market_id)).size,
396
+ recompute_checks: crosschecks.length,
397
+ recompute_matches: matched,
398
+ // Reported even when zero checks were made — a blank panel would read as
399
+ // "everything reconciled".
400
+ mismatches: crosschecks.length - matched,
401
+ },
402
+ trades: closed,
403
+ fills,
404
+ calibration: calibration(closed),
405
+ // Same markets, same sizing — the average size the strategy actually
406
+ // traded, so the comparison is like for like.
407
+ baselines: baselines(marketSummaries ?? [], {
408
+ size: closed.length ? closed.reduce((a, t) => a + (t.size ?? 0), 0) / closed.length : 1,
409
+ }),
410
+ split: splitByMarket(closed, marketMeta ?? new Map()),
411
+ slippage: slippage(fills ?? []),
412
+ latency: latencyPanel(latency),
413
+ sweep,
414
+ coverage,
415
+ budget,
416
+ };
417
+ }
@@ -0,0 +1,190 @@
1
+ // Turning archived rows into replayable events.
2
+ //
3
+ // Extracted so the two things that read the archive share ONE implementation:
4
+ // the worker (which streams it out of R2) and the `ot` CLI (which reads a
5
+ // cloned sample repo off disk). They differ only in where the bytes come from.
6
+ //
7
+ // The lesson this is applying is the one runner/conformance already enforces
8
+ // for the two engines: two implementations of the same rules drift, and the
9
+ // drift is silent. "The identical files, same checksums, same coverage report"
10
+ // is a published promise about `ot run` versus a queued run — it cannot be true
11
+ // if local and remote decode the archive differently.
12
+
13
+ import { classifyPath } from '../api/lib/data-taxonomy.mjs';
14
+ import { resolveSettlementStream } from '../api/lib/backtest-datasets.mjs';
15
+
16
+ /**
17
+ * Coerce a field to a number, or null.
18
+ *
19
+ * The empty check is load-bearing: `Number('')` is 0, not NaN, and an empty CSV
20
+ * cell is the normal case. Without it a missing `close_ts_ms` becomes 0 — a
21
+ * market that closed at the Unix epoch — and settlement and hold expiry both
22
+ * behave nonsensically off it.
23
+ */
24
+ export const num = (v) => {
25
+ if (v == null) return null;
26
+ if (typeof v === 'string' && v.trim() === '') return null;
27
+ const n = Number(v);
28
+ return Number.isFinite(n) ? n : null;
29
+ };
30
+
31
+ /** `[[px, size], ...]` from either a JSON array or a "px:size|px:size" string. */
32
+ export function parseLevels(v) {
33
+ if (Array.isArray(v)) return v;
34
+ if (typeof v !== 'string' || !v) return [];
35
+ if (v.startsWith('[')) {
36
+ try { return JSON.parse(v); } catch { return []; }
37
+ }
38
+ return v.split('|').filter(Boolean).map((pair) => {
39
+ const [px, size] = pair.split(':');
40
+ return [Number(px), Number(size)];
41
+ }).filter(([px, size]) => Number.isFinite(px) && Number.isFinite(size));
42
+ }
43
+
44
+ /**
45
+ * Which stream backed each market, from the markets metadata.
46
+ *
47
+ * Fail-closed: a market whose config we cannot read gets `null` and is dropped,
48
+ * because feeding a strategy a stream the market did not settle on is the one
49
+ * error this product cannot make. An ABSENT config is a read failure, not
50
+ * evidence of the pre-TWAP regime — see resolveSettlementStream.
51
+ */
52
+ export function indexMarkets(rows) {
53
+ const byId = new Map();
54
+ for (const row of rows) {
55
+ const id = row.market_id ?? row.id ?? row.condition_id;
56
+ if (!id) continue;
57
+ byId.set(String(id), {
58
+ market_id: String(id),
59
+ asset: row.asset ?? null,
60
+ interval: row.interval ?? null,
61
+ strike: num(row.strike),
62
+ outcome: row.outcome === 'UP' || row.outcome === 'DOWN' ? row.outcome : null,
63
+ open_ts_ms: num(row.open_ts_ms ?? row.start_ts_ms),
64
+ close_ts_ms: num(row.close_ts_ms ?? row.end_ts_ms),
65
+ stream: resolveSettlementStream(row),
66
+ raw: row,
67
+ });
68
+ }
69
+ return byId;
70
+ }
71
+
72
+ /**
73
+ * The events one archived row produces, as [marketId, event] pairs.
74
+ *
75
+ * A settlement-feed row belongs to every market that settles on THAT stream —
76
+ * which is decided per market from its own config, never from the date. One row
77
+ * therefore fans out to many markets, and a row for a stream no market in scope
78
+ * settles on produces nothing.
79
+ *
80
+ * @param {string} filePath the archive path the row came from
81
+ * @param {object} row
82
+ * @param {Map} markets from indexMarkets
83
+ */
84
+ export function eventsFromRow(filePath, row, markets) {
85
+ const meta = classifyPath(filePath);
86
+ const ts = num(row.ts_ms ?? row.timestamp_ms ?? row.event_ts_ms);
87
+ if (ts == null) return [];
88
+
89
+ if (meta.dataset === 'prices' || meta.dataset === 'twap30s' || meta.dataset === 'twap60s') {
90
+ const out = [];
91
+ for (const [id, m] of markets) {
92
+ if (m.stream !== meta.dataset) continue;
93
+ if (m.asset && meta.asset && !String(m.asset).startsWith(meta.asset)) continue;
94
+ out.push([id, {
95
+ kind: 'tick',
96
+ ts_ms: ts,
97
+ market_id: id,
98
+ value: num(row.value ?? row.price ?? row.answer),
99
+ source: meta.dataset,
100
+ // Three timestamps kept apart on every row: it is what lets a fill be
101
+ // re-priced at an arbitrary delay instead of assumed instant.
102
+ server_ts_ms: num(row.server_ts_ms),
103
+ recv_ts_ms: num(row.recv_ts_ms),
104
+ }]);
105
+ }
106
+ return out;
107
+ }
108
+
109
+ const marketId = row.market_id ?? row.asset_id ?? null;
110
+ if (!marketId) return [];
111
+
112
+ if (meta.dataset === 'book' || meta.dataset === 'orderbook') {
113
+ return [[String(marketId), {
114
+ kind: 'book',
115
+ ts_ms: ts,
116
+ snapshot: true,
117
+ levels: {
118
+ UP: { asks: parseLevels(row.up_asks), bids: parseLevels(row.up_bids) },
119
+ DOWN: { asks: parseLevels(row.down_asks), bids: parseLevels(row.down_bids) },
120
+ },
121
+ }]];
122
+ }
123
+ if (meta.dataset === 'price_change') {
124
+ return [[String(marketId), {
125
+ kind: 'book',
126
+ ts_ms: ts,
127
+ snapshot: false,
128
+ side: row.side === 'DOWN' ? 'DOWN' : 'UP',
129
+ ladder: row.ladder === 'bids' ? 'bids' : 'asks',
130
+ px: num(row.price),
131
+ size: num(row.size),
132
+ }]];
133
+ }
134
+ if (meta.dataset === 'last_trade_price') {
135
+ return [[String(marketId), {
136
+ kind: 'trade',
137
+ ts_ms: ts,
138
+ market_id: String(marketId),
139
+ px: num(row.price),
140
+ size: num(row.size),
141
+ side: row.side === 'DOWN' ? 'DOWN' : 'UP',
142
+ }]];
143
+ }
144
+ return [];
145
+ }
146
+
147
+ /**
148
+ * Order one market's events and cut them at its close.
149
+ *
150
+ * The truncation is not tidying. The settlement feed is a per-DAY stream and
151
+ * every row of it lands on every market settling on that stream, so a market
152
+ * closing at 10:00 collects 14:00's rows too. Cutting here — upstream of both
153
+ * the engine and the closing-quote scan below — is what keeps a post-close book
154
+ * out of the strategy's view AND out of the report's baselines.
155
+ */
156
+ export function finaliseMarket(events, market) {
157
+ // Stable sort by event time. Ties keep insertion order, which puts a book
158
+ // update before the tick that arrived in the same millisecond — a strategy
159
+ // reacting to that tick should see the book as it already was.
160
+ events.sort((a, b) => a.ts_ms - b.ts_ms);
161
+
162
+ const close = market.close_ts_ms;
163
+ const inWindow = close == null ? events : events.filter((e) => e.ts_ms <= close);
164
+
165
+ // The closing quote on each side, from the events that survived the cut. The
166
+ // report's naive baselines are priced off this.
167
+ let upPx = null;
168
+ let downPx = null;
169
+ for (let i = inWindow.length - 1; i >= 0; i -= 1) {
170
+ const ev = inWindow[i];
171
+ if (ev.kind !== 'book' || !ev.snapshot) continue;
172
+ upPx = ev.levels?.UP?.asks?.[0]?.[0] ?? null;
173
+ downPx = ev.levels?.DOWN?.asks?.[0]?.[0] ?? null;
174
+ if (upPx != null || downPx != null) break;
175
+ }
176
+
177
+ return { events: inWindow, up_px: upPx, down_px: downPx };
178
+ }
179
+
180
+ /** Parse one archive line, CSV or JSONL, given a header for CSV. */
181
+ export function parseRow(line, { isCsv, header }) {
182
+ if (!isCsv) {
183
+ try { return JSON.parse(line); } catch { return null; }
184
+ }
185
+ if (!header) return null;
186
+ const cells = line.split(',');
187
+ const row = {};
188
+ for (let i = 0; i < header.length; i += 1) row[header[i]] = cells[i];
189
+ return row;
190
+ }