outcometick 1.5.2 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -1
- package/api/lib/backtest-contract.mjs +265 -15
- package/api/lib/backtest-datasets.mjs +125 -1
- package/api/lib/backtest-manifest.mjs +52 -4
- package/api/lib/coverage-window.mjs +63 -1
- package/api/lib/data-taxonomy.mjs +14 -0
- package/cli/commands/run.mjs +196 -37
- package/cli/commands/submit.mjs +30 -2
- package/cli/local-data.mjs +81 -15
- package/cli/ot.mjs +22 -3
- package/index.d.ts +31 -4
- package/package.json +1 -1
- package/runner/archive.mjs +24 -13
- package/runner/engine/book.mjs +12 -1
- package/runner/engine/feed.mjs +116 -0
- package/runner/engine/portfolio.mjs +77 -5
- package/runner/engine/replay.mjs +136 -14
- package/runner/engine/report.mjs +23 -34
- package/runner/events.mjs +713 -55
- package/runner/harness/node/harness.mjs +138 -12
- package/runner/harness/node/sdk/index.d.ts +31 -4
- package/runner/harness/node/sdk/index.mjs +43 -2
- package/runner/harness/protocol.mjs +31 -3
- package/runner/harness/python/harness.py +123 -18
- package/runner/harness/python/otengine.py +116 -9
- package/runner/harness/python/otfeed.py +109 -0
- package/runner/harness/python/otreplay.py +90 -8
- package/runner/harness/python/outcometick.py +44 -4
- package/runner/series-data.mjs +220 -0
package/index.d.ts
CHANGED
|
@@ -132,10 +132,37 @@ export interface Ctx<P = Record<string, unknown>> {
|
|
|
132
132
|
assert_outcome(market: unknown, outcome: Side): void;
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
-
|
|
135
|
+
/**
|
|
136
|
+
* Size an order in contracts, or in money.
|
|
137
|
+
*
|
|
138
|
+
* A union rather than two optional fields, so `{ size, notional }` together is
|
|
139
|
+
* a compile error rather than a run-time rejection: they answer the same
|
|
140
|
+
* question two ways and there is no sensible reading of both.
|
|
141
|
+
*/
|
|
142
|
+
export type OrderSizing =
|
|
143
|
+
| {
|
|
144
|
+
/** Contracts. Must be positive. */
|
|
145
|
+
size: number;
|
|
146
|
+
notional?: never;
|
|
147
|
+
}
|
|
148
|
+
| {
|
|
149
|
+
size?: never;
|
|
150
|
+
/**
|
|
151
|
+
* Spend at most this much, converted to contracts as
|
|
152
|
+
* `floor(notional / limit)`.
|
|
153
|
+
*
|
|
154
|
+
* REQUIRES `limit`, which is why this arm makes it non-optional: a
|
|
155
|
+
* contract costs whatever it fills at and a marketable order walks the
|
|
156
|
+
* book, so dividing by the current best price overspends the moment there
|
|
157
|
+
* is any slippage. The limit is the price you have already said you will
|
|
158
|
+
* not exceed, which is what makes "at most" true.
|
|
159
|
+
*/
|
|
160
|
+
notional: number;
|
|
161
|
+
limit: number;
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
export type OrderInit = OrderSizing & {
|
|
136
165
|
side: Side;
|
|
137
|
-
/** Contracts. Must be positive. */
|
|
138
|
-
size: number;
|
|
139
166
|
/**
|
|
140
167
|
* A bound in whichever direction protects you: a ceiling when opening, a
|
|
141
168
|
* floor when reducing. Must be within [0, 1] — a binary outcome token
|
|
@@ -149,7 +176,7 @@ export interface OrderInit {
|
|
|
149
176
|
/** Only 'ioc' is modelled; anything else is rejected at construction. */
|
|
150
177
|
tif?: 'ioc';
|
|
151
178
|
tag?: string | null;
|
|
152
|
-
}
|
|
179
|
+
};
|
|
153
180
|
|
|
154
181
|
/**
|
|
155
182
|
* An order a hook returns.
|
package/package.json
CHANGED
package/runner/archive.mjs
CHANGED
|
@@ -123,10 +123,26 @@ function csvField(v) {
|
|
|
123
123
|
}
|
|
124
124
|
|
|
125
125
|
/** Rows to CSV with a fixed column order, so a diff between runs is meaningful. */
|
|
126
|
+
/**
|
|
127
|
+
* A UTF-8 byte-order mark.
|
|
128
|
+
*
|
|
129
|
+
* Excel opens a .csv as the system ANSI code page unless the file says
|
|
130
|
+
* otherwise, and the only thing it accepts as saying otherwise is this. The
|
|
131
|
+
* symptom was a calibration bucket reading `0.30 釴?0.40` — an en dash, three
|
|
132
|
+
* UTF-8 bytes, read as GBK. That is not a Chinese-locale problem: it is every
|
|
133
|
+
* non-ASCII byte in every one of these files, including the `tag` a strategy
|
|
134
|
+
* puts on its own fills, which we do not control at all.
|
|
135
|
+
*
|
|
136
|
+
* Parsers that do not expect it see one stray character on the first header;
|
|
137
|
+
* `encoding='utf-8-sig'` is the standard remedy. A spreadsheet that mangles
|
|
138
|
+
* the whole file is the worse failure, and it is the one that was happening.
|
|
139
|
+
*/
|
|
140
|
+
const BOM = '\uFEFF';
|
|
141
|
+
|
|
126
142
|
export function toCsv(rows, columns) {
|
|
127
143
|
const out = [columns.join(',')];
|
|
128
144
|
for (const row of rows) out.push(columns.map((c) => csvField(row[c])).join(','));
|
|
129
|
-
return `${out.join('\n')}\n`;
|
|
145
|
+
return `${BOM}${out.join('\n')}\n`;
|
|
130
146
|
}
|
|
131
147
|
|
|
132
148
|
const TRADE_COLUMNS = [
|
|
@@ -141,14 +157,17 @@ const FILL_COLUMNS = [
|
|
|
141
157
|
/**
|
|
142
158
|
* Assemble the archive a customer downloads.
|
|
143
159
|
*
|
|
144
|
-
*
|
|
145
|
-
* to the exact code that produced it
|
|
146
|
-
*
|
|
160
|
+
* THE SOURCE IS NOT IN IT. It used to be, so that a report could be tied back
|
|
161
|
+
* to the exact code that produced it — "which version of my strategy was
|
|
162
|
+
* this?" is the first question anyone asks a week later. That question is now
|
|
163
|
+
* answered by `source_sha256` in report.json instead: the same identification,
|
|
164
|
+
* without handing back a copy of the code. Shipping the strategy inside the
|
|
165
|
+
* deliverable made a report something you cannot forward to anyone.
|
|
147
166
|
*
|
|
148
167
|
* sha256sums.txt covers every other entry, so the whole thing is verifiable
|
|
149
168
|
* without trusting the transport.
|
|
150
169
|
*/
|
|
151
|
-
export async function buildArchive({ runId, report, trades, fills, logs
|
|
170
|
+
export async function buildArchive({ runId, report, trades, fills, logs }) {
|
|
152
171
|
const entries = [
|
|
153
172
|
{ name: 'report.json', data: `${JSON.stringify(report, null, 2)}\n` },
|
|
154
173
|
{ name: 'trades.csv', data: toCsv(trades, TRADE_COLUMNS) },
|
|
@@ -158,18 +177,10 @@ export async function buildArchive({ runId, report, trades, fills, logs, source
|
|
|
158
177
|
name: 'calibration.csv',
|
|
159
178
|
data: toCsv(report.calibration ?? [], ['bucket', 'implied', 'realized', 'edge_cents', 'trades']),
|
|
160
179
|
},
|
|
161
|
-
{
|
|
162
|
-
name: 'latency.csv',
|
|
163
|
-
data: toCsv(report.latency ?? [], ['label', 'delay_ms', 'net_pnl', 'ratio', 'unprofitable']),
|
|
164
|
-
},
|
|
165
180
|
{ name: 'coverage.json', data: `${JSON.stringify(report.coverage ?? {}, null, 2)}\n` },
|
|
166
181
|
{ name: 'logs.txt', data: logs ?? '' },
|
|
167
182
|
];
|
|
168
183
|
|
|
169
|
-
for (const f of source ?? []) {
|
|
170
|
-
entries.push({ name: `strategy/${f.name}`, data: f.content });
|
|
171
|
-
}
|
|
172
|
-
|
|
173
184
|
// Checksums last, over everything above.
|
|
174
185
|
const sums = entries
|
|
175
186
|
.map(({ name, data }) => {
|
package/runner/engine/book.mjs
CHANGED
|
@@ -209,7 +209,18 @@ export function matchOrder(book, order) {
|
|
|
209
209
|
const quotedPx = ladder.best();
|
|
210
210
|
|
|
211
211
|
const { fills, remaining, notional } = ladder.take(size, order.limit ?? null);
|
|
212
|
-
|
|
212
|
+
// SUMMED FROM WHAT WAS TAKEN, not derived as `size - remaining`.
|
|
213
|
+
//
|
|
214
|
+
// `remaining` is the requested size with each level's depth subtracted from
|
|
215
|
+
// it, and at IEEE-754 precision `1e308 - 1000` is still `1e308`. So a huge
|
|
216
|
+
// but finite order consumed the whole ladder while reporting `filled: 0` —
|
|
217
|
+
// no position, no cash, no trade row, and an empty book for every order
|
|
218
|
+
// after it in that market. The report said nothing happened; the book said
|
|
219
|
+
// otherwise.
|
|
220
|
+
//
|
|
221
|
+
// Adding up the levels actually taken cannot drift from what was removed,
|
|
222
|
+
// because it IS what was removed. otengine.py mirrors this.
|
|
223
|
+
const filled = fills.reduce((a, f) => a + f.size, 0);
|
|
213
224
|
|
|
214
225
|
return {
|
|
215
226
|
fills,
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// Point-in-time views over an out-of-band series.
|
|
2
|
+
//
|
|
3
|
+
// `ctx.ref(name)` and `ctx.ext(name)` both call `feed.viewAt(ctx.now)` and hand
|
|
4
|
+
// the result to the strategy. Until now nothing implemented `viewAt`: the
|
|
5
|
+
// manifest validator accepted `reference` and `series`, the run was queued and
|
|
6
|
+
// billed, and the strategy crashed on its first `ctx.ref(...)` with a message
|
|
7
|
+
// claiming the feed had not been declared — which it had.
|
|
8
|
+
//
|
|
9
|
+
// Two properties this file exists to guarantee:
|
|
10
|
+
//
|
|
11
|
+
// 1. NOTHING STAMPED AFTER ctx.now IS REACHABLE. Not filtered on the way out
|
|
12
|
+
// — the cursor never advances past `now`, so a later row is not something
|
|
13
|
+
// the strategy can ask for. That is the same promise the event replay
|
|
14
|
+
// makes, and a reference feed is exactly where it would otherwise leak:
|
|
15
|
+
// the whole point of an outside series is that we hold all of it up front.
|
|
16
|
+
//
|
|
17
|
+
// 2. WHAT THE STRATEGY GETS IS A COPY. `ctx.book()` and `ctx.history()` were
|
|
18
|
+
// both caught handing out live objects a strategy could rewrite, and a
|
|
19
|
+
// rewritten reference row would poison every later window() over it.
|
|
20
|
+
//
|
|
21
|
+
// `lagMs` models publication delay: a row is not visible until ts_ms + lagMs,
|
|
22
|
+
// which is how you backtest a signal you could not have had instantly.
|
|
23
|
+
|
|
24
|
+
/** Frozen, prototype-less copy of one row. */
|
|
25
|
+
function freezeRow(row) {
|
|
26
|
+
const out = Object.create(null);
|
|
27
|
+
for (const k of Object.keys(row)) out[k] = row[k];
|
|
28
|
+
return Object.freeze(out);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class PointInTimeFeed {
|
|
32
|
+
/**
|
|
33
|
+
* @param rows ascending by ts_ms. Not copied — this class owns them and
|
|
34
|
+
* never hands one out directly.
|
|
35
|
+
* @param lagMs publication delay; a row becomes visible at ts_ms + lagMs.
|
|
36
|
+
*/
|
|
37
|
+
constructor(rows, { lagMs = 0 } = {}) {
|
|
38
|
+
this.rows = rows;
|
|
39
|
+
this.lagMs = Number(lagMs) || 0;
|
|
40
|
+
// Monotone cursor: the replay only ever moves forward, so the whole feed is
|
|
41
|
+
// walked once across a market-day rather than binary-searched per call.
|
|
42
|
+
// `ctx.now` never goes backwards within a market — but a fresh feed is
|
|
43
|
+
// built per market, so this is not an assumption about the strategy.
|
|
44
|
+
this.cursor = 0;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Index one past the last row visible at `now`. */
|
|
48
|
+
_visibleCount(now) {
|
|
49
|
+
const limit = now - this.lagMs;
|
|
50
|
+
while (this.cursor < this.rows.length && this.rows[this.cursor].ts_ms <= limit) {
|
|
51
|
+
this.cursor += 1;
|
|
52
|
+
}
|
|
53
|
+
// Defensive: if time moved backwards, do not report rows from the future.
|
|
54
|
+
if (this.cursor > 0 && this.rows[this.cursor - 1].ts_ms > limit) {
|
|
55
|
+
let i = this.cursor;
|
|
56
|
+
while (i > 0 && this.rows[i - 1].ts_ms > limit) i -= 1;
|
|
57
|
+
return i;
|
|
58
|
+
}
|
|
59
|
+
return this.cursor;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
viewAt(now) {
|
|
63
|
+
const n = this._visibleCount(now);
|
|
64
|
+
const rows = this.rows;
|
|
65
|
+
// Captured, not read off `this`: the object below is what the strategy
|
|
66
|
+
// holds, so `this` inside its methods is that frozen view — `this.lagMs`
|
|
67
|
+
// was undefined there and turned the clamp into NaN, which made at() return
|
|
68
|
+
// null for every timestamp.
|
|
69
|
+
const { lagMs } = this;
|
|
70
|
+
const horizon = now - lagMs;
|
|
71
|
+
return Object.freeze({
|
|
72
|
+
/** The most recent row at or before now, or null. */
|
|
73
|
+
get last() { return n > 0 ? freezeRow(rows[n - 1]) : null; },
|
|
74
|
+
|
|
75
|
+
/** The last `k` visible rows, oldest first. Never more than exist. */
|
|
76
|
+
window(k) {
|
|
77
|
+
const want = Math.max(0, Math.min(Number(k) || 0, n));
|
|
78
|
+
const out = [];
|
|
79
|
+
for (let i = n - want; i < n; i += 1) out.push(freezeRow(rows[i]));
|
|
80
|
+
return out;
|
|
81
|
+
},
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The row in effect at `ts`. Clamped to now: asking for a later
|
|
85
|
+
* timestamp cannot reach a later row.
|
|
86
|
+
*/
|
|
87
|
+
at(ts) {
|
|
88
|
+
const asked = Number(ts);
|
|
89
|
+
const t = Number.isFinite(asked) ? Math.min(asked, horizon) : horizon;
|
|
90
|
+
let lo = 0;
|
|
91
|
+
let hi = n - 1;
|
|
92
|
+
let found = -1;
|
|
93
|
+
while (lo <= hi) {
|
|
94
|
+
const mid = (lo + hi) >> 1;
|
|
95
|
+
if (rows[mid].ts_ms <= t) { found = mid; lo = mid + 1; } else { hi = mid - 1; }
|
|
96
|
+
}
|
|
97
|
+
return found >= 0 ? freezeRow(rows[found]) : null;
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Build the feeds a run declared, keyed by the name the strategy will ask for.
|
|
105
|
+
*
|
|
106
|
+
* @param declared manifest names, e.g. ['binance:btcusdt:spot:1s']
|
|
107
|
+
* @param rowsByName name -> ascending rows
|
|
108
|
+
* @param lagByName name -> publication lag in ms
|
|
109
|
+
*/
|
|
110
|
+
export function buildFeeds(declared, rowsByName, lagByName = {}) {
|
|
111
|
+
const out = new Map();
|
|
112
|
+
for (const name of declared ?? []) {
|
|
113
|
+
out.set(name, new PointInTimeFeed(rowsByName[name] ?? [], { lagMs: lagByName[name] ?? 0 }));
|
|
114
|
+
}
|
|
115
|
+
return out;
|
|
116
|
+
}
|
|
@@ -129,8 +129,74 @@ export class Portfolio {
|
|
|
129
129
|
if (!isSide(order?.side)) { this.rejected += 1; return null; }
|
|
130
130
|
|
|
131
131
|
const leg = this.#legs(marketId)[order.side];
|
|
132
|
-
|
|
133
|
-
|
|
132
|
+
|
|
133
|
+
// ONE PLACE THAT DECIDES WHETHER AN ORDER IS USABLE.
|
|
134
|
+
//
|
|
135
|
+
// This was three separate checks bolted on one at a time, and each time a
|
|
136
|
+
// new shape of bad input walked past the ones already there — an infinite
|
|
137
|
+
// notional, then a size and a notional together, then a bad `limit` on a
|
|
138
|
+
// plain size order. Whack-a-mole on a consumption point is how you end up
|
|
139
|
+
// with an engine that rejects what it happens to have been asked about.
|
|
140
|
+
//
|
|
141
|
+
// Every unusable order is COUNTED AND RETURNS NULL. Never thrown: a bad
|
|
142
|
+
// order is one order, and otengine.py raising ValueError/OverflowError on
|
|
143
|
+
// the same input turned "reject one order" into "fail the whole run" —
|
|
144
|
+
// same input, two different outcomes, in a product whose whole promise is
|
|
145
|
+
// that the two engines are the same engine. That mirror is held to this
|
|
146
|
+
// table by runner/conformance.
|
|
147
|
+
// A NUMBER, not something Number() is willing to turn into one.
|
|
148
|
+
//
|
|
149
|
+
// Coercion is not validation: `Number('10')` is 10, `Number([10])` is 10,
|
|
150
|
+
// `Number(true)` is 1. Python's `float()` agrees about the string and the
|
|
151
|
+
// bool and RAISES on the list — so `{ size: [10] }` filled ten contracts
|
|
152
|
+
// in JS and was rejected in Python, from one strategy, on one input.
|
|
153
|
+
//
|
|
154
|
+
// The SDK's Order requires a number and the docs say a number. Anything
|
|
155
|
+
// else is a mistake in the strategy, and a mistake that fills is worse
|
|
156
|
+
// than one that is counted.
|
|
157
|
+
const finite = (v) => (
|
|
158
|
+
typeof v === 'number' && Number.isFinite(v) ? v : null
|
|
159
|
+
);
|
|
160
|
+
|
|
161
|
+
const hasSize = order.size != null;
|
|
162
|
+
const hasNotional = order.notional != null;
|
|
163
|
+
// Contradictory instructions. Choosing one silently would trade a contract
|
|
164
|
+
// count while the author believed they had set a spending cap.
|
|
165
|
+
if (hasSize === hasNotional) { this.rejected += 1; return null; }
|
|
166
|
+
|
|
167
|
+
// A limit is optional, but a limit that is PRESENT must be a price: an
|
|
168
|
+
// outcome token trades in [0, 1], and `limit: 2` or `limit: Infinity`
|
|
169
|
+
// means "pay anything" — which is what it silently did.
|
|
170
|
+
let limit = null;
|
|
171
|
+
if (order.limit != null) {
|
|
172
|
+
limit = finite(order.limit);
|
|
173
|
+
if (limit == null || limit < 0 || limit > 1) { this.rejected += 1; return null; }
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
let size;
|
|
177
|
+
if (hasNotional) {
|
|
178
|
+
// MONEY -> CONTRACTS, here rather than in the SDK's Order constructor:
|
|
179
|
+
// a hook may return a plain object literal — most of the JS examples do
|
|
180
|
+
// — and one that never passed through `new Order()` would carry a
|
|
181
|
+
// `notional` nobody converted.
|
|
182
|
+
//
|
|
183
|
+
// The divisor is the limit, never the current best price: a contract
|
|
184
|
+
// costs whatever it fills at and a marketable order walks the book, so
|
|
185
|
+
// dividing by the touch overspends the moment there is any slippage.
|
|
186
|
+
const budget = finite(order.notional);
|
|
187
|
+
if (budget == null || budget <= 0 || limit == null || limit <= 0) {
|
|
188
|
+
this.rejected += 1; return null;
|
|
189
|
+
}
|
|
190
|
+
// The QUOTIENT can overflow from two finite inputs: 1e308 / 0.01 is
|
|
191
|
+
// Infinity. Checked before the floor, because that is where Python
|
|
192
|
+
// raises.
|
|
193
|
+
const q = budget / limit;
|
|
194
|
+
if (!Number.isFinite(q)) { this.rejected += 1; return null; }
|
|
195
|
+
size = Math.floor(q);
|
|
196
|
+
} else {
|
|
197
|
+
size = finite(order.size);
|
|
198
|
+
}
|
|
199
|
+
if (size == null || !(size > 0)) { this.rejected += 1; return null; }
|
|
134
200
|
|
|
135
201
|
// reduce_only is clamped to what is open. A strategy asking to close more
|
|
136
202
|
// than it holds must not accidentally open the other way.
|
|
@@ -139,9 +205,15 @@ export class Portfolio {
|
|
|
139
205
|
if (!(size > EPS)) { this.rejected += 1; return null; }
|
|
140
206
|
}
|
|
141
207
|
|
|
142
|
-
|
|
208
|
+
// ONE effective order from here on: the derived size has to reach the fill
|
|
209
|
+
// row as well as the match. It did not, and the row's `requested` came out
|
|
210
|
+
// as NaN for a notional-only order — the fill was executed (the fee was
|
|
211
|
+
// identical) but the ROW was rejected by the parser and never emitted, so
|
|
212
|
+
// the run silently lost its fill log. otengine.py does the same.
|
|
213
|
+
const eff = { ...order, size, limit };
|
|
214
|
+
const res = matchOrder(book, eff);
|
|
143
215
|
if (res.filled <= 0) {
|
|
144
|
-
this.fills.push(this.#fillRow({ ts, marketId, order, res, tag, realised: 0, fee: 0 }));
|
|
216
|
+
this.fills.push(this.#fillRow({ ts, marketId, order: eff, res, tag, realised: 0, fee: 0 }));
|
|
145
217
|
return res;
|
|
146
218
|
}
|
|
147
219
|
|
|
@@ -177,7 +249,7 @@ export class Portfolio {
|
|
|
177
249
|
this.cash -= res.notional + fee;
|
|
178
250
|
}
|
|
179
251
|
|
|
180
|
-
this.fills.push(this.#fillRow({ ts, marketId, order, res, tag, realised, fee }));
|
|
252
|
+
this.fills.push(this.#fillRow({ ts, marketId, order: eff, res, tag, realised, fee }));
|
|
181
253
|
return res;
|
|
182
254
|
}
|
|
183
255
|
|
package/runner/engine/replay.mjs
CHANGED
|
@@ -15,25 +15,50 @@
|
|
|
15
15
|
import { Book } from './book.mjs';
|
|
16
16
|
import { Portfolio } from './portfolio.mjs';
|
|
17
17
|
|
|
18
|
+
|
|
19
|
+
// Mirrors LIMITS.logLineChars / LIMITS.logBytesPerRun, and does NOT import
|
|
20
|
+
// them. This file is baked into the sandbox image, which contains runner/ and
|
|
21
|
+
// nothing else — an import of api/ resolves fine on a developer's machine and
|
|
22
|
+
// on the worker, and then fails inside every container with
|
|
23
|
+
// ERR_MODULE_NOT_FOUND. otreplay.py mirrors the same two numbers for the same
|
|
24
|
+
// reason. replay-limits.test.mjs asserts both copies still equal the contract.
|
|
25
|
+
const LOG_LINE_CHARS = 512;
|
|
26
|
+
const LOG_BYTES_PER_RUN = 2 * 1024 * 1024;
|
|
27
|
+
|
|
28
|
+
// A settlement recompute is a once-per-market claim, so these are generous.
|
|
29
|
+
// They exist because `crosschecks` rides the same result line as everything
|
|
30
|
+
// else: unbounded, it is an output channel with no budget. Mirrored in
|
|
31
|
+
// otreplay.py.
|
|
32
|
+
const MAX_CROSSCHECKS_PER_MARKET = 16;
|
|
33
|
+
const CROSSCHECK_CLAIMED_CHARS = 32;
|
|
34
|
+
|
|
35
|
+
|
|
18
36
|
/** Event kinds the loop understands, in the order they dispatch. */
|
|
19
37
|
export const EVENT_KINDS = Object.freeze(['tick', 'book', 'trade']);
|
|
20
38
|
|
|
21
39
|
/**
|
|
22
40
|
* Per-event budget.
|
|
23
41
|
*
|
|
24
|
-
* Measured over the strategy's own hook, not the loop around it.
|
|
25
|
-
* kills the shard rather than the run: one pathological market must
|
|
26
|
-
* the customer the other 719.
|
|
42
|
+
* Measured over the strategy's own hook, not the loop around it. Sustained cost
|
|
43
|
+
* over budget kills the shard rather than the run: one pathological market must
|
|
44
|
+
* not cost the customer the other 719.
|
|
27
45
|
*/
|
|
28
46
|
export class BudgetMonitor {
|
|
29
|
-
constructor({ limitMicros = 400, sampleFloor =
|
|
47
|
+
constructor({ limitMicros = 400, sampleFloor = 2000 } = {}) {
|
|
30
48
|
this.limitMicros = limitMicros;
|
|
31
49
|
this.sampleFloor = sampleFloor;
|
|
32
|
-
this.tolerance = tolerance;
|
|
33
50
|
this.count = 0;
|
|
34
51
|
this.breaches = 0;
|
|
35
52
|
this.maxMicros = 0;
|
|
36
53
|
this.totalMicros = 0;
|
|
54
|
+
// The most recent `sampleFloor` events, as a ring. One monitor covers the
|
|
55
|
+
// WHOLE run, so a lifetime mean is diluted by however much came before: a
|
|
56
|
+
// strategy that runs 18,000 events at 8us and then 2,000 at 2,000us has a
|
|
57
|
+
// lifetime mean of 207us and passes, while its last 2,000 events are
|
|
58
|
+
// continuously 5x over budget. The window is what makes "sustained" local.
|
|
59
|
+
this.window = new Float64Array(sampleFloor);
|
|
60
|
+
this.windowSum = 0;
|
|
61
|
+
this.windowAt = 0;
|
|
37
62
|
}
|
|
38
63
|
|
|
39
64
|
record(micros) {
|
|
@@ -41,14 +66,64 @@ export class BudgetMonitor {
|
|
|
41
66
|
this.totalMicros += micros;
|
|
42
67
|
if (micros > this.maxMicros) this.maxMicros = micros;
|
|
43
68
|
if (micros > this.limitMicros) this.breaches += 1;
|
|
69
|
+
this.windowSum += micros - this.window[this.windowAt];
|
|
70
|
+
this.window[this.windowAt] = micros;
|
|
71
|
+
this.windowAt = (this.windowAt + 1) % this.window.length;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Mean of the most recent `sampleFloor` events. Zero until the window fills. */
|
|
75
|
+
get windowMicros() {
|
|
76
|
+
return this.count >= this.sampleFloor ? this.windowSum / this.window.length : 0;
|
|
44
77
|
}
|
|
45
78
|
|
|
46
79
|
/**
|
|
47
|
-
*
|
|
48
|
-
*
|
|
80
|
+
* SUSTAINED cost, which is what the limit is for and what customers are told
|
|
81
|
+
* it means ("sustained breach kills the shard"): the mean of the most recent
|
|
82
|
+
* `sampleFloor` events, not the tail and not the lifetime.
|
|
83
|
+
*
|
|
84
|
+
* This judged the breach RATE against a 1% tolerance, and it was measuring
|
|
85
|
+
* the wrong machine. The budget brackets each hook with two wall-clock reads,
|
|
86
|
+
* on a 2-core box where the worker is decompressing and feeding stdin the
|
|
87
|
+
* whole time and the sandbox holds one vCPU — so an event that gets
|
|
88
|
+
* descheduled is recorded as an event the strategy spent 4ms in. Measured
|
|
89
|
+
* inside the real image: the same strategy on an idle host averages 7.8us
|
|
90
|
+
* with a 766us worst case, and under contention averages 20.9us with a 4090us
|
|
91
|
+
* worst case. Nothing about the strategy changed.
|
|
92
|
+
*
|
|
93
|
+
* The page's own sample was rejected in production at avg 72us — a fifth of
|
|
94
|
+
* its 400us budget — because 1.1% of its events had been interrupted. The
|
|
95
|
+
* breaches were not even front-loaded, so a longer warm-up floor could never
|
|
96
|
+
* have fixed it: measured in-image they land at events 171, 2368, 3201 and so
|
|
97
|
+
* on, which is the shape of GC and scheduling, not of a slow strategy.
|
|
98
|
+
*
|
|
99
|
+
* The mean is what actually predicts the thing this protects — the 20-minute
|
|
100
|
+
* wall clock is mean times event count — and it is not fooled by a machine
|
|
101
|
+
* that takes the CPU away. A strategy that really is slow raises the mean; an
|
|
102
|
+
* interrupted one does not. A single hook that never returns is still caught,
|
|
103
|
+
* by the run deadline, which is where that belongs.
|
|
104
|
+
*
|
|
105
|
+
* WINDOWED, NOT LIFETIME. One monitor covers the whole run, so a lifetime
|
|
106
|
+
* mean lets a cheap prefix pay for an expensive phase: 18,000 events at 8us
|
|
107
|
+
* followed by 2,000 at 2,000us averages 207us and passes, while the strategy
|
|
108
|
+
* has been 5x over budget for its last two thousand events. The window is
|
|
109
|
+
* what makes "sustained" mean sustained rather than "on average, eventually".
|
|
110
|
+
*
|
|
111
|
+
* `breaches` and `max_micros` stay in the summary. They are good diagnostics.
|
|
112
|
+
* They are not a verdict.
|
|
113
|
+
*
|
|
114
|
+
* KNOWN AND DELIBERATE GAP: a low-frequency, very heavy event slips through.
|
|
115
|
+
* One 500ms hook among 2,000 events at 8us is a window mean of 258us, inside
|
|
116
|
+
* budget; it would take 800ms to trip on its own. A per-hook hard cap would
|
|
117
|
+
* close it, and it is not being added, because this limit has now been wrong
|
|
118
|
+
* twice and BOTH times the same way — a one-off cost judged as a sustained
|
|
119
|
+
* one, killing a strategy that was fine. Building an index on the first tick
|
|
120
|
+
* is a legitimate 300ms hook. The wall clock already bounds total resource
|
|
121
|
+
* use, so the cap would buy protection against a failure nobody has seen at
|
|
122
|
+
* the cost of the exact mistake already made twice. Add it when a real run
|
|
123
|
+
* demonstrates the need, not before.
|
|
49
124
|
*/
|
|
50
125
|
get breached() {
|
|
51
|
-
return this.count >= this.sampleFloor && this.
|
|
126
|
+
return this.count >= this.sampleFloor && this.windowMicros > this.limitMicros;
|
|
52
127
|
}
|
|
53
128
|
|
|
54
129
|
get avgMicros() { return this.count ? this.totalMicros / this.count : 0; }
|
|
@@ -59,6 +134,10 @@ export class BudgetMonitor {
|
|
|
59
134
|
breaches: this.breaches,
|
|
60
135
|
breach_rate: this.count ? this.breaches / this.count : 0,
|
|
61
136
|
avg_micros: this.avgMicros,
|
|
137
|
+
// The number the verdict is actually made on. Without it a rejection
|
|
138
|
+
// shows a lifetime average comfortably inside budget and reads as a lie.
|
|
139
|
+
window_micros: this.windowMicros,
|
|
140
|
+
window_events: this.sampleFloor,
|
|
62
141
|
max_micros: this.maxMicros,
|
|
63
142
|
limit_micros: this.limitMicros,
|
|
64
143
|
};
|
|
@@ -124,7 +203,7 @@ function bookView(book) {
|
|
|
124
203
|
});
|
|
125
204
|
}
|
|
126
205
|
|
|
127
|
-
function createCtx({ params, portfolio, marketId, market,
|
|
206
|
+
function createCtx({ params, portfolio, marketId, market, logBudget, references, series, rng }) {
|
|
128
207
|
const history = [];
|
|
129
208
|
const logs = [];
|
|
130
209
|
const crosschecks = [];
|
|
@@ -171,8 +250,20 @@ function createCtx({ params, portfolio, marketId, market, logLimit, references,
|
|
|
171
250
|
},
|
|
172
251
|
|
|
173
252
|
log(msg) {
|
|
174
|
-
|
|
175
|
-
|
|
253
|
+
// BYTES FOR THE WHOLE RUN, not lines per market. The old shape — 10,000
|
|
254
|
+
// lines per market, no length cap — let a run emit the archive it had
|
|
255
|
+
// just paid a market-day for into a file the customer downloads. See
|
|
256
|
+
// LIMITS.logBytesPerRun for the arithmetic that picks these numbers.
|
|
257
|
+
if (logBudget.spent >= logBudget.bytes) { logTruncated = true; return; }
|
|
258
|
+
const line = `${now} ${String(msg)}`.slice(0, logBudget.lineChars);
|
|
259
|
+
// BYTES, not string length. logs.txt is UTF-8, and `.length` counts
|
|
260
|
+
// UTF-16 units — so a run logging Chinese spent a third of what it
|
|
261
|
+
// wrote, and the archive could reach three times the 2 MB this budget
|
|
262
|
+
// advertises. The line cap stays in characters because it is about
|
|
263
|
+
// being readable; the run cap is about how much data leaves with the
|
|
264
|
+
// customer, and that is measured in bytes.
|
|
265
|
+
logBudget.spent += Buffer.byteLength(line, 'utf8') + 1;
|
|
266
|
+
logs.push(line);
|
|
176
267
|
},
|
|
177
268
|
|
|
178
269
|
/** Seeded generator — the only randomness available, and it is recorded. */
|
|
@@ -240,9 +331,19 @@ function createCtx({ params, portfolio, marketId, market, logLimit, references,
|
|
|
240
331
|
// recompute match that never happened. The cross-check panel's whole
|
|
241
332
|
// value is that it is the ARCHIVE's answer, not the strategy's.
|
|
242
333
|
const official = market?.outcome ?? null;
|
|
334
|
+
// BOUNDED, for the same reason ctx.log is. `outcome` is whatever the
|
|
335
|
+
// strategy passed and this can be called on every event, so an unbounded
|
|
336
|
+
// push here is an unmetered output channel wearing a different name: the
|
|
337
|
+
// whole array is serialised onto the authenticated result line, sent,
|
|
338
|
+
// and parsed by the worker before anything downstream gets to ignore it.
|
|
339
|
+
// The panel only ever shows an aggregate, so nothing of value is lost by
|
|
340
|
+
// capping — a settlement recompute is a once-per-market claim.
|
|
341
|
+
if (crosschecks.length >= MAX_CROSSCHECKS_PER_MARKET) return;
|
|
243
342
|
crosschecks.push({
|
|
244
343
|
market_id: marketId,
|
|
245
|
-
claimed: outcome
|
|
344
|
+
claimed: typeof outcome === 'string'
|
|
345
|
+
? outcome.slice(0, CROSSCHECK_CLAIMED_CHARS)
|
|
346
|
+
: String(outcome).slice(0, CROSSCHECK_CLAIMED_CHARS),
|
|
246
347
|
official,
|
|
247
348
|
match: official === outcome,
|
|
248
349
|
});
|
|
@@ -293,11 +394,32 @@ const HOOK_FOR = { tick: 'on_tick', book: 'on_book', trade: 'on_trade' };
|
|
|
293
394
|
* decision. Zero is "as captured"; the latency panel is this same replay at
|
|
294
395
|
* 100ms, 250ms, 500ms, 1s and 2s.
|
|
295
396
|
*/
|
|
397
|
+
/**
|
|
398
|
+
* A log allowance for one run.
|
|
399
|
+
*
|
|
400
|
+
* Bytes, not lines, and shared by every market in the run — see
|
|
401
|
+
* LIMITS.logBytesPerRun for why those two choices are the whole fix.
|
|
402
|
+
*/
|
|
403
|
+
export function makeLogBudget({
|
|
404
|
+
bytes = LOG_BYTES_PER_RUN,
|
|
405
|
+
lineChars = LOG_LINE_CHARS,
|
|
406
|
+
} = {}) {
|
|
407
|
+
return { bytes, lineChars, spent: 0 };
|
|
408
|
+
}
|
|
409
|
+
|
|
296
410
|
export function replayMarket({
|
|
297
411
|
market, events, strategy, hooks,
|
|
298
412
|
portfolio = null,
|
|
299
413
|
fillDelayMs = 0,
|
|
300
|
-
|
|
414
|
+
/**
|
|
415
|
+
* The run's remaining log allowance, SHARED ACROSS MARKETS.
|
|
416
|
+
*
|
|
417
|
+
* Passed in rather than created here, because a per-market allowance is what
|
|
418
|
+
* the old limit was and what made the log channel an export route: 386
|
|
419
|
+
* markets a day each got their own budget. One object for the whole run is
|
|
420
|
+
* the fix — the harness makes it once and hands the same one to every market.
|
|
421
|
+
*/
|
|
422
|
+
logBudget = null,
|
|
301
423
|
budget = null,
|
|
302
424
|
references = null,
|
|
303
425
|
series = null,
|
|
@@ -338,7 +460,7 @@ export function replayMarket({
|
|
|
338
460
|
portfolio: pf,
|
|
339
461
|
marketId,
|
|
340
462
|
market: engineMarket,
|
|
341
|
-
|
|
463
|
+
logBudget: logBudget ?? makeLogBudget(),
|
|
342
464
|
references,
|
|
343
465
|
series,
|
|
344
466
|
rng: makeRng(seed),
|