outcometick 1.5.2 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -1
- package/api/lib/backtest-contract.mjs +335 -16
- package/api/lib/backtest-datasets.mjs +125 -1
- package/api/lib/backtest-manifest.mjs +52 -4
- package/api/lib/coverage-window.mjs +63 -1
- package/api/lib/data-taxonomy.mjs +15 -1
- package/cli/commands/run.mjs +206 -37
- package/cli/commands/submit.mjs +40 -3
- package/cli/local-data.mjs +86 -15
- package/cli/ot.mjs +26 -3
- package/index.d.ts +31 -4
- package/package.json +1 -1
- package/runner/archive.mjs +24 -13
- package/runner/engine/book.mjs +12 -1
- package/runner/engine/feed.mjs +116 -0
- package/runner/engine/portfolio.mjs +77 -5
- package/runner/engine/replay.mjs +136 -14
- package/runner/engine/report.mjs +96 -34
- package/runner/events.mjs +788 -55
- package/runner/harness/node/harness.mjs +166 -13
- package/runner/harness/node/sdk/index.d.ts +31 -4
- package/runner/harness/node/sdk/index.mjs +46 -2
- package/runner/harness/protocol.mjs +31 -3
- package/runner/harness/python/harness.py +137 -19
- package/runner/harness/python/otengine.py +116 -9
- package/runner/harness/python/otfeed.py +109 -0
- package/runner/harness/python/otreplay.py +90 -8
- package/runner/harness/python/outcometick.py +48 -4
- package/runner/series-data.mjs +220 -0
|
@@ -22,6 +22,53 @@ export const FIRST_COMPLETE_DAY = Object.freeze({
|
|
|
22
22
|
/** The product as a whole starts when its earliest venue is complete. */
|
|
23
23
|
export const PRODUCT_FIRST_COMPLETE_DAY = FIRST_COMPLETE_DAY.polymarket;
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* First UTC day a BACKTEST can use. Different question from the one above.
|
|
27
|
+
*
|
|
28
|
+
* A replay has to know which price stream settled each market, and that answer
|
|
29
|
+
* comes from the market's own `raw.cryptoMarketConfig.twapLookbackSeconds`.
|
|
30
|
+
* MEASURED: polymarket markets carry it from 2026-08-08 and not before —
|
|
31
|
+
* 0/98 on 08-05, 08-06 and 08-07, 98/98 on 08-08. Days before that decode
|
|
32
|
+
* fine and then drop every market ("settlement stream could not be resolved"),
|
|
33
|
+
* so a run over them burns twenty minutes and refunds in full.
|
|
34
|
+
*
|
|
35
|
+
* WHY NOT INFER THE OLD RULE. It was tried, with the archive as the judge:
|
|
36
|
+
* every settled market records its strike and its actual outcome, so a
|
|
37
|
+
* candidate rule can be scored against reality. "Raw chainlink price at close"
|
|
38
|
+
* reproduces 97.9–100% of settled markets across three sampled days — close,
|
|
39
|
+
* and therefore worse than useless: a table built on it would decide about
|
|
40
|
+
* 1.5% of markets the wrong way, in reports that look completely ordinary.
|
|
41
|
+
* The TWAP hypotheses could not even be tested, because a recomputed TWAP
|
|
42
|
+
* failed its control against the archived one (differing by up to $57, enough
|
|
43
|
+
* to settle a market the other way). Owner's call, 2026-08-25: offer only the
|
|
44
|
+
* days the archive can answer for.
|
|
45
|
+
*
|
|
46
|
+
* This is DATA COVERAGE, not a limit — `/v1/backtest/capacity` reports it and
|
|
47
|
+
* the page offers only what it can deliver.
|
|
48
|
+
*/
|
|
49
|
+
export const FIRST_BACKTEST_DAY = Object.freeze({
|
|
50
|
+
polymarket: '2026-08-08',
|
|
51
|
+
// Predict settles off fields carried on the market row itself rather than a
|
|
52
|
+
// separate stream, so it is not affected by the polymarket cutover. Pinned
|
|
53
|
+
// to its first complete day until measured otherwise.
|
|
54
|
+
predict: FIRST_COMPLETE_DAY.predict,
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The days a backtest may actually be sold on a venue.
|
|
59
|
+
*
|
|
60
|
+
* Both floors apply: a day has to be complete AND has to be one the archive
|
|
61
|
+
* can attribute a settlement stream to.
|
|
62
|
+
*/
|
|
63
|
+
export function backtestDayList(days, venue) {
|
|
64
|
+
const v = String(venue ?? '').toLowerCase();
|
|
65
|
+
const floor = [
|
|
66
|
+
FIRST_COMPLETE_DAY[v] ?? PRODUCT_FIRST_COMPLETE_DAY,
|
|
67
|
+
FIRST_BACKTEST_DAY[v] ?? FIRST_BACKTEST_DAY.polymarket,
|
|
68
|
+
].sort().pop();
|
|
69
|
+
return completeDayList(days, floor);
|
|
70
|
+
}
|
|
71
|
+
|
|
25
72
|
/**
|
|
26
73
|
* The advertised window over a sorted list of archived days.
|
|
27
74
|
*
|
|
@@ -31,8 +78,23 @@ export const PRODUCT_FIRST_COMPLETE_DAY = FIRST_COMPLETE_DAY.polymarket;
|
|
|
31
78
|
*
|
|
32
79
|
* @param {string[]} days sorted ascending
|
|
33
80
|
*/
|
|
81
|
+
/**
|
|
82
|
+
* The days a run may actually be sold, out of everything the archive holds.
|
|
83
|
+
*
|
|
84
|
+
* One definition, used by both the advertised count and the billable window.
|
|
85
|
+
* A partial day is real data and the API says it exists, but a full credit buys
|
|
86
|
+
* a market-DAY — charging one for a day collection only caught part of is the
|
|
87
|
+
* same gap-must-reduce-billing rule the quote already follows, pointed at the
|
|
88
|
+
* front of the archive instead of the middle.
|
|
89
|
+
*
|
|
90
|
+
* @param {string[]} days sorted ascending
|
|
91
|
+
*/
|
|
92
|
+
export function completeDayList(days, firstComplete = PRODUCT_FIRST_COMPLETE_DAY) {
|
|
93
|
+
return (days ?? []).filter((d) => d >= firstComplete);
|
|
94
|
+
}
|
|
95
|
+
|
|
34
96
|
export function completeWindow(days, firstComplete = PRODUCT_FIRST_COMPLETE_DAY) {
|
|
35
|
-
const full = (days
|
|
97
|
+
const full = completeDayList(days, firstComplete);
|
|
36
98
|
return {
|
|
37
99
|
firstCompleteDay: full[0] ?? null,
|
|
38
100
|
// Length of the list, not a date subtraction: a gap in the archive must
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { venueOfPath } from './venue-path.mjs';
|
|
12
12
|
|
|
13
13
|
/** Asset symbols we collect, longest-first so BNBUSDT matches before BNB. */
|
|
14
|
-
const ASSETS = ['BTC', 'ETH', 'SOL', 'XRP', 'DOGE', 'BNB', 'HYPE', 'ZEC'];
|
|
14
|
+
export const ASSETS = ['BTC', 'ETH', 'SOL', 'XRP', 'DOGE', 'BNB', 'HYPE', 'ZEC'];
|
|
15
15
|
|
|
16
16
|
/** Datasets, as a customer would name them. */
|
|
17
17
|
export const DATASETS = {
|
|
@@ -114,6 +114,20 @@ export function classifyPath(filePath) {
|
|
|
114
114
|
* No real dimension value is 'none' (intervals are 1s…1mo, assets are BTC…ZEC),
|
|
115
115
|
* so the token cannot collide with data.
|
|
116
116
|
*/
|
|
117
|
+
/**
|
|
118
|
+
* In the archive, never offered publicly.
|
|
119
|
+
*
|
|
120
|
+
* ZEC markets were collected but never went live on the venue, so counting it
|
|
121
|
+
* makes every public figure one too high — "Polymarket 8 assets" printed beside
|
|
122
|
+
* a venue that shows seven. Excluded from what we ADVERTISE, not from what we
|
|
123
|
+
* serve: a subscriber querying the archive still gets what the archive holds.
|
|
124
|
+
*/
|
|
125
|
+
export const UNLISTED_ASSETS = Object.freeze(['ZEC']);
|
|
126
|
+
|
|
127
|
+
/** The assets a public-facing figure should count. */
|
|
128
|
+
export const publicAssets = (assets) =>
|
|
129
|
+
[...assets].filter((a) => !UNLISTED_ASSETS.includes(a)).sort();
|
|
130
|
+
|
|
117
131
|
export const NO_VALUE = 'none';
|
|
118
132
|
|
|
119
133
|
/**
|
package/cli/commands/run.mjs
CHANGED
|
@@ -20,7 +20,12 @@ import { fileURLToPath } from 'node:url';
|
|
|
20
20
|
|
|
21
21
|
import { LANGUAGES, HOOK_NAMES, LIMITS } from '../../api/lib/backtest-contract.mjs';
|
|
22
22
|
import { CHANNEL, EXIT, parseTrade, parseFill, parseResult, parseOutputLine } from '../../runner/harness/protocol.mjs';
|
|
23
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
countMarketDays, countStreams, buildCoverage, mergeReferenceRows, makeBookThrottle,
|
|
25
|
+
sortMarketsForReplay,
|
|
26
|
+
} from '../../runner/events.mjs';
|
|
27
|
+
import { loadSeries } from '../../runner/series-data.mjs';
|
|
28
|
+
import { buildReport } from '../../runner/engine/report.mjs';
|
|
24
29
|
import { buildArchive } from '../../runner/archive.mjs';
|
|
25
30
|
import { loadLocalDay, localDays, looksLikeArchive } from '../local-data.mjs';
|
|
26
31
|
import { readSubmission, validate } from '../ot.mjs';
|
|
@@ -29,36 +34,65 @@ const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
|
29
34
|
const RUNNER = path.join(HERE, '..', '..', 'runner');
|
|
30
35
|
|
|
31
36
|
/** Run one pass of the local harness over the given markets. */
|
|
32
|
-
function runHarness({
|
|
37
|
+
function runHarness({
|
|
38
|
+
languageId, jobDir, job, markets, outputKey,
|
|
39
|
+
// The submitter's own CSV, already parsed. Threaded in rather than read here
|
|
40
|
+
// so it is parsed once for the whole run, exactly as the worker does.
|
|
41
|
+
seriesRows = {}, seriesLags = {}, seriesNames = [],
|
|
42
|
+
}) {
|
|
33
43
|
const cmd = languageId === 'python' ? (process.env.OT_PYTHON || 'python3') : process.execPath;
|
|
34
44
|
const argv = languageId === 'python'
|
|
35
45
|
? [path.join(RUNNER, 'harness/python/harness.py'), jobDir]
|
|
36
46
|
: [path.join(RUNNER, 'harness/node/harness.mjs'), jobDir];
|
|
37
47
|
|
|
38
48
|
return new Promise((resolve, reject) => {
|
|
39
|
-
|
|
49
|
+
// stdout is the result channel (see runner/harness/protocol.mjs), so it
|
|
50
|
+
// cannot be inherited: a strategy's own output would land mid-line. The
|
|
51
|
+
// harness discards that output for the same reason a container does.
|
|
52
|
+
const child = spawn(cmd, argv, { stdio: ['pipe', 'pipe', 'pipe'] });
|
|
40
53
|
const lines = [];
|
|
54
|
+
let forged = 0;
|
|
41
55
|
let tail = '';
|
|
42
56
|
let stderr = '';
|
|
43
57
|
child.stderr.on('data', (d) => { stderr += d; });
|
|
44
|
-
child.
|
|
58
|
+
child.stdout.on('data', (d) => {
|
|
45
59
|
tail += d;
|
|
46
60
|
const parts = tail.split('\n');
|
|
47
61
|
tail = parts.pop();
|
|
48
62
|
for (const line of parts) {
|
|
49
63
|
if (!line) continue;
|
|
50
64
|
const parsed = parseOutputLine(outputKey, line);
|
|
65
|
+
// Counted, not shrugged off. The queued worker counts the same bytes as
|
|
66
|
+
// forged, and a stray write between two real lines corrupts the one
|
|
67
|
+
// after it — so a local replay that quietly drops them can hand back a
|
|
68
|
+
// report the queued run would never have produced. Silence here is the
|
|
69
|
+
// exact shape of the divergence this CLI exists to rule out.
|
|
51
70
|
if (parsed) lines.push(parsed);
|
|
71
|
+
else forged += 1;
|
|
52
72
|
}
|
|
53
73
|
});
|
|
54
74
|
child.on('error', reject);
|
|
55
|
-
child.on('close', (code) => resolve({ code, stderr, lines }));
|
|
75
|
+
child.on('close', (code) => resolve({ code, stderr, lines, forged }));
|
|
56
76
|
|
|
57
77
|
child.stdin.on('error', () => {});
|
|
58
78
|
child.stdin.write(`${JSON.stringify({ ...job, outputKey })}\n`);
|
|
59
79
|
for (const m of markets) {
|
|
60
|
-
|
|
61
|
-
|
|
80
|
+
// Series rows are INTERLEAVED into the same stream in event time, exactly
|
|
81
|
+
// as the worker sends them — and `lags` travels with them, or a signal
|
|
82
|
+
// that declared a publication delay would be visible the instant its row
|
|
83
|
+
// was stamped rather than when it could have existed.
|
|
84
|
+
const lines = m.events.map((ev) => JSON.stringify(ev));
|
|
85
|
+
const merged = seriesNames.length
|
|
86
|
+
? mergeReferenceRows(lines, seriesRows, m.market, 'ext', seriesLags)
|
|
87
|
+
: lines;
|
|
88
|
+
child.stdin.write(`${JSON.stringify({
|
|
89
|
+
market: m.market,
|
|
90
|
+
stream: m.stream,
|
|
91
|
+
n: merged.length,
|
|
92
|
+
...(seriesNames.length ? { series: seriesNames } : {}),
|
|
93
|
+
...(Object.keys(seriesLags).length ? { lags: seriesLags } : {}),
|
|
94
|
+
})}\n`);
|
|
95
|
+
for (const line of merged) child.stdin.write(`${line}\n`);
|
|
62
96
|
}
|
|
63
97
|
child.stdin.end();
|
|
64
98
|
});
|
|
@@ -68,20 +102,26 @@ function demux(lines) {
|
|
|
68
102
|
const trades = [];
|
|
69
103
|
const fills = [];
|
|
70
104
|
const logs = [];
|
|
105
|
+
// Counted, not swallowed. The queue publishes `dropped_rows`, and a local run
|
|
106
|
+
// that discarded the same rows in silence reported a clean report over the
|
|
107
|
+
// identical archive — the number exists precisely so a customer can tell that
|
|
108
|
+
// something did not parse.
|
|
109
|
+
let malformed = 0;
|
|
71
110
|
let result = parseResult({});
|
|
72
111
|
for (const { channel, payload } of lines) {
|
|
73
112
|
if (channel === CHANNEL.log) { logs.push(payload); continue; }
|
|
113
|
+
if (channel === CHANNEL.progress) continue;
|
|
74
114
|
if (channel === CHANNEL.result) {
|
|
75
|
-
try { result = parseResult(JSON.parse(payload)); } catch {
|
|
115
|
+
try { result = parseResult(JSON.parse(payload)); } catch { malformed += 1; }
|
|
76
116
|
continue;
|
|
77
117
|
}
|
|
78
118
|
let raw;
|
|
79
|
-
try { raw = JSON.parse(payload); } catch { continue; }
|
|
119
|
+
try { raw = JSON.parse(payload); } catch { malformed += 1; continue; }
|
|
80
120
|
const row = channel === CHANNEL.trade ? parseTrade(raw) : parseFill(raw);
|
|
81
|
-
if (!row) continue;
|
|
121
|
+
if (!row) { malformed += 1; continue; }
|
|
82
122
|
(channel === CHANNEL.trade ? trades : fills).push(row);
|
|
83
123
|
}
|
|
84
|
-
return { trades, fills, logs: logs.join('\n'), result };
|
|
124
|
+
return { trades, fills, logs: logs.join('\n'), result, malformed };
|
|
85
125
|
}
|
|
86
126
|
|
|
87
127
|
export async function cmdRun({ dir, flags }) {
|
|
@@ -107,23 +147,116 @@ export async function cmdRun({ dir, flags }) {
|
|
|
107
147
|
throw new Error(`${unknown.join(', ')} not in ${path.resolve(dataRoot)} — it holds ${available[0]}..${available[available.length - 1]}`);
|
|
108
148
|
}
|
|
109
149
|
|
|
150
|
+
// REFERENCE FEEDS ARE REFUSED. SERIES ARE NOT.
|
|
151
|
+
//
|
|
152
|
+
// The distinction is where the data lives, and it took a review to get right:
|
|
153
|
+
// a reference feed is the Binance archive, which sits on the worker's own disk
|
|
154
|
+
// and was deliberately never put in R2 — no local run can reach it. A series
|
|
155
|
+
// is the submitter's own CSV, and it is right here in the directory being run.
|
|
156
|
+
//
|
|
157
|
+
// Refusing both was the safe-looking answer and the wrong one: it would have
|
|
158
|
+
// forced every strategy using `ctx.ext()` to spend credits in the queue to
|
|
159
|
+
// discover a runtime mistake it could have found locally in a second. What
|
|
160
|
+
// must not happen is running with an EMPTY feed, which produces a report that
|
|
161
|
+
// looks comparable to the queued one and is not.
|
|
162
|
+
const refs = (manifest.reference ?? []).map((r) => (typeof r === 'string' ? r : r.name));
|
|
163
|
+
if (refs.length) {
|
|
164
|
+
throw new Error(
|
|
165
|
+
`this manifest declares reference feeds a local replay cannot supply:\n ${refs.join('\n ')}\n`
|
|
166
|
+
+ ' They come from an archive held on the worker, so `ot run` would hand\n'
|
|
167
|
+
+ ' your strategy empty feeds and a report that does not match the queued\n'
|
|
168
|
+
+ ' one. Submit it instead:\n'
|
|
169
|
+
+ ' ot submit . --assets btc --range "30 days"',
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// The submitter's own CSV, parsed by the SAME reader the queue uses, so a
|
|
174
|
+
// header it would reject there is rejected here too.
|
|
175
|
+
const seriesRows = manifest.series?.length
|
|
176
|
+
? loadSeries(manifest.series, checked.files)
|
|
177
|
+
: { rowsByName: {}, problems: [] };
|
|
178
|
+
const fatalSeries = (seriesRows.problems ?? []).filter((x) => x.fatal !== false);
|
|
179
|
+
if (fatalSeries.length) {
|
|
180
|
+
throw new Error(`series could not be read:\n ${
|
|
181
|
+
fatalSeries.map((x) => `${x.name} (${x.file}): ${x.problem}`).join('\n ')}`);
|
|
182
|
+
}
|
|
183
|
+
const seriesLags = Object.fromEntries(
|
|
184
|
+
(manifest.series ?? []).filter((x) => x.lag_ms > 0).map((x) => [x.name, x.lag_ms]),
|
|
185
|
+
);
|
|
186
|
+
const seriesNames = Object.keys(seriesRows.rowsByName ?? {});
|
|
187
|
+
|
|
110
188
|
const venue = flags.venue ?? 'polymarket';
|
|
111
189
|
const assets = (flags.assets ?? '').split(',').map((a) => a.trim().toUpperCase()).filter(Boolean);
|
|
112
190
|
|
|
113
191
|
const markets = [];
|
|
192
|
+
// Gaps are part of the answer, not noise to drop.
|
|
193
|
+
//
|
|
194
|
+
// A queued run's coverage names every market it could not use; a local run
|
|
195
|
+
// reading the same archive said nothing, so the two disagreed about what had
|
|
196
|
+
// been covered while agreeing about everything else. That is the harder
|
|
197
|
+
// discrepancy to notice, because the report looks complete.
|
|
198
|
+
const missing = [];
|
|
114
199
|
for (const day of days) {
|
|
115
200
|
const loaded = await loadLocalDay({
|
|
116
201
|
root: dataRoot, day, venue,
|
|
117
202
|
assets: assets.length ? assets : ['BTC', 'ETH', 'SOL', 'XRP'],
|
|
118
203
|
datasets: manifest.datasets,
|
|
204
|
+
intervals: manifest.intervals,
|
|
205
|
+
// The SAME cadence the queue would replay this range at — built from the
|
|
206
|
+
// whole range, not this day, so an asset's density never changes partway
|
|
207
|
+
// through a run. Per asset, not run-wide: see makeBookThrottle.
|
|
208
|
+
throttle: makeBookThrottle({
|
|
209
|
+
venue,
|
|
210
|
+
assets: assets.length ? assets : ['BTC', 'ETH', 'SOL', 'XRP'],
|
|
211
|
+
from: days[0],
|
|
212
|
+
to: days[days.length - 1],
|
|
213
|
+
}),
|
|
119
214
|
});
|
|
120
215
|
if (loaded.markets.length === 0) {
|
|
121
216
|
process.stderr.write(` ${day}: ${loaded.reason}\n`);
|
|
217
|
+
missing.push({
|
|
218
|
+
day,
|
|
219
|
+
reason: loaded.reason ?? 'no markets',
|
|
220
|
+
...(loaded.unusable?.length
|
|
221
|
+
? {
|
|
222
|
+
partial: false,
|
|
223
|
+
markets: loaded.unusable.length,
|
|
224
|
+
dropped: loaded.unusable.slice(0, 10),
|
|
225
|
+
reasons: [...new Set(loaded.unusable.map((u) => u.why))],
|
|
226
|
+
}
|
|
227
|
+
: {}),
|
|
228
|
+
});
|
|
122
229
|
continue;
|
|
123
230
|
}
|
|
231
|
+
if (loaded.unusable?.length) {
|
|
232
|
+
const why = `${loaded.unusable.length} market(s) dropped: ${loaded.unusable[0].why}`;
|
|
233
|
+
process.stderr.write(` ${day}: ${why}\n`);
|
|
234
|
+
missing.push({
|
|
235
|
+
day, partial: true, markets: loaded.unusable.length, reason: why,
|
|
236
|
+
// Same bound the queue applies — see MAX_DROPPED_LISTED there.
|
|
237
|
+
dropped: loaded.unusable.slice(0, 10),
|
|
238
|
+
...(loaded.unusable.length > 10
|
|
239
|
+
? { dropped_truncated: loaded.unusable.length - 10 } : {}),
|
|
240
|
+
reasons: [...new Set(loaded.unusable.map((u) => u.why))],
|
|
241
|
+
});
|
|
242
|
+
}
|
|
124
243
|
markets.push(...loaded.markets);
|
|
125
244
|
}
|
|
126
245
|
if (markets.length === 0) throw new Error('no market-days could be read from that archive');
|
|
246
|
+
// SESSION IS ONE STREAM ACROSS THE RANGE, so it is ordered once over every
|
|
247
|
+
// day — the same thing fetchMarketDays does for the queue. Ordering it a day
|
|
248
|
+
// at a time leaves the stream day-major, which is chronological only by
|
|
249
|
+
// accident and stops being so as soon as two assets are in scope. Session
|
|
250
|
+
// shares one Portfolio across every market, so this is part of the ANSWER,
|
|
251
|
+
// not of the log.
|
|
252
|
+
if ((manifest.mode ?? 'market') === 'session') {
|
|
253
|
+
sortMarketsForReplay(markets, { mode: 'session' });
|
|
254
|
+
}
|
|
255
|
+
// ONE ASSET ON ONE UTC DAY — the unit the queue bills in. `markets` is one
|
|
256
|
+
// entry per market, and a day of BTC 15-minute markets is ninety-six of them,
|
|
257
|
+
// so counting entries reported a run as being a hundred times bigger than the
|
|
258
|
+
// customer is charged for and disagreed with the queue's own coverage.
|
|
259
|
+
const marketDaysScanned = countMarketDays(markets);
|
|
127
260
|
|
|
128
261
|
const jobDir = await mkdtemp(path.join(tmpdir(), 'ot-run-'));
|
|
129
262
|
try {
|
|
@@ -158,30 +291,53 @@ export async function cmdRun({ dir, flags }) {
|
|
|
158
291
|
}
|
|
159
292
|
|
|
160
293
|
if (!flags.json) {
|
|
161
|
-
process.stdout.write(`\n ${
|
|
294
|
+
process.stdout.write(`\n ${marketDaysScanned} market-days · ${manifest.language} · local replay\n`);
|
|
162
295
|
}
|
|
163
296
|
|
|
164
297
|
const passes = [];
|
|
165
|
-
for
|
|
298
|
+
// ONE pass, at whatever delay the manifest asked for — the same shape the
|
|
299
|
+
// queue runs (runner/worker.mjs). These two have drifted eight times and
|
|
300
|
+
// every one was "we shared the decoder and nothing else".
|
|
301
|
+
const delayMs = manifest.latency ?? 0;
|
|
302
|
+
const steps = [{ label: delayMs ? `+${delayMs} ms` : '0 ms', ms: delayMs }];
|
|
303
|
+
for (const step of steps) {
|
|
166
304
|
const outputKey = randomBytes(32).toString('hex');
|
|
167
305
|
const res = await runHarness({
|
|
168
306
|
languageId, jobDir, outputKey, markets,
|
|
169
307
|
job: { ...baseJob, fillDelayMs: step.ms },
|
|
308
|
+
seriesRows: seriesRows.rowsByName ?? {}, seriesLags, seriesNames,
|
|
170
309
|
});
|
|
171
310
|
const out = demux(res.lines);
|
|
172
|
-
if (
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
311
|
+
if (res.forged > 0) {
|
|
312
|
+
// Almost always a dependency writing to stdout: the harness takes
|
|
313
|
+
// console away from the strategy before loading it, but a library that
|
|
314
|
+
// reaches the descriptor another way still lands on the channel.
|
|
315
|
+
const err = new Error(
|
|
316
|
+
`${res.forged} line(s) on the result channel did not authenticate.`
|
|
317
|
+
+ ' Something in this strategy or its dependencies writes to stdout;'
|
|
318
|
+
+ ' a queued run would count the same bytes as forged and could lose'
|
|
319
|
+
+ ' results. Use ctx.log() for output.',
|
|
320
|
+
);
|
|
321
|
+
err.code = 'E_RUNTIME';
|
|
322
|
+
err.detail = err.message;
|
|
323
|
+
throw err;
|
|
181
324
|
}
|
|
325
|
+
// UNCONDITIONAL. This used to be wrapped in `if (step.ms === 0)`, from
|
|
326
|
+
// when pass 0 was the report and the rest were a comparison curve whose
|
|
327
|
+
// failures were survivable. There is one pass now, and its delay is
|
|
328
|
+
// whatever the manifest asked for — so the guard silently stopped
|
|
329
|
+
// running the moment anyone wrote `latency: 250`, and a rejected or
|
|
330
|
+
// over-budget replay became a report: locally successful, marked
|
|
331
|
+
// `fill_delay_ms: 250`, and refused by the queue.
|
|
332
|
+
if (res.code === EXIT.rejected || res.code === EXIT.budget) {
|
|
333
|
+
const r = out.result.rejection ?? { code: 'E_RUNTIME', detail: res.stderr.slice(0, 2000) };
|
|
334
|
+
const err = new Error(r.detail);
|
|
335
|
+
err.code = r.code;
|
|
336
|
+
err.detail = r.detail;
|
|
337
|
+
throw err;
|
|
338
|
+
}
|
|
339
|
+
if (res.code !== EXIT.ok) throw new Error(res.stderr.slice(0, 2000) || `harness exited ${res.code}`);
|
|
182
340
|
passes.push({ delayMs: step.ms, ...out, ok: res.code === EXIT.ok });
|
|
183
|
-
// A strategy that never traded has no latency curve to draw.
|
|
184
|
-
if (step.ms === 0 && out.trades.length === 0) break;
|
|
185
341
|
}
|
|
186
342
|
|
|
187
343
|
const base = passes[0];
|
|
@@ -199,36 +355,49 @@ export async function cmdRun({ dir, flags }) {
|
|
|
199
355
|
runId: `local_${days[0]}`,
|
|
200
356
|
submittedAt: 0,
|
|
201
357
|
manifest,
|
|
358
|
+
// Local runs have no stored submission to hash, and saying null is
|
|
359
|
+
// truthful: this report is not tied to a submission at all.
|
|
360
|
+
sourceSha256: null,
|
|
202
361
|
scope: {
|
|
203
362
|
venue,
|
|
204
363
|
assets: assets.length ? assets : [...new Set(markets.map((m) => m.market.asset).filter(Boolean))],
|
|
205
364
|
from: days[0],
|
|
206
365
|
to: days[days.length - 1],
|
|
207
|
-
marketDays:
|
|
366
|
+
marketDays: marketDaysScanned,
|
|
208
367
|
archivedDayCount: days.length,
|
|
209
368
|
},
|
|
210
|
-
scanned: {
|
|
369
|
+
scanned: {
|
|
370
|
+
markets: base.result.marketsRun,
|
|
371
|
+
market_days: marketDaysScanned,
|
|
372
|
+
events: base.result.eventsSeen,
|
|
373
|
+
},
|
|
211
374
|
trades: base.trades,
|
|
212
375
|
fills: base.fills,
|
|
213
376
|
marketSummaries: [...marketMeta.values()],
|
|
214
377
|
marketMeta,
|
|
215
378
|
feesPaid: base.result.feesPaid,
|
|
216
|
-
|
|
217
|
-
delayMs: p.delayMs, netPnl: metrics(p.trades).net_pnl ?? 0,
|
|
218
|
-
})),
|
|
379
|
+
fillDelayMs: delayMs,
|
|
219
380
|
sweep: null,
|
|
220
381
|
crosschecks: base.result.crosschecks,
|
|
221
382
|
seed: Number(flags.seed ?? 1),
|
|
222
|
-
coverage: {
|
|
383
|
+
coverage: buildCoverage({
|
|
384
|
+
marketDaysScanned,
|
|
385
|
+
// NOT backfilled from `scanned`. Filling it in that way meant the
|
|
386
|
+
// headline always said "asked for N, scanned N" — so a day that was
|
|
387
|
+
// entirely unusable was recorded in `missing` and simultaneously denied
|
|
388
|
+
// at the top of the same file. A local run does not know what was asked
|
|
389
|
+
// for; null says that, and saying it is the point.
|
|
390
|
+
marketDaysRequested: null,
|
|
391
|
+
marketsReportedByRunner: base.result.marketsRun,
|
|
392
|
+
missing,
|
|
393
|
+
// Always empty here: a manifest that declares one is refused above,
|
|
394
|
+
// because a local replay cannot supply it.
|
|
395
|
+
referenceDeclared: [],
|
|
396
|
+
streams: countStreams([...marketMeta.values()]),
|
|
397
|
+
droppedRows: base.malformed ?? 0,
|
|
223
398
|
local: true,
|
|
224
399
|
source: path.resolve(dataRoot),
|
|
225
|
-
|
|
226
|
-
streams: [...marketMeta.values()].reduce((acc, m) => {
|
|
227
|
-
const k = m.stream ?? 'unknown';
|
|
228
|
-
acc[k] = (acc[k] ?? 0) + 1;
|
|
229
|
-
return acc;
|
|
230
|
-
}, {}),
|
|
231
|
-
},
|
|
400
|
+
}),
|
|
232
401
|
budget: base.result.budget,
|
|
233
402
|
});
|
|
234
403
|
|
package/cli/commands/submit.mjs
CHANGED
|
@@ -30,9 +30,37 @@ export async function cmdSubmit({ dir, flags }) {
|
|
|
30
30
|
|
|
31
31
|
// Locally first. The rejection codes are identical either way, so a customer
|
|
32
32
|
// who fixes what `ot check` said will not be told something different here.
|
|
33
|
-
await validate(files);
|
|
33
|
+
const checked = await validate(files);
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
// Series go straight to R2, the same way the web editor sends them.
|
|
36
|
+
//
|
|
37
|
+
// Not an optimisation for the CLI's sake — it is what makes "a series never
|
|
38
|
+
// passes through the API box" true rather than true-for-browsers. Inlining a
|
|
39
|
+
// 4MB CSV cost 69MB resident there for one submission, and leaving one client
|
|
40
|
+
// doing it would have kept the whole cost while claiming it was gone.
|
|
41
|
+
const seriesNames = new Set((checked.manifest.series ?? []).map((x) => x.file));
|
|
42
|
+
const uploads = {};
|
|
43
|
+
for (const f of files.filter((x) => seriesNames.has(x.name))) {
|
|
44
|
+
const bytes = Buffer.byteLength(f.content, 'utf8');
|
|
45
|
+
const sign = await post(api, '/v1/backtest/upload', { bytes }, key);
|
|
46
|
+
if (sign.status !== 200 || !sign.json?.url) {
|
|
47
|
+
throw new Error(`could not stage ${f.name}: ${sign.json?.error ?? sign.text}`);
|
|
48
|
+
}
|
|
49
|
+
// Exactly the headers the server signed. `if-none-match: *` is among them
|
|
50
|
+
// and makes the url write-once; omitting any of them is a 403.
|
|
51
|
+
const put = await fetch(sign.json.url, {
|
|
52
|
+
method: 'PUT', headers: sign.json.headers, body: f.content,
|
|
53
|
+
});
|
|
54
|
+
if (!put.ok) throw new Error(`could not upload ${f.name}: HTTP ${put.status}`);
|
|
55
|
+
uploads[f.name] = sign.json.key;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const body = {
|
|
59
|
+
...scope,
|
|
60
|
+
files: files.filter((x) => !seriesNames.has(x.name)),
|
|
61
|
+
...(Object.keys(uploads).length ? { uploads } : {}),
|
|
62
|
+
...(flags.email ? { email: flags.email } : {}),
|
|
63
|
+
};
|
|
36
64
|
const { status, json, text } = await post(api, '/v1/backtest/submit', body, key);
|
|
37
65
|
|
|
38
66
|
if (status === 202 && json?.run_id) {
|
|
@@ -51,7 +79,16 @@ export async function cmdSubmit({ dir, flags }) {
|
|
|
51
79
|
}
|
|
52
80
|
process.stdout.write(` cost ${json.credits_held} cr\n`);
|
|
53
81
|
process.stdout.write(` source sha256 ${String(json.source_sha256).slice(0, 16)}…\n\n`);
|
|
54
|
-
|
|
82
|
+
// DO NOT PROMISE THE EMAIL WHEN NONE WAS ASKED FOR. `--email` is what
|
|
83
|
+
// fills deliver_to, and without it the delivery poller correctly skips the
|
|
84
|
+
// run — so the old unconditional "(or wait for the email)" told every CLI
|
|
85
|
+
// submitter to wait for something that was never going to arrive. The flag
|
|
86
|
+
// was implemented and undocumented, which is the same failure from the
|
|
87
|
+
// other side: nobody could use the thing this line advertised.
|
|
88
|
+
process.stdout.write(flags.email
|
|
89
|
+
? ` ot status ${json.run_id} (or wait for the email)\n\n`
|
|
90
|
+
: ` ot status ${json.run_id}\n`
|
|
91
|
+
+ ' (no --email, so nothing will be sent — note that id down)\n\n');
|
|
55
92
|
return 0;
|
|
56
93
|
}
|
|
57
94
|
|