@tradejs/cli 1.0.12 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +96 -0
- package/README.md +9 -1
- package/dist/cli.js +143 -93
- package/dist/lib/aiTrainResearch.js +0 -1
- package/dist/lib/derivativesContextBackfill.js +127 -83
- package/dist/lib/marketContextPrepare.js +123 -83
- package/dist/scripts/aiTrain.js +0 -1
- package/dist/scripts/backtest.js +123 -83
- package/dist/scripts/derivativesIngest.js +11 -4
- package/dist/scripts/derivativesIngestCoinalyzeAll.js +8 -4
- package/dist/scripts/replay.js +123 -83
- package/dist/scripts/replayRunner.js +123 -83
- package/dist/scripts/runtimeParity.js +123 -83
- package/dist/scripts/signals.js +123 -83
- package/dist/scripts/signalsDaemon.js +123 -83
- package/package.json +11 -11
package/dist/scripts/backtest.js
CHANGED
|
@@ -2495,7 +2495,7 @@ var import_userSettings = require("@tradejs/infra/userSettings");
|
|
|
2495
2495
|
var HOUR_MS = 60 * 60 * 1e3;
|
|
2496
2496
|
var DAY_MS2 = 24 * HOUR_MS;
|
|
2497
2497
|
var DEFAULT_LOOKBACK_HOURS = 48;
|
|
2498
|
-
var
|
|
2498
|
+
var SOURCE_INTERVALS = ["15m"];
|
|
2499
2499
|
var coinalyzeIntervalMap = {
|
|
2500
2500
|
"15m": "15min",
|
|
2501
2501
|
"1h": "1hour"
|
|
@@ -2556,10 +2556,7 @@ var isDerivativesContextBackfillEnabled = (env) => {
|
|
|
2556
2556
|
var shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
|
|
2557
2557
|
var shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
|
|
2558
2558
|
var resolveDerivativesContextIntervals = () => {
|
|
2559
|
-
|
|
2560
|
-
process.env.DERIVATIVES_CONTEXT_INTERVALS
|
|
2561
|
-
);
|
|
2562
|
-
return intervals.length ? intervals : DEFAULT_INTERVALS;
|
|
2559
|
+
return [...SOURCE_INTERVALS];
|
|
2563
2560
|
};
|
|
2564
2561
|
var resolveDerivativesContextLookbackMs = () => {
|
|
2565
2562
|
const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
|
|
@@ -2594,9 +2591,10 @@ var resolveDerivativesContextBackfillWindow = (params) => {
|
|
|
2594
2591
|
};
|
|
2595
2592
|
var resolveDerivativesContextIntervalBackfillWindow = (params) => {
|
|
2596
2593
|
const intervalMs = derivativesIntervalMs(params.interval);
|
|
2594
|
+
const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs) * intervalMs;
|
|
2597
2595
|
return {
|
|
2598
2596
|
fromMs: Math.floor(params.fromMs / intervalMs) * intervalMs,
|
|
2599
|
-
toMs:
|
|
2597
|
+
toMs: alignedToMs,
|
|
2600
2598
|
intervalMs
|
|
2601
2599
|
};
|
|
2602
2600
|
};
|
|
@@ -2614,6 +2612,7 @@ var resolveDerivativesContextMissingFetchFromMs = (params) => {
|
|
|
2614
2612
|
}
|
|
2615
2613
|
return null;
|
|
2616
2614
|
};
|
|
2615
|
+
var resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
|
|
2617
2616
|
var countBackfillWindows = (params) => {
|
|
2618
2617
|
let count = 0;
|
|
2619
2618
|
let cursor = params.fromMs;
|
|
@@ -2677,6 +2676,14 @@ var groupDerivativesContextMissingFetchRanges = (ranges) => {
|
|
|
2677
2676
|
}
|
|
2678
2677
|
return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
|
|
2679
2678
|
};
|
|
2679
|
+
var getMissingClosedDerivativesSymbols = (params) => {
|
|
2680
|
+
const availableSymbols = new Set(
|
|
2681
|
+
params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
|
|
2682
|
+
);
|
|
2683
|
+
return params.symbols.filter(
|
|
2684
|
+
(symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
|
|
2685
|
+
);
|
|
2686
|
+
};
|
|
2680
2687
|
var coverageKey = (params) => [
|
|
2681
2688
|
params.symbol.trim().toUpperCase(),
|
|
2682
2689
|
params.interval,
|
|
@@ -2956,7 +2963,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
2956
2963
|
...resolveDerivativesContextIntervalBackfillWindow({
|
|
2957
2964
|
fromMs,
|
|
2958
2965
|
toMs: safeEndMs,
|
|
2959
|
-
interval: interval2
|
|
2966
|
+
interval: interval2,
|
|
2967
|
+
closedOnly: mode === "signals"
|
|
2960
2968
|
})
|
|
2961
2969
|
})).filter((item) => item.toMs > item.fromMs);
|
|
2962
2970
|
if (!intervalWindows.length) {
|
|
@@ -3015,7 +3023,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3015
3023
|
}),
|
|
3016
3024
|
0
|
|
3017
3025
|
);
|
|
3018
|
-
const allBackfillWindowsCached = intervalWindows.every((window2) => {
|
|
3026
|
+
const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window2) => {
|
|
3019
3027
|
const edgesBySymbol = edgesByInterval.get(window2.interval);
|
|
3020
3028
|
const coverageKeys = coverageKeysByInterval.get(window2.interval);
|
|
3021
3029
|
const backfillWindows = buildBackfillWindows({
|
|
@@ -3145,7 +3153,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3145
3153
|
return null;
|
|
3146
3154
|
}
|
|
3147
3155
|
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3148
|
-
const coverageFromMs = resolveDerivativesContextMissingCoverageFetchFromMs({
|
|
3156
|
+
const coverageFromMs = mode === "signals" ? cursor : resolveDerivativesContextMissingCoverageFetchFromMs({
|
|
3149
3157
|
ranges: coverageRangesBySymbol.get(normalizedSymbol),
|
|
3150
3158
|
fromMs: cursor,
|
|
3151
3159
|
toMs,
|
|
@@ -3155,16 +3163,20 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3155
3163
|
return null;
|
|
3156
3164
|
}
|
|
3157
3165
|
const edges = edgesBySymbol.get(normalizedSymbol);
|
|
3158
|
-
const edgesFromMs =
|
|
3166
|
+
const edgesFromMs = resolveDerivativesContextFetchFromMs({
|
|
3159
3167
|
edges,
|
|
3160
3168
|
fromMs: cursor,
|
|
3161
3169
|
toMs,
|
|
3162
|
-
intervalMs
|
|
3170
|
+
intervalMs,
|
|
3171
|
+
refreshClosedTail: mode === "signals"
|
|
3163
3172
|
});
|
|
3164
3173
|
if (edgesFromMs == null) {
|
|
3165
3174
|
return null;
|
|
3166
3175
|
}
|
|
3167
|
-
return {
|
|
3176
|
+
return {
|
|
3177
|
+
item,
|
|
3178
|
+
fromMs: Math.max(coverageFromMs, edgesFromMs)
|
|
3179
|
+
};
|
|
3168
3180
|
}).filter(
|
|
3169
3181
|
(item) => item != null
|
|
3170
3182
|
);
|
|
@@ -3178,85 +3190,113 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3178
3190
|
const marketSymbols = missingBatch.map(
|
|
3179
3191
|
(item) => item.marketSymbol
|
|
3180
3192
|
);
|
|
3181
|
-
const
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
endpoint: liqPath,
|
|
3201
|
-
metric: "liq",
|
|
3202
|
-
marketSymbols,
|
|
3203
|
-
apiKey,
|
|
3204
|
-
interval: interval2,
|
|
3205
|
-
fromMs: group.fromMs,
|
|
3206
|
-
toMs
|
|
3207
|
-
});
|
|
3208
|
-
const rows = missingBatch.flatMap((item) => {
|
|
3209
|
-
const marketSymbol = item.marketSymbol.toUpperCase();
|
|
3210
|
-
const points = (0, import_indicators.mergeCoinalyzeMetrics)({
|
|
3211
|
-
symbol: item.symbol,
|
|
3212
|
-
oiRaw: oiMap.get(marketSymbol) ?? [],
|
|
3213
|
-
fundingRaw: fundingMap.get(marketSymbol) ?? [],
|
|
3214
|
-
liqRaw: liqMap.get(marketSymbol) ?? []
|
|
3193
|
+
const maxClosedBarAttempts = mode === "signals" ? asInt2(
|
|
3194
|
+
process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
|
|
3195
|
+
3
|
|
3196
|
+
) : 1;
|
|
3197
|
+
const closedBarRetryDelayMs = asInt2(
|
|
3198
|
+
process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
|
|
3199
|
+
2e3
|
|
3200
|
+
);
|
|
3201
|
+
let rows = [];
|
|
3202
|
+
let missingClosedSymbols = [];
|
|
3203
|
+
for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
|
|
3204
|
+
const oiMap = await fetchMetricBatch({
|
|
3205
|
+
endpoint: oiPath,
|
|
3206
|
+
metric: "oi",
|
|
3207
|
+
marketSymbols,
|
|
3208
|
+
apiKey,
|
|
3209
|
+
interval: interval2,
|
|
3210
|
+
fromMs: group.fromMs,
|
|
3211
|
+
toMs
|
|
3215
3212
|
});
|
|
3216
|
-
|
|
3217
|
-
|
|
3213
|
+
const fundingMap = await fetchMetricBatch({
|
|
3214
|
+
endpoint: fundingPath,
|
|
3215
|
+
metric: "funding",
|
|
3216
|
+
marketSymbols,
|
|
3217
|
+
apiKey,
|
|
3218
|
+
interval: interval2,
|
|
3219
|
+
fromMs: group.fromMs,
|
|
3220
|
+
toMs
|
|
3221
|
+
});
|
|
3222
|
+
const liqMap = await fetchMetricBatch({
|
|
3223
|
+
endpoint: liqPath,
|
|
3224
|
+
metric: "liq",
|
|
3225
|
+
marketSymbols,
|
|
3226
|
+
apiKey,
|
|
3227
|
+
interval: interval2,
|
|
3228
|
+
fromMs: group.fromMs,
|
|
3229
|
+
toMs
|
|
3230
|
+
});
|
|
3231
|
+
rows = missingBatch.flatMap((item) => {
|
|
3232
|
+
const marketSymbol = item.marketSymbol.toUpperCase();
|
|
3233
|
+
const points = (0, import_indicators.mergeCoinalyzeMetrics)({
|
|
3234
|
+
symbol: item.symbol,
|
|
3235
|
+
oiRaw: oiMap.get(marketSymbol) ?? [],
|
|
3236
|
+
fundingRaw: fundingMap.get(marketSymbol) ?? [],
|
|
3237
|
+
liqRaw: liqMap.get(marketSymbol) ?? []
|
|
3238
|
+
});
|
|
3239
|
+
return (0, import_indicators.coinalyzePointsToRows)(points, interval2, "coinalyze");
|
|
3240
|
+
});
|
|
3241
|
+
missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
|
|
3242
|
+
symbols: missingBatch.map((item) => item.symbol),
|
|
3243
|
+
rows,
|
|
3244
|
+
expectedTimestamp: toMs
|
|
3245
|
+
}) : [];
|
|
3246
|
+
if (!missingClosedSymbols.length) break;
|
|
3247
|
+
if (attempt < maxClosedBarAttempts) {
|
|
3248
|
+
await (0, import_async.delay)(closedBarRetryDelayMs);
|
|
3249
|
+
}
|
|
3250
|
+
}
|
|
3251
|
+
if (missingClosedSymbols.length) {
|
|
3252
|
+
throw new Error(
|
|
3253
|
+
`Coinalyze closed ${interval2} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
|
|
3254
|
+
);
|
|
3255
|
+
}
|
|
3218
3256
|
if (rows.length) {
|
|
3219
3257
|
await (0, import_timescale2.upsertDerivatives)(rows);
|
|
3220
3258
|
totalRows += rows.length;
|
|
3221
3259
|
}
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
const
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
(rowsCountBySymbol.get(symbol) ?? 0) + 1
|
|
3228
|
-
);
|
|
3229
|
-
}
|
|
3230
|
-
const coverageRows = missingBatch.map((item) => {
|
|
3231
|
-
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3232
|
-
const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
|
|
3233
|
-
return {
|
|
3234
|
-
source: "coinalyze",
|
|
3235
|
-
symbol: item.symbol,
|
|
3236
|
-
interval: interval2,
|
|
3237
|
-
fromMs: cursor,
|
|
3238
|
-
toMs,
|
|
3239
|
-
rowsCount
|
|
3240
|
-
};
|
|
3241
|
-
});
|
|
3242
|
-
await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
|
|
3243
|
-
for (const coverageRow of coverageRows) {
|
|
3244
|
-
const symbol = coverageRow.symbol.toUpperCase();
|
|
3245
|
-
edgesBySymbol.set(
|
|
3246
|
-
symbol,
|
|
3247
|
-
extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
|
|
3248
|
-
);
|
|
3249
|
-
coverageKeys.add(
|
|
3250
|
-
coverageKey({
|
|
3260
|
+
if (mode === "backtest") {
|
|
3261
|
+
const rowsCountBySymbol = /* @__PURE__ */ new Map();
|
|
3262
|
+
for (const row of rows) {
|
|
3263
|
+
const symbol = row.symbol.toUpperCase();
|
|
3264
|
+
rowsCountBySymbol.set(
|
|
3251
3265
|
symbol,
|
|
3266
|
+
(rowsCountBySymbol.get(symbol) ?? 0) + 1
|
|
3267
|
+
);
|
|
3268
|
+
}
|
|
3269
|
+
const coverageRows = missingBatch.map((item) => {
|
|
3270
|
+
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3271
|
+
const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
|
|
3272
|
+
return {
|
|
3273
|
+
source: "coinalyze",
|
|
3274
|
+
symbol: item.symbol,
|
|
3252
3275
|
interval: interval2,
|
|
3253
3276
|
fromMs: cursor,
|
|
3254
|
-
toMs
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3277
|
+
toMs,
|
|
3278
|
+
rowsCount
|
|
3279
|
+
};
|
|
3280
|
+
});
|
|
3281
|
+
await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
|
|
3282
|
+
for (const coverageRow of coverageRows) {
|
|
3283
|
+
const symbol = coverageRow.symbol.toUpperCase();
|
|
3284
|
+
edgesBySymbol.set(
|
|
3285
|
+
symbol,
|
|
3286
|
+
extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
|
|
3287
|
+
);
|
|
3288
|
+
coverageKeys.add(
|
|
3289
|
+
coverageKey({
|
|
3290
|
+
symbol,
|
|
3291
|
+
interval: interval2,
|
|
3292
|
+
fromMs: cursor,
|
|
3293
|
+
toMs
|
|
3294
|
+
})
|
|
3295
|
+
);
|
|
3296
|
+
const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
|
|
3297
|
+
coverageRanges.push({ fromMs: cursor, toMs });
|
|
3298
|
+
coverageRangesBySymbol.set(symbol, coverageRanges);
|
|
3299
|
+
}
|
|
3260
3300
|
}
|
|
3261
3301
|
}
|
|
3262
3302
|
}
|
|
@@ -41,11 +41,11 @@ var import_timescale = require("@tradejs/infra/timescale");
|
|
|
41
41
|
var import_userSettings = require("@tradejs/infra/userSettings");
|
|
42
42
|
var import_connectors = require("@tradejs/connectors");
|
|
43
43
|
import_args.default.example(
|
|
44
|
-
"yarn ts-node ./src/scripts/derivativesIngest --provider coinalyze --symbols BTCUSDT,ETHUSDT --intervals 15m
|
|
44
|
+
"yarn ts-node ./src/scripts/derivativesIngest --provider coinalyze --symbols BTCUSDT,ETHUSDT --intervals 15m --days 120",
|
|
45
45
|
"Ingest market features (derivatives/spread) into Timescale by provider"
|
|
46
46
|
);
|
|
47
47
|
import_args.default.option(["s", "symbols"], "Comma-separated symbols", "BTCUSDT,ETHUSDT");
|
|
48
|
-
import_args.default.option(["t", "intervals"], "Comma-separated intervals: 15m,1h", "15m
|
|
48
|
+
import_args.default.option(["t", "intervals"], "Comma-separated intervals: 15m,1h", "15m");
|
|
49
49
|
import_args.default.option(["d", "days"], "Lookback in days", 120);
|
|
50
50
|
import_args.default.option(
|
|
51
51
|
["p", "provider"],
|
|
@@ -82,9 +82,16 @@ var main = async () => {
|
|
|
82
82
|
let totalSpreadRows = 0;
|
|
83
83
|
for (const symbol of symbols) {
|
|
84
84
|
for (const interval of intervals) {
|
|
85
|
+
const lastClosedStartMs = (0, import_indicators.getLastClosedDerivativesBarStartMs)(
|
|
86
|
+
now,
|
|
87
|
+
interval
|
|
88
|
+
);
|
|
85
89
|
let cursor = fromMs;
|
|
86
|
-
while (cursor <
|
|
87
|
-
const toMs = Math.min(
|
|
90
|
+
while (cursor < lastClosedStartMs) {
|
|
91
|
+
const toMs = Math.min(
|
|
92
|
+
lastClosedStartMs,
|
|
93
|
+
cursor + batchDays * 24 * 60 * 60 * 1e3
|
|
94
|
+
);
|
|
88
95
|
process.stdout.write(
|
|
89
96
|
`\r${import_chalk.default.cyan(providerName)} ${import_chalk.default.yellow(symbol)} ${interval} ${new Date(cursor).toISOString()} .. ${new Date(toMs).toISOString()} `
|
|
90
97
|
);
|
|
@@ -49,7 +49,7 @@ var coinalyzeIntervalMap = {
|
|
|
49
49
|
"1h": "1hour"
|
|
50
50
|
};
|
|
51
51
|
import_args.default.example(
|
|
52
|
-
"yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m
|
|
52
|
+
"yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m",
|
|
53
53
|
"Fetch derivatives for all getTickers symbols matched to Coinalyze markets"
|
|
54
54
|
);
|
|
55
55
|
import_args.default.option(["U", "user"], "User settings profile name from Redis", "root");
|
|
@@ -57,7 +57,7 @@ import_args.default.option(["t", "tickers"], "Comma-separated include symbols");
|
|
|
57
57
|
import_args.default.option(["e", "exclude"], "Comma-separated exclude symbols");
|
|
58
58
|
import_args.default.option(["l", "tickersLimit"], "Tickers limit");
|
|
59
59
|
import_args.default.option(["c", "chunk"], "Chunk selector, e.g. 1/4");
|
|
60
|
-
import_args.default.option(["i", "intervals"], "Intervals: 15m,1h", "15m
|
|
60
|
+
import_args.default.option(["i", "intervals"], "Intervals: 15m,1h", "15m");
|
|
61
61
|
import_args.default.option(["d", "days"], "Lookback in days", 120);
|
|
62
62
|
import_args.default.option(["b", "batchDays"], "Request chunk size in days", 120);
|
|
63
63
|
import_args.default.option(
|
|
@@ -326,12 +326,16 @@ var main = async () => {
|
|
|
326
326
|
let totalRows = 0;
|
|
327
327
|
let failedWindows = 0;
|
|
328
328
|
for (const interval of intervals) {
|
|
329
|
+
const lastClosedStartMs = (0, import_indicators.getLastClosedDerivativesBarStartMs)(now, interval);
|
|
329
330
|
for (let batchIdx = 0; batchIdx < symbolBatches.length; batchIdx += 1) {
|
|
330
331
|
const batch = symbolBatches[batchIdx];
|
|
331
332
|
const marketSymbols = batch.map((item) => item.marketSymbol);
|
|
332
333
|
let cursor = fromMs;
|
|
333
|
-
while (cursor <
|
|
334
|
-
const toMs = Math.min(
|
|
334
|
+
while (cursor < lastClosedStartMs) {
|
|
335
|
+
const toMs = Math.min(
|
|
336
|
+
lastClosedStartMs,
|
|
337
|
+
cursor + batchDays * 24 * 60 * 60 * 1e3
|
|
338
|
+
);
|
|
335
339
|
try {
|
|
336
340
|
const oiMap = await fetchMetricBatch({
|
|
337
341
|
endpoint: oiPath,
|
package/dist/scripts/replay.js
CHANGED
|
@@ -3106,7 +3106,7 @@ var import_userSettings = require("@tradejs/infra/userSettings");
|
|
|
3106
3106
|
var HOUR_MS = 60 * 60 * 1e3;
|
|
3107
3107
|
var DAY_MS2 = 24 * HOUR_MS;
|
|
3108
3108
|
var DEFAULT_LOOKBACK_HOURS = 48;
|
|
3109
|
-
var
|
|
3109
|
+
var SOURCE_INTERVALS = ["15m"];
|
|
3110
3110
|
var coinalyzeIntervalMap = {
|
|
3111
3111
|
"15m": "15min",
|
|
3112
3112
|
"1h": "1hour"
|
|
@@ -3167,10 +3167,7 @@ var isDerivativesContextBackfillEnabled = (env) => {
|
|
|
3167
3167
|
var shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
|
|
3168
3168
|
var shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
|
|
3169
3169
|
var resolveDerivativesContextIntervals = () => {
|
|
3170
|
-
|
|
3171
|
-
process.env.DERIVATIVES_CONTEXT_INTERVALS
|
|
3172
|
-
);
|
|
3173
|
-
return intervals.length ? intervals : DEFAULT_INTERVALS;
|
|
3170
|
+
return [...SOURCE_INTERVALS];
|
|
3174
3171
|
};
|
|
3175
3172
|
var resolveDerivativesContextLookbackMs = () => {
|
|
3176
3173
|
const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
|
|
@@ -3205,9 +3202,10 @@ var resolveDerivativesContextBackfillWindow = (params) => {
|
|
|
3205
3202
|
};
|
|
3206
3203
|
var resolveDerivativesContextIntervalBackfillWindow = (params) => {
|
|
3207
3204
|
const intervalMs = derivativesIntervalMs(params.interval);
|
|
3205
|
+
const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs) * intervalMs;
|
|
3208
3206
|
return {
|
|
3209
3207
|
fromMs: Math.floor(params.fromMs / intervalMs) * intervalMs,
|
|
3210
|
-
toMs:
|
|
3208
|
+
toMs: alignedToMs,
|
|
3211
3209
|
intervalMs
|
|
3212
3210
|
};
|
|
3213
3211
|
};
|
|
@@ -3225,6 +3223,7 @@ var resolveDerivativesContextMissingFetchFromMs = (params) => {
|
|
|
3225
3223
|
}
|
|
3226
3224
|
return null;
|
|
3227
3225
|
};
|
|
3226
|
+
var resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
|
|
3228
3227
|
var countBackfillWindows = (params) => {
|
|
3229
3228
|
let count = 0;
|
|
3230
3229
|
let cursor = params.fromMs;
|
|
@@ -3288,6 +3287,14 @@ var groupDerivativesContextMissingFetchRanges = (ranges) => {
|
|
|
3288
3287
|
}
|
|
3289
3288
|
return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
|
|
3290
3289
|
};
|
|
3290
|
+
var getMissingClosedDerivativesSymbols = (params) => {
|
|
3291
|
+
const availableSymbols = new Set(
|
|
3292
|
+
params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
|
|
3293
|
+
);
|
|
3294
|
+
return params.symbols.filter(
|
|
3295
|
+
(symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
|
|
3296
|
+
);
|
|
3297
|
+
};
|
|
3291
3298
|
var coverageKey = (params) => [
|
|
3292
3299
|
params.symbol.trim().toUpperCase(),
|
|
3293
3300
|
params.interval,
|
|
@@ -3567,7 +3574,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3567
3574
|
...resolveDerivativesContextIntervalBackfillWindow({
|
|
3568
3575
|
fromMs,
|
|
3569
3576
|
toMs: safeEndMs,
|
|
3570
|
-
interval
|
|
3577
|
+
interval,
|
|
3578
|
+
closedOnly: mode === "signals"
|
|
3571
3579
|
})
|
|
3572
3580
|
})).filter((item) => item.toMs > item.fromMs);
|
|
3573
3581
|
if (!intervalWindows.length) {
|
|
@@ -3626,7 +3634,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3626
3634
|
}),
|
|
3627
3635
|
0
|
|
3628
3636
|
);
|
|
3629
|
-
const allBackfillWindowsCached = intervalWindows.every((window2) => {
|
|
3637
|
+
const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window2) => {
|
|
3630
3638
|
const edgesBySymbol = edgesByInterval.get(window2.interval);
|
|
3631
3639
|
const coverageKeys = coverageKeysByInterval.get(window2.interval);
|
|
3632
3640
|
const backfillWindows = buildBackfillWindows({
|
|
@@ -3756,7 +3764,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3756
3764
|
return null;
|
|
3757
3765
|
}
|
|
3758
3766
|
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3759
|
-
const coverageFromMs = resolveDerivativesContextMissingCoverageFetchFromMs({
|
|
3767
|
+
const coverageFromMs = mode === "signals" ? cursor : resolveDerivativesContextMissingCoverageFetchFromMs({
|
|
3760
3768
|
ranges: coverageRangesBySymbol.get(normalizedSymbol),
|
|
3761
3769
|
fromMs: cursor,
|
|
3762
3770
|
toMs,
|
|
@@ -3766,16 +3774,20 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3766
3774
|
return null;
|
|
3767
3775
|
}
|
|
3768
3776
|
const edges = edgesBySymbol.get(normalizedSymbol);
|
|
3769
|
-
const edgesFromMs =
|
|
3777
|
+
const edgesFromMs = resolveDerivativesContextFetchFromMs({
|
|
3770
3778
|
edges,
|
|
3771
3779
|
fromMs: cursor,
|
|
3772
3780
|
toMs,
|
|
3773
|
-
intervalMs
|
|
3781
|
+
intervalMs,
|
|
3782
|
+
refreshClosedTail: mode === "signals"
|
|
3774
3783
|
});
|
|
3775
3784
|
if (edgesFromMs == null) {
|
|
3776
3785
|
return null;
|
|
3777
3786
|
}
|
|
3778
|
-
return {
|
|
3787
|
+
return {
|
|
3788
|
+
item,
|
|
3789
|
+
fromMs: Math.max(coverageFromMs, edgesFromMs)
|
|
3790
|
+
};
|
|
3779
3791
|
}).filter(
|
|
3780
3792
|
(item) => item != null
|
|
3781
3793
|
);
|
|
@@ -3789,85 +3801,113 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3789
3801
|
const marketSymbols = missingBatch.map(
|
|
3790
3802
|
(item) => item.marketSymbol
|
|
3791
3803
|
);
|
|
3792
|
-
const
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
const marketSymbol = item.marketSymbol.toUpperCase();
|
|
3821
|
-
const points = (0, import_indicators.mergeCoinalyzeMetrics)({
|
|
3822
|
-
symbol: item.symbol,
|
|
3823
|
-
oiRaw: oiMap.get(marketSymbol) ?? [],
|
|
3824
|
-
fundingRaw: fundingMap.get(marketSymbol) ?? [],
|
|
3825
|
-
liqRaw: liqMap.get(marketSymbol) ?? []
|
|
3804
|
+
const maxClosedBarAttempts = mode === "signals" ? asInt2(
|
|
3805
|
+
process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
|
|
3806
|
+
3
|
|
3807
|
+
) : 1;
|
|
3808
|
+
const closedBarRetryDelayMs = asInt2(
|
|
3809
|
+
process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
|
|
3810
|
+
2e3
|
|
3811
|
+
);
|
|
3812
|
+
let rows = [];
|
|
3813
|
+
let missingClosedSymbols = [];
|
|
3814
|
+
for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
|
|
3815
|
+
const oiMap = await fetchMetricBatch({
|
|
3816
|
+
endpoint: oiPath,
|
|
3817
|
+
metric: "oi",
|
|
3818
|
+
marketSymbols,
|
|
3819
|
+
apiKey,
|
|
3820
|
+
interval,
|
|
3821
|
+
fromMs: group.fromMs,
|
|
3822
|
+
toMs
|
|
3823
|
+
});
|
|
3824
|
+
const fundingMap = await fetchMetricBatch({
|
|
3825
|
+
endpoint: fundingPath,
|
|
3826
|
+
metric: "funding",
|
|
3827
|
+
marketSymbols,
|
|
3828
|
+
apiKey,
|
|
3829
|
+
interval,
|
|
3830
|
+
fromMs: group.fromMs,
|
|
3831
|
+
toMs
|
|
3826
3832
|
});
|
|
3827
|
-
|
|
3828
|
-
|
|
3833
|
+
const liqMap = await fetchMetricBatch({
|
|
3834
|
+
endpoint: liqPath,
|
|
3835
|
+
metric: "liq",
|
|
3836
|
+
marketSymbols,
|
|
3837
|
+
apiKey,
|
|
3838
|
+
interval,
|
|
3839
|
+
fromMs: group.fromMs,
|
|
3840
|
+
toMs
|
|
3841
|
+
});
|
|
3842
|
+
rows = missingBatch.flatMap((item) => {
|
|
3843
|
+
const marketSymbol = item.marketSymbol.toUpperCase();
|
|
3844
|
+
const points = (0, import_indicators.mergeCoinalyzeMetrics)({
|
|
3845
|
+
symbol: item.symbol,
|
|
3846
|
+
oiRaw: oiMap.get(marketSymbol) ?? [],
|
|
3847
|
+
fundingRaw: fundingMap.get(marketSymbol) ?? [],
|
|
3848
|
+
liqRaw: liqMap.get(marketSymbol) ?? []
|
|
3849
|
+
});
|
|
3850
|
+
return (0, import_indicators.coinalyzePointsToRows)(points, interval, "coinalyze");
|
|
3851
|
+
});
|
|
3852
|
+
missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
|
|
3853
|
+
symbols: missingBatch.map((item) => item.symbol),
|
|
3854
|
+
rows,
|
|
3855
|
+
expectedTimestamp: toMs
|
|
3856
|
+
}) : [];
|
|
3857
|
+
if (!missingClosedSymbols.length) break;
|
|
3858
|
+
if (attempt < maxClosedBarAttempts) {
|
|
3859
|
+
await (0, import_async.delay)(closedBarRetryDelayMs);
|
|
3860
|
+
}
|
|
3861
|
+
}
|
|
3862
|
+
if (missingClosedSymbols.length) {
|
|
3863
|
+
throw new Error(
|
|
3864
|
+
`Coinalyze closed ${interval} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
|
|
3865
|
+
);
|
|
3866
|
+
}
|
|
3829
3867
|
if (rows.length) {
|
|
3830
3868
|
await (0, import_timescale2.upsertDerivatives)(rows);
|
|
3831
3869
|
totalRows += rows.length;
|
|
3832
3870
|
}
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
const
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
(rowsCountBySymbol.get(symbol) ?? 0) + 1
|
|
3839
|
-
);
|
|
3840
|
-
}
|
|
3841
|
-
const coverageRows = missingBatch.map((item) => {
|
|
3842
|
-
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3843
|
-
const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
|
|
3844
|
-
return {
|
|
3845
|
-
source: "coinalyze",
|
|
3846
|
-
symbol: item.symbol,
|
|
3847
|
-
interval,
|
|
3848
|
-
fromMs: cursor,
|
|
3849
|
-
toMs,
|
|
3850
|
-
rowsCount
|
|
3851
|
-
};
|
|
3852
|
-
});
|
|
3853
|
-
await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
|
|
3854
|
-
for (const coverageRow of coverageRows) {
|
|
3855
|
-
const symbol = coverageRow.symbol.toUpperCase();
|
|
3856
|
-
edgesBySymbol.set(
|
|
3857
|
-
symbol,
|
|
3858
|
-
extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
|
|
3859
|
-
);
|
|
3860
|
-
coverageKeys.add(
|
|
3861
|
-
coverageKey({
|
|
3871
|
+
if (mode === "backtest") {
|
|
3872
|
+
const rowsCountBySymbol = /* @__PURE__ */ new Map();
|
|
3873
|
+
for (const row of rows) {
|
|
3874
|
+
const symbol = row.symbol.toUpperCase();
|
|
3875
|
+
rowsCountBySymbol.set(
|
|
3862
3876
|
symbol,
|
|
3877
|
+
(rowsCountBySymbol.get(symbol) ?? 0) + 1
|
|
3878
|
+
);
|
|
3879
|
+
}
|
|
3880
|
+
const coverageRows = missingBatch.map((item) => {
|
|
3881
|
+
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3882
|
+
const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
|
|
3883
|
+
return {
|
|
3884
|
+
source: "coinalyze",
|
|
3885
|
+
symbol: item.symbol,
|
|
3863
3886
|
interval,
|
|
3864
3887
|
fromMs: cursor,
|
|
3865
|
-
toMs
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3888
|
+
toMs,
|
|
3889
|
+
rowsCount
|
|
3890
|
+
};
|
|
3891
|
+
});
|
|
3892
|
+
await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
|
|
3893
|
+
for (const coverageRow of coverageRows) {
|
|
3894
|
+
const symbol = coverageRow.symbol.toUpperCase();
|
|
3895
|
+
edgesBySymbol.set(
|
|
3896
|
+
symbol,
|
|
3897
|
+
extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
|
|
3898
|
+
);
|
|
3899
|
+
coverageKeys.add(
|
|
3900
|
+
coverageKey({
|
|
3901
|
+
symbol,
|
|
3902
|
+
interval,
|
|
3903
|
+
fromMs: cursor,
|
|
3904
|
+
toMs
|
|
3905
|
+
})
|
|
3906
|
+
);
|
|
3907
|
+
const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
|
|
3908
|
+
coverageRanges.push({ fromMs: cursor, toMs });
|
|
3909
|
+
coverageRangesBySymbol.set(symbol, coverageRanges);
|
|
3910
|
+
}
|
|
3871
3911
|
}
|
|
3872
3912
|
}
|
|
3873
3913
|
}
|