@tradejs/cli 3.1.29-beta.292 → 3.1.29-beta.297

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1731 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/scripts/coverageBackfill.ts
31
+ var coverageBackfill_exports = {};
32
+ __export(coverageBackfill_exports, {
33
+ main: () => main
34
+ });
35
+ module.exports = __toCommonJS(coverageBackfill_exports);
36
+ var import_connectors = require("@tradejs/connectors");
37
+ var import_candles = require("@tradejs/infra/timescale/candles");
38
+ var import_client3 = require("@tradejs/infra/timescale/client");
39
+ var import_dataCoverage = require("@tradejs/infra/timescale/dataCoverage");
40
+ var import_derivatives2 = require("@tradejs/infra/timescale/derivatives");
41
+ var import_marketContext2 = require("@tradejs/infra/timescale/marketContext");
42
+ var import_spread = require("@tradejs/infra/timescale/spread");
43
+ var import_strategies = require("@tradejs/node/strategies");
44
+
45
+ // src/lib/binanceMarketData.ts
46
+ var MARKET_FEATURE_INTERVAL_MS = {
47
+ "1m": 6e4,
48
+ "5m": 3e5,
49
+ "15m": 9e5,
50
+ "1h": 36e5
51
+ };
52
+ var safeDivide = (num, den) => Number.isFinite(num) && Number.isFinite(den) && den !== 0 ? num / den : null;
53
+ var mean = (values) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
54
+ var standardDeviation = (values) => {
55
+ const avg = mean(values);
56
+ if (avg == null || values.length < 2) return null;
57
+ const variance = values.reduce((sum, value) => sum + (value - avg) ** 2, 0) / values.length;
58
+ return Math.sqrt(variance);
59
+ };
60
+ var buildPrefixSum = (values) => {
61
+ const prefix = [0];
62
+ for (const value of values) prefix.push(prefix[prefix.length - 1] + value);
63
+ return prefix;
64
+ };
65
+ var sumPrefixRange = (prefix, startIndex, endIndexInclusive) => {
66
+ if (endIndexInclusive < startIndex) return 0;
67
+ const start = Math.max(0, startIndex);
68
+ const end = Math.min(prefix.length - 2, endIndexInclusive);
69
+ if (end < start) return 0;
70
+ return prefix[end + 1] - prefix[start];
71
+ };
72
+ var windowReturn = (candles, index, lookbackBars) => {
73
+ const previous = candles[index - lookbackBars];
74
+ const current = candles[index];
75
+ return previous && previous.close > 0 ? (current.close - previous.close) / previous.close : null;
76
+ };
77
+ var classifyBtcAltRegime = ({
78
+ btcReturn24h,
79
+ altBasketReturn24h,
80
+ btcVsAltReturn24h
81
+ }) => {
82
+ if (btcReturn24h == null || altBasketReturn24h == null || btcVsAltReturn24h == null) {
83
+ return "unknown";
84
+ }
85
+ if (btcReturn24h < -0.015 && altBasketReturn24h < -0.025) {
86
+ return "risk_off";
87
+ }
88
+ if (btcReturn24h > 5e-3 && altBasketReturn24h > 5e-3 && altBasketReturn24h > btcReturn24h) {
89
+ return "risk_on";
90
+ }
91
+ if (btcVsAltReturn24h > 5e-3) return "btc_lead";
92
+ if (btcVsAltReturn24h < -5e-3) return "alt_lead";
93
+ return "neutral";
94
+ };
95
+ var buildKlineTradeFlowRows = ({
96
+ symbol,
97
+ interval,
98
+ candles,
99
+ source = "binance_klines"
100
+ }) => candles.map((candle) => {
101
+ const buyBaseVolume = candle.takerBuyBaseVolume;
102
+ const buyQuoteVolume = candle.takerBuyQuoteVolume;
103
+ if (buyBaseVolume == null || buyQuoteVolume == null || !Number.isFinite(buyBaseVolume) || !Number.isFinite(buyQuoteVolume)) {
104
+ return null;
105
+ }
106
+ const sellBaseVolume = candle.takerSellBaseVolume != null && Number.isFinite(candle.takerSellBaseVolume) ? candle.takerSellBaseVolume : Math.max(0, candle.volume - buyBaseVolume);
107
+ const sellQuoteVolume = candle.takerSellQuoteVolume != null && Number.isFinite(candle.takerSellQuoteVolume) ? candle.takerSellQuoteVolume : Math.max(0, candle.turnover - buyQuoteVolume);
108
+ const trades = candle.trades != null && Number.isFinite(candle.trades) ? Math.max(0, Math.trunc(candle.trades)) : 0;
109
+ return {
110
+ symbol,
111
+ interval,
112
+ ts: new Date(candle.timestamp),
113
+ trades,
114
+ buyBaseVolume,
115
+ sellBaseVolume,
116
+ buyQuoteVolume,
117
+ sellQuoteVolume,
118
+ netBaseDelta: buyBaseVolume - sellBaseVolume,
119
+ netQuoteDelta: buyQuoteVolume - sellQuoteVolume,
120
+ buyPressurePct: safeDivide(
121
+ buyBaseVolume,
122
+ buyBaseVolume + sellBaseVolume
123
+ ),
124
+ source
125
+ };
126
+ }).filter((row) => row != null);
127
+ var buildMarketBreadthRows = ({
128
+ universe,
129
+ interval,
130
+ candlesBySymbol,
131
+ btcCandles,
132
+ source = "binance_klines"
133
+ }) => {
134
+ const timestamps = /* @__PURE__ */ new Set();
135
+ const indexedSymbols = Object.values(candlesBySymbol).map((candles) => ({
136
+ candles,
137
+ byTimestamp: new Map(
138
+ candles.map((candle, index) => [candle.timestamp, index])
139
+ ),
140
+ turnoverPrefix: buildPrefixSum(
141
+ candles.map((candle) => Math.max(0, candle.turnover ?? 0))
142
+ ),
143
+ closePrefix: buildPrefixSum(candles.map((candle) => candle.close))
144
+ }));
145
+ const btcIndexed = btcCandles ? {
146
+ candles: btcCandles,
147
+ byTimestamp: new Map(
148
+ btcCandles.map((candle, index) => [candle.timestamp, index])
149
+ ),
150
+ turnoverPrefix: buildPrefixSum(
151
+ btcCandles.map((candle) => Math.max(0, candle.turnover ?? 0))
152
+ )
153
+ } : null;
154
+ const intervalMs = MARKET_FEATURE_INTERVAL_MS[interval];
155
+ const bars1h = Math.max(1, Math.round(36e5 / intervalMs));
156
+ const bars4h = Math.max(1, Math.round(144e5 / intervalMs));
157
+ const bars24h = Math.max(1, Math.round(864e5 / intervalMs));
158
+ for (const candles of Object.values(candlesBySymbol)) {
159
+ for (const candle of candles) timestamps.add(candle.timestamp);
160
+ }
161
+ return [...timestamps].sort((a, b) => a - b).map((timestamp) => {
162
+ const returns = [];
163
+ const weightedReturns = [];
164
+ let advancers = 0;
165
+ let decliners = 0;
166
+ let unchanged = 0;
167
+ let aboveMa20 = 0;
168
+ let aboveMa20Eligible = 0;
169
+ let aboveMa50 = 0;
170
+ let aboveMa50Eligible = 0;
171
+ const altReturns1h = [];
172
+ const altReturns4h = [];
173
+ const altReturns24h = [];
174
+ let altTurnover1h = 0;
175
+ let altTurnover24h = 0;
176
+ let previousAltTurnover24h = 0;
177
+ for (const indexed of indexedSymbols) {
178
+ const { candles, byTimestamp, turnoverPrefix, closePrefix } = indexed;
179
+ const index = byTimestamp.get(timestamp);
180
+ if (index == null) continue;
181
+ const candle = candles[index];
182
+ const previous = candles[index - 1];
183
+ const ret = previous && previous.close > 0 ? (candle.close - previous.close) / previous.close : 0;
184
+ if (ret > 0) advancers += 1;
185
+ else if (ret < 0) decliners += 1;
186
+ else unchanged += 1;
187
+ returns.push(ret);
188
+ weightedReturns.push({
189
+ value: ret,
190
+ weight: Math.max(0, candle.turnover ?? 0)
191
+ });
192
+ const ma20Start = index - 19;
193
+ if (ma20Start >= 0) {
194
+ aboveMa20Eligible += 1;
195
+ const ma20 = sumPrefixRange(closePrefix, ma20Start, index) / 20;
196
+ if (candle.close > ma20) aboveMa20 += 1;
197
+ }
198
+ const ma50Start = index - 49;
199
+ if (ma50Start >= 0) {
200
+ aboveMa50Eligible += 1;
201
+ const ma50 = sumPrefixRange(closePrefix, ma50Start, index) / 50;
202
+ if (candle.close > ma50) aboveMa50 += 1;
203
+ }
204
+ const altReturn1h = windowReturn(candles, index, bars1h);
205
+ const altReturn4h = windowReturn(candles, index, bars4h);
206
+ const altReturn24h = windowReturn(candles, index, bars24h);
207
+ if (altReturn1h != null) altReturns1h.push(altReturn1h);
208
+ if (altReturn4h != null) altReturns4h.push(altReturn4h);
209
+ if (altReturn24h != null) altReturns24h.push(altReturn24h);
210
+ altTurnover1h += sumPrefixRange(
211
+ turnoverPrefix,
212
+ index - bars1h + 1,
213
+ index
214
+ );
215
+ altTurnover24h += sumPrefixRange(
216
+ turnoverPrefix,
217
+ index - bars24h + 1,
218
+ index
219
+ );
220
+ previousAltTurnover24h += sumPrefixRange(
221
+ turnoverPrefix,
222
+ index - bars24h * 2 + 1,
223
+ index - bars24h
224
+ );
225
+ }
226
+ const weightSum = weightedReturns.reduce(
227
+ (sum, item) => sum + item.weight,
228
+ 0
229
+ );
230
+ const volumeWeightedReturn = weightSum > 0 ? weightedReturns.reduce(
231
+ (sum, item) => sum + item.value * item.weight,
232
+ 0
233
+ ) / weightSum : null;
234
+ const btcIndex = btcIndexed?.byTimestamp.get(timestamp);
235
+ const btcReturn1h = btcIndexed && btcIndex != null ? windowReturn(btcIndexed.candles, btcIndex, bars1h) : null;
236
+ const btcReturn4h = btcIndexed && btcIndex != null ? windowReturn(btcIndexed.candles, btcIndex, bars4h) : null;
237
+ const btcReturn24h = btcIndexed && btcIndex != null ? windowReturn(btcIndexed.candles, btcIndex, bars24h) : null;
238
+ const btcTurnover1h = btcIndexed && btcIndex != null ? sumPrefixRange(
239
+ btcIndexed.turnoverPrefix,
240
+ btcIndex - bars1h + 1,
241
+ btcIndex
242
+ ) : null;
243
+ const btcTurnover24h = btcIndexed && btcIndex != null ? sumPrefixRange(
244
+ btcIndexed.turnoverPrefix,
245
+ btcIndex - bars24h + 1,
246
+ btcIndex
247
+ ) : null;
248
+ const previousBtcTurnover24h = btcIndexed && btcIndex != null ? sumPrefixRange(
249
+ btcIndexed.turnoverPrefix,
250
+ btcIndex - bars24h * 2 + 1,
251
+ btcIndex - bars24h
252
+ ) : null;
253
+ const altBasketReturn1h = mean(altReturns1h);
254
+ const altBasketReturn4h = mean(altReturns4h);
255
+ const altBasketReturn24h = mean(altReturns24h);
256
+ const btcVsAltReturn1h = btcReturn1h == null || altBasketReturn1h == null ? null : btcReturn1h - altBasketReturn1h;
257
+ const btcVsAltReturn4h = btcReturn4h == null || altBasketReturn4h == null ? null : btcReturn4h - altBasketReturn4h;
258
+ const btcVsAltReturn24h = btcReturn24h == null || altBasketReturn24h == null ? null : btcReturn24h - altBasketReturn24h;
259
+ const btcTurnoverShare1h = btcTurnover1h == null ? null : safeDivide(btcTurnover1h, btcTurnover1h + altTurnover1h);
260
+ const btcTurnoverShare24h = btcTurnover24h == null ? null : safeDivide(btcTurnover24h, btcTurnover24h + altTurnover24h);
261
+ const previousBtcTurnoverShare24h = previousBtcTurnover24h == null ? null : safeDivide(
262
+ previousBtcTurnover24h,
263
+ previousBtcTurnover24h + previousAltTurnover24h
264
+ );
265
+ const btcTurnoverShareChange24h = btcTurnoverShare24h == null || previousBtcTurnoverShare24h == null ? null : btcTurnoverShare24h - previousBtcTurnoverShare24h;
266
+ return {
267
+ universe,
268
+ interval,
269
+ ts: new Date(timestamp),
270
+ symbolsCount: returns.length,
271
+ advancers,
272
+ decliners,
273
+ unchanged,
274
+ advanceDeclineRatio: safeDivide(advancers, decliners || 1),
275
+ pctAboveMa20: safeDivide(aboveMa20, aboveMa20Eligible),
276
+ pctAboveMa50: safeDivide(aboveMa50, aboveMa50Eligible),
277
+ equalWeightedReturn: mean(returns),
278
+ volumeWeightedReturn,
279
+ dispersion: standardDeviation(returns),
280
+ btcReturn1h,
281
+ btcReturn4h,
282
+ btcReturn24h,
283
+ altBasketReturn1h,
284
+ altBasketReturn4h,
285
+ altBasketReturn24h,
286
+ btcVsAltReturn1h,
287
+ btcVsAltReturn4h,
288
+ btcVsAltReturn24h,
289
+ btcTurnoverShare1h,
290
+ btcTurnoverShare24h,
291
+ btcTurnoverShareChange24h,
292
+ altVolToBtcVol24h: btcTurnover24h == null ? null : safeDivide(altTurnover24h, btcTurnover24h),
293
+ altDispersion24h: standardDeviation(altReturns24h),
294
+ btcAltRegime: classifyBtcAltRegime({
295
+ btcReturn24h,
296
+ altBasketReturn24h,
297
+ btcVsAltReturn24h
298
+ }),
299
+ source
300
+ };
301
+ }).filter((row) => row.symbolsCount > 0);
302
+ };
303
+
304
+ // src/lib/coinMarketCapContextBackfill.ts
305
+ var import_chalk = __toESM(require("chalk"));
306
+ var import_async = require("@tradejs/core/async");
307
+ var import_marketContext = require("@tradejs/infra/timescale/marketContext");
308
+ var import_client = require("@tradejs/infra/timescale/client");
309
+ var import_userSettings = require("@tradejs/infra/userSettings");
310
+ var DAY_MS = 864e5;
311
+ var DEFAULT_CONTEXT_MAX_AGE_MS = 48 * 60 * 6e4;
312
+ var DEFAULT_FEAR_GREED_STALE_RETRY_MS = 60 * 6e4;
313
+ var SOURCE_GLOBAL_DAILY = "coinmarketcap_global";
314
+ var SOURCE_REFERENCE = "coinmarketcap_reference_asset";
315
+ var SOURCE_EXCHANGE_LIQUIDITY = "coinmarketcap_exchange_liquidity";
316
+ var SOURCE_FEAR_GREED = "coinmarketcap_fear_greed";
317
+ var SOURCE_INDEX = "coinmarketcap_index";
318
+ var COVERAGE_SCOPE_ALL = "all";
319
+ var REFERENCE_ASSETS = [
320
+ { symbol: "BTCUSDT", cmcId: 1 },
321
+ { symbol: "ETHUSDT", cmcId: 1027 }
322
+ ];
323
+ var CMC_INDEXES = [
324
+ { slug: "cmc100", path: "/v3/index/cmc100-historical" },
325
+ { slug: "cmc20", path: "/v3/index/cmc20-historical" }
326
+ ];
327
+ var DEFAULT_EXCHANGE_SLUGS = [
328
+ "binance",
329
+ "coinbase-exchange",
330
+ "okx",
331
+ "bybit",
332
+ "kraken"
333
+ ];
334
+ var lastRequestTs = 0;
335
+ var asInt = (value, fallback) => {
336
+ const parsed = Number.parseInt(String(value ?? ""), 10);
337
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
338
+ };
339
+ var parseEnabledFlag = (value, defaultValue) => {
340
+ const normalized = String(value ?? "").trim().toLowerCase();
341
+ if (!normalized) return defaultValue;
342
+ if (["1", "true", "yes", "on"].includes(normalized)) return true;
343
+ if (["0", "false", "no", "off"].includes(normalized)) return false;
344
+ return defaultValue;
345
+ };
346
+ var getBaseUrl = () => process.env.COINMARKETCAP_BASE_URL?.trim() || "https://pro-api.coinmarketcap.com";
347
+ var getRequestDelayMs = () => asInt(process.env.COINMARKETCAP_MIN_REQUEST_DELAY_MS, 2100);
348
+ var getMaxRetries = () => asInt(process.env.COINMARKETCAP_MAX_RETRIES, 4);
349
+ var getMaxBackfillDays = () => asInt(process.env.COINMARKETCAP_CONTEXT_BACKFILL_MAX_DAYS, 1098);
350
+ var getHistoricalAccessMonths = () => asInt(process.env.COINMARKETCAP_CONTEXT_HISTORICAL_ACCESS_MONTHS, 36);
351
+ var isExchangeLiquidityBackfillEnabled = () => parseEnabledFlag(
352
+ process.env.COINMARKETCAP_CONTEXT_EXCHANGE_LIQUIDITY_ENABLED,
353
+ true
354
+ );
355
+ var isFearGreedBackfillEnabled = () => parseEnabledFlag(process.env.COINMARKETCAP_CONTEXT_FEAR_GREED_ENABLED, true);
356
+ var isVerboseRequestLoggingEnabled = () => parseEnabledFlag(process.env.COINMARKETCAP_CONTEXT_VERBOSE_REQUESTS, false);
357
+ var getFearGreedPageSize = () => asInt(process.env.COINMARKETCAP_CONTEXT_FEAR_GREED_PAGE_SIZE, 500);
358
+ var getContextMaxAgeMs = () => asInt(
359
+ process.env.COINMARKETCAP_CONTEXT_MAX_AGE_MS,
360
+ DEFAULT_CONTEXT_MAX_AGE_MS
361
+ );
362
+ var getFearGreedStaleRetryMs = () => asInt(
363
+ process.env.COINMARKETCAP_CONTEXT_FEAR_GREED_STALE_RETRY_MS,
364
+ DEFAULT_FEAR_GREED_STALE_RETRY_MS
365
+ );
366
+ var getExchangeSlugs = () => {
367
+ const raw = process.env.COINMARKETCAP_CONTEXT_EXCHANGE_SLUGS;
368
+ const values = String(raw ?? "").split(",").map((item) => item.trim().toLowerCase()).filter(Boolean);
369
+ return values.length ? [...new Set(values)] : [...DEFAULT_EXCHANGE_SLUGS];
370
+ };
371
+ var toFiniteNumberOrNull = (value) => {
372
+ const numeric = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
373
+ return Number.isFinite(numeric) ? numeric : null;
374
+ };
375
+ var toIntOrNull = (value) => {
376
+ const numeric = toFiniteNumberOrNull(value);
377
+ return numeric == null ? null : Math.trunc(numeric);
378
+ };
379
+ var getRecord = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
380
+ var getNestedRecord = (value, key) => getRecord(value[key]);
381
+ var safeDivide2 = (numerator, denominator) => numerator != null && denominator != null && denominator > 0 ? numerator / denominator : null;
382
+ var sumFinite = (values) => values.reduce((sum, value) => sum + (value ?? 0), 0);
383
+ var toExchangeLiquidityRegime = ({
384
+ binanceVolumeShare,
385
+ topExchangeVolumeShare
386
+ }) => {
387
+ if (binanceVolumeShare == null && topExchangeVolumeShare == null) {
388
+ return "unknown";
389
+ }
390
+ if ((binanceVolumeShare ?? 0) >= 0.55) return "binance_led";
391
+ if ((topExchangeVolumeShare ?? 0) >= 0.6) return "concentrated";
392
+ return "balanced";
393
+ };
394
+ var normalizeFearGreedClassification = (value) => {
395
+ const normalized = String(value ?? "").trim().toLowerCase();
396
+ if (normalized === "extreme fear") return "Extreme Fear";
397
+ if (normalized === "fear") return "Fear";
398
+ if (normalized === "neutral") return "Neutral";
399
+ if (normalized === "greed") return "Greed";
400
+ if (normalized === "extreme greed") return "Extreme Greed";
401
+ return "Unknown";
402
+ };
403
+ var toFearGreedRegime = ({
404
+ value,
405
+ classification
406
+ }) => {
407
+ if (value == null) return "unknown";
408
+ if (classification === "Extreme Fear" || value <= 24) return "capitulation";
409
+ if (classification === "Fear" || value <= 44) return "risk_off";
410
+ if (classification === "Extreme Greed" || value >= 75) return "euphoric";
411
+ if (classification === "Greed" || value >= 55) return "risk_on";
412
+ if (classification === "Neutral" || value >= 45 && value <= 54) {
413
+ return "neutral";
414
+ }
415
+ return "unknown";
416
+ };
417
+ var dayStartUtc = (ms) => {
418
+ const date = new Date(ms);
419
+ return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate());
420
+ };
421
+ var toIsoDate = (ms) => new Date(ms).toISOString().slice(0, 10);
422
+ var subtractUtcMonths = (ms, months) => {
423
+ const date = new Date(ms);
424
+ const targetMonthIndex = date.getUTCMonth() - months;
425
+ const targetMonthStart = new Date(
426
+ Date.UTC(
427
+ date.getUTCFullYear(),
428
+ targetMonthIndex,
429
+ 1,
430
+ date.getUTCHours(),
431
+ date.getUTCMinutes(),
432
+ date.getUTCSeconds(),
433
+ date.getUTCMilliseconds()
434
+ )
435
+ );
436
+ const daysInTargetMonth = new Date(
437
+ Date.UTC(
438
+ targetMonthStart.getUTCFullYear(),
439
+ targetMonthStart.getUTCMonth() + 1,
440
+ 0
441
+ )
442
+ ).getUTCDate();
443
+ targetMonthStart.setUTCDate(Math.min(date.getUTCDate(), daysInTargetMonth));
444
+ return targetMonthStart.getTime();
445
+ };
446
+ var nextUtcDayStart = (ms) => dayStartUtc(ms) + DAY_MS;
447
+ var resolveCoinMarketCapBackfillWindow = (params) => {
448
+ const warmupMs = asInt(process.env.COINMARKETCAP_CONTEXT_BACKFILL_WARMUP_DAYS, 35) * DAY_MS;
449
+ const maxWindowMs = getMaxBackfillDays() * DAY_MS;
450
+ const requestedStart = params.preloadStartMs ?? params.startMs - warmupMs;
451
+ const cappedStart = Math.max(requestedStart, params.endMs - maxWindowMs);
452
+ const accessFloorMs = nextUtcDayStart(
453
+ subtractUtcMonths(params.nowMs ?? Date.now(), getHistoricalAccessMonths())
454
+ );
455
+ return {
456
+ fromMs: dayStartUtc(Math.max(cappedStart, accessFloorMs)),
457
+ toMs: dayStartUtc(params.endMs)
458
+ };
459
+ };
460
+ var hasIntervalCoverage = (coverage, fromMs, toMs, intervalMs) => {
461
+ if (!coverage) return false;
462
+ const expectedRows = Math.max(1, Math.floor((toMs - fromMs) / intervalMs));
463
+ return coverage.firstMs <= fromMs + intervalMs && coverage.lastMs >= toMs - intervalMs && coverage.rows >= Math.floor(expectedRows * 0.9);
464
+ };
465
+ var hasDailyCoverage = (coverage, fromMs, toMs) => hasIntervalCoverage(coverage, fromMs, toMs, DAY_MS);
466
+ var coverageKey = (params) => [
467
+ params.source.trim().toLowerCase(),
468
+ params.scope.trim().toLowerCase(),
469
+ params.interval.trim().toLowerCase(),
470
+ Math.trunc(params.fromMs),
471
+ Math.trunc(params.toMs)
472
+ ].join(":");
473
+ var isMarketContextBackfillCoverageDataBearing = (row) => Number(row.rowsCount ?? 0) > 0;
474
+ var coverageRowsToKeySet = (rows) => new Set(
475
+ rows.filter(isMarketContextBackfillCoverageDataBearing).map(
476
+ (row) => coverageKey({
477
+ source: row.source,
478
+ scope: row.scope,
479
+ interval: row.interval,
480
+ fromMs: row.fromMs,
481
+ toMs: row.toMs
482
+ })
483
+ )
484
+ );
485
+ var hasBackfillCoverage = (coverageKeys, params) => coverageKeys.has(coverageKey(params));
486
+ var hasRecentBackfillCoverage = (rows, params) => {
487
+ const key = coverageKey(params);
488
+ return rows.some((row) => {
489
+ if (!isMarketContextBackfillCoverageDataBearing(row)) return false;
490
+ if (coverageKey({
491
+ source: row.source,
492
+ scope: row.scope,
493
+ interval: row.interval,
494
+ fromMs: row.fromMs,
495
+ toMs: row.toMs
496
+ }) !== key) {
497
+ return false;
498
+ }
499
+ const checkedAtMs = Number(row.checkedAtMs);
500
+ return Number.isFinite(checkedAtMs) && Math.max(0, params.atMs - checkedAtMs) <= params.maxAgeMs;
501
+ });
502
+ };
503
+ var formatCoverageRange = (coverage) => {
504
+ if (!coverage) return "coverage=none";
505
+ return `coverage=${new Date(coverage.firstMs).toISOString()}..${new Date(
506
+ coverage.lastMs
507
+ ).toISOString()} rows=${coverage.rows}`;
508
+ };
509
+ var formatReferenceCoverage = (coverage) => REFERENCE_ASSETS.map(
510
+ (asset) => `${asset.symbol}:${formatCoverageRange(coverage.get(asset.symbol))}`
511
+ ).join(" ");
512
+ var resolveBackfillStatus = ({
513
+ enabled = true,
514
+ cached,
515
+ covered
516
+ }) => {
517
+ if (!enabled) return "disabled";
518
+ if (cached) return "cached";
519
+ if (covered) return "backfill_covered";
520
+ return "fetch_pending";
521
+ };
522
+ var formatSourceStatus = ({
523
+ name,
524
+ status,
525
+ rows,
526
+ coverage
527
+ }) => `${name}=${status}${rows == null ? "" : ` rows=${rows}`} ${coverage}`;
528
+ var createCoinMarketCapFetchStats = () => ({
529
+ requests: 0,
530
+ totalCredits: 0,
531
+ byPath: /* @__PURE__ */ new Map()
532
+ });
533
+ var getCoinMarketCapPathLabel = (path) => {
534
+ if (path.includes("global-metrics")) return "global";
535
+ if (path.includes("cryptocurrency/quotes")) return "reference";
536
+ if (path.includes("exchange/quotes")) return "exchangeLiquidity";
537
+ if (path.includes("fear-and-greed")) return "fearGreed";
538
+ if (path.includes("cmc100")) return "cmc100";
539
+ if (path.includes("cmc20")) return "cmc20";
540
+ return path;
541
+ };
542
+ var recordCoinMarketCapFetchStats = (stats, path, creditCount) => {
543
+ if (!stats) return;
544
+ const existing = stats.byPath.get(path) ?? { requests: 0, credits: 0 };
545
+ existing.requests += 1;
546
+ existing.credits += creditCount ?? 0;
547
+ stats.requests += 1;
548
+ stats.totalCredits += creditCount ?? 0;
549
+ stats.byPath.set(path, existing);
550
+ };
551
+ var formatCoinMarketCapFetchStats = (stats) => {
552
+ const byPath = [...stats.byPath.entries()].map(
553
+ ([path, item]) => `${getCoinMarketCapPathLabel(path)}=${item.requests}req/${item.credits}cr`
554
+ ).join(" ");
555
+ return `apiRequests=${stats.requests} apiCredits=${stats.totalCredits}${byPath ? ` ${byPath}` : ""}`;
556
+ };
557
+ var formatRowTimeRange = (rows) => {
558
+ if (!rows.length) return "range=none";
559
+ const times = rows.map((row) => row.ts.getTime()).filter((value) => Number.isFinite(value));
560
+ if (!times.length) return "range=none";
561
+ return `range=${new Date(Math.min(...times)).toISOString()}..${new Date(
562
+ Math.max(...times)
563
+ ).toISOString()}`;
564
+ };
565
+ var logBackfillSourceDone = (params) => {
566
+ console.log(
567
+ import_chalk.default.gray(
568
+ `coinmarketcap context source done: ${params.name}=${params.status} rows=${params.rows}${params.details ? ` ${params.details}` : ""}`
569
+ )
570
+ );
571
+ };
572
+ var markBackfillCoverage = (rows) => (0, import_marketContext.upsertMarketContextBackfillCoverage)(rows);
573
+ var coinMarketCapFetch = async (params) => {
574
+ const url = new URL(`${getBaseUrl()}${params.path}`);
575
+ for (const [key, value] of Object.entries(params.searchParams)) {
576
+ url.searchParams.set(key, value);
577
+ }
578
+ for (let attempt = 0; attempt <= getMaxRetries(); attempt += 1) {
579
+ const waitMs = Math.max(
580
+ 0,
581
+ lastRequestTs + getRequestDelayMs() - Date.now()
582
+ );
583
+ if (waitMs > 0) {
584
+ await (0, import_async.delay)(waitMs);
585
+ }
586
+ lastRequestTs = Date.now();
587
+ const response = await fetch(url.toString(), {
588
+ headers: {
589
+ accept: "application/json",
590
+ "X-CMC_PRO_API_KEY": params.apiKey
591
+ }
592
+ });
593
+ if (response.ok) {
594
+ const payload = await response.json();
595
+ const creditCount = toFiniteNumberOrNull(
596
+ getRecord(getRecord(payload).status).credit_count
597
+ );
598
+ recordCoinMarketCapFetchStats(params.stats, params.path, creditCount);
599
+ if (creditCount != null && isVerboseRequestLoggingEnabled()) {
600
+ console.log(
601
+ import_chalk.default.gray(`coinmarketcap ${params.path}: credits=${creditCount}`)
602
+ );
603
+ }
604
+ return payload;
605
+ }
606
+ const text = await response.text();
607
+ const retryAfterRaw = Number(response.headers.get("retry-after") ?? "");
608
+ const retryAfterMs = Number.isFinite(retryAfterRaw) && retryAfterRaw > 0 ? retryAfterRaw * 1e3 : null;
609
+ const transient = response.status === 429 || response.status >= 500;
610
+ if (attempt < getMaxRetries() && transient) {
611
+ await (0, import_async.delay)(retryAfterMs ?? Math.min(3e4, 1e3 * 2 ** attempt));
612
+ continue;
613
+ }
614
+ throw new Error(`CoinMarketCap ${params.path} ${response.status}: ${text}`);
615
+ }
616
+ return null;
617
+ };
618
+ var fetchCoinMarketCapIndexRows = async (params) => {
619
+ const rows = [];
620
+ const maxChunkDays = 4;
621
+ const totalChunks = Math.max(
622
+ 1,
623
+ Math.ceil(
624
+ (params.toMs - params.fromMs + DAY_MS) / ((maxChunkDays + 1) * DAY_MS)
625
+ )
626
+ );
627
+ let chunk = 0;
628
+ for (let cursorMs = params.fromMs; cursorMs <= params.toMs; cursorMs += (maxChunkDays + 1) * DAY_MS) {
629
+ chunk += 1;
630
+ const chunkEndMs = Math.min(params.toMs, cursorMs + maxChunkDays * DAY_MS);
631
+ const payload = await coinMarketCapFetch({
632
+ path: params.index.path,
633
+ apiKey: params.apiKey,
634
+ stats: params.stats,
635
+ searchParams: {
636
+ time_start: new Date(cursorMs).toISOString(),
637
+ time_end: new Date(chunkEndMs).toISOString(),
638
+ interval: "daily"
639
+ }
640
+ });
641
+ rows.push(
642
+ ...coinMarketCapIndexPayloadToRows(payload, params.index.slug, "1d")
643
+ );
644
+ if (chunk === 1 || chunk === totalChunks || chunk % 25 === 0) {
645
+ console.log(
646
+ import_chalk.default.gray(
647
+ `coinmarketcap index backfill progress: ${params.index.slug} chunk=${chunk}/${totalChunks} rows=${rows.length} chunkRange=${new Date(cursorMs).toISOString()}..${new Date(chunkEndMs).toISOString()}`
648
+ )
649
+ );
650
+ }
651
+ }
652
+ return { rows, chunks: chunk };
653
+ };
654
+ var coinMarketCapGlobalPayloadToRows = (payload, source = SOURCE_GLOBAL_DAILY) => {
655
+ const quotes = getRecord(getRecord(payload).data).quotes;
656
+ if (!Array.isArray(quotes)) return [];
657
+ return quotes.map((item) => {
658
+ const record = getRecord(item);
659
+ const quote = getRecord(getRecord(record.quote).USD);
660
+ const tsRaw = record.timestamp ?? quote.timestamp;
661
+ const ts = typeof tsRaw === "string" ? new Date(tsRaw) : null;
662
+ if (!ts || Number.isNaN(ts.getTime())) return null;
663
+ const totalMarketCapUsd = toFiniteNumberOrNull(quote.total_market_cap);
664
+ const altMarketCapUsd = toFiniteNumberOrNull(quote.altcoin_market_cap);
665
+ const btcToAltMarketCapRatio = totalMarketCapUsd != null && altMarketCapUsd != null && altMarketCapUsd > 0 ? Math.max(0, totalMarketCapUsd - altMarketCapUsd) / altMarketCapUsd : null;
666
+ return {
667
+ source,
668
+ ts,
669
+ updatedAt: ts,
670
+ activeCryptocurrencies: toIntOrNull(record.active_cryptocurrencies),
671
+ activeExchanges: toIntOrNull(record.active_exchanges),
672
+ activeMarketPairs: toIntOrNull(record.active_market_pairs),
673
+ markets: toIntOrNull(record.active_exchanges),
674
+ totalMarketCapUsd,
675
+ totalVolumeUsd: toFiniteNumberOrNull(quote.total_volume_24h),
676
+ totalVolumeReportedUsd: toFiniteNumberOrNull(
677
+ quote.total_volume_24h_reported
678
+ ),
679
+ btcDominancePct: toFiniteNumberOrNull(record.btc_dominance),
680
+ ethDominancePct: toFiniteNumberOrNull(record.eth_dominance),
681
+ altMarketCapUsd,
682
+ altVolumeUsd: toFiniteNumberOrNull(quote.altcoin_volume_24h),
683
+ altVolumeReportedUsd: toFiniteNumberOrNull(
684
+ quote.altcoin_volume_24h_reported
685
+ ),
686
+ btcToAltMarketCapRatio,
687
+ marketCapChangePct24hUsd: null
688
+ };
689
+ }).filter((row) => row != null);
690
+ };
691
+ var normalizeAssetDataItems = (payload) => {
692
+ const data = getRecord(payload).data;
693
+ if (!data) return [];
694
+ if (Array.isArray(data)) {
695
+ return data.filter(
696
+ (item) => item != null && typeof item === "object" && !Array.isArray(item)
697
+ );
698
+ }
699
+ if (typeof data === "object" && !Array.isArray(data)) {
700
+ const record = data;
701
+ if (Array.isArray(record.quotes)) return [record];
702
+ return Object.values(record).filter(
703
+ (item) => item != null && typeof item === "object" && !Array.isArray(item)
704
+ );
705
+ }
706
+ return [];
707
+ };
708
+ var coinMarketCapHistoricalQuotesPayloadToRows = (payload, interval = "1d") => {
709
+ return normalizeAssetDataItems(payload).flatMap((asset) => {
710
+ const cmcId = toIntOrNull(asset.id);
711
+ const symbolRaw = String(asset.symbol ?? "").trim().toUpperCase();
712
+ const target = REFERENCE_ASSETS.find(
713
+ (item) => item.cmcId === cmcId || item.symbol.replace("USDT", "") === symbolRaw
714
+ );
715
+ const quotes = Array.isArray(asset.quotes) ? asset.quotes : Object.values(getRecord(asset.quotes));
716
+ if (!target || !cmcId || !quotes.length) return [];
717
+ return quotes.map((item) => {
718
+ const record = getRecord(item);
719
+ const quote = getNestedRecord(getRecord(record.quote), "USD");
720
+ const tsRaw = record.timestamp ?? quote.timestamp ?? quote.last_updated;
721
+ const ts = typeof tsRaw === "string" ? new Date(tsRaw) : null;
722
+ if (!ts || Number.isNaN(ts.getTime())) return null;
723
+ return {
724
+ source: SOURCE_REFERENCE,
725
+ symbol: target.symbol,
726
+ cmcId,
727
+ interval,
728
+ ts,
729
+ openUsd: null,
730
+ highUsd: null,
731
+ lowUsd: null,
732
+ closeUsd: toFiniteNumberOrNull(quote.price),
733
+ volumeUsd: toFiniteNumberOrNull(quote.volume_24h),
734
+ marketCapUsd: toFiniteNumberOrNull(quote.market_cap)
735
+ };
736
+ }).filter((row) => row != null);
737
+ });
738
+ };
739
+ var coinMarketCapExchangeQuotesPayloadToLiquidityRows = (payload, interval = "1d") => {
740
+ const data = getRecord(payload).data;
741
+ const exchangeItems = Array.isArray(data) ? data : Object.values(getRecord(data));
742
+ const rowsByTs = /* @__PURE__ */ new Map();
743
+ for (const item of exchangeItems) {
744
+ const exchange = getRecord(item);
745
+ const slug = String(exchange.slug ?? "").trim().toLowerCase();
746
+ const name = String(exchange.name ?? "").trim().toLowerCase();
747
+ const quotes = Array.isArray(exchange.quotes) ? exchange.quotes : Array.isArray(exchange.quote) ? exchange.quote : [exchange];
748
+ for (const quoteItem of quotes) {
749
+ const record = getRecord(quoteItem);
750
+ const quote = getNestedRecord(getNestedRecord(record, "quote"), "USD");
751
+ const fallbackQuote = getNestedRecord(exchange, "quote");
752
+ const usd = Object.keys(quote).length ? quote : getNestedRecord(fallbackQuote, "USD");
753
+ const tsRaw = record.timestamp ?? usd.timestamp ?? usd.last_updated ?? exchange.last_updated;
754
+ const ts = typeof tsRaw === "string" ? new Date(tsRaw) : null;
755
+ if (!ts || Number.isNaN(ts.getTime())) continue;
756
+ const volumeUsd = toFiniteNumberOrNull(usd.volume_24h) ?? toFiniteNumberOrNull(usd.volume_24h_adjusted) ?? toFiniteNumberOrNull(usd.spot_volume_usd) ?? toFiniteNumberOrNull(usd.reported_volume_24h);
757
+ const key = ts.getTime();
758
+ const rows = rowsByTs.get(key) ?? [];
759
+ rows.push({ slug, name, volumeUsd });
760
+ rowsByTs.set(key, rows);
761
+ }
762
+ }
763
+ return [...rowsByTs.entries()].sort(([left], [right]) => left - right).map(([ts, exchanges]) => {
764
+ const volumes = exchanges.map((exchange) => exchange.volumeUsd);
765
+ const totalVolumeUsd = sumFinite(volumes);
766
+ const binanceVolumeUsd = exchanges.find(
767
+ (exchange) => exchange.slug === "binance" || exchange.name.includes("binance")
768
+ )?.volumeUsd ?? null;
769
+ const topExchangeVolume = Math.max(
770
+ ...volumes.map((value) => value ?? 0),
771
+ 0
772
+ );
773
+ const binanceVolumeShare = safeDivide2(binanceVolumeUsd, totalVolumeUsd);
774
+ const topExchangeVolumeShare = safeDivide2(
775
+ topExchangeVolume,
776
+ totalVolumeUsd
777
+ );
778
+ return {
779
+ source: SOURCE_EXCHANGE_LIQUIDITY,
780
+ interval,
781
+ ts: new Date(ts),
782
+ exchangesCount: exchanges.length,
783
+ totalVolumeUsd,
784
+ binanceVolumeUsd,
785
+ binanceVolumeShare,
786
+ topExchangeVolumeShare,
787
+ liquidityRegime: toExchangeLiquidityRegime({
788
+ binanceVolumeShare,
789
+ topExchangeVolumeShare
790
+ })
791
+ };
792
+ });
793
+ };
794
+ var coinMarketCapFearGreedPayloadToRows = (payload) => {
795
+ const data = getRecord(payload).data;
796
+ const items = Array.isArray(data) ? data : [];
797
+ return items.map((item) => {
798
+ const record = getRecord(item);
799
+ const timestampValue = toFiniteNumberOrNull(record.timestamp);
800
+ const value = toIntOrNull(record.value);
801
+ if (timestampValue == null || value == null) return null;
802
+ const ts = new Date(timestampValue * 1e3);
803
+ if (Number.isNaN(ts.getTime())) return null;
804
+ const classification = normalizeFearGreedClassification(
805
+ record.value_classification
806
+ );
807
+ return {
808
+ source: SOURCE_FEAR_GREED,
809
+ interval: "1d",
810
+ ts,
811
+ value,
812
+ classification,
813
+ sentimentRegime: toFearGreedRegime({ value, classification })
814
+ };
815
+ }).filter((row) => row != null);
816
+ };
817
+ var toCmcIndexWeightPct = (value) => {
818
+ if (typeof value === "string") {
819
+ return toFiniteNumberOrNull(value.replace("%", ""));
820
+ }
821
+ return toFiniteNumberOrNull(value);
822
+ };
823
+ var coinMarketCapIndexPayloadToRows = (payload, indexSlug, interval = "1d") => {
824
+ const data = getRecord(payload).data;
825
+ const items = Array.isArray(data) ? data : [];
826
+ return items.map((item) => {
827
+ const record = getRecord(item);
828
+ const tsRaw = record.update_time ?? record.timestamp;
829
+ const ts = typeof tsRaw === "string" ? new Date(tsRaw) : null;
830
+ const value = toFiniteNumberOrNull(record.value);
831
+ if (!ts || Number.isNaN(ts.getTime()) || value == null) return null;
832
+ const constituentsRaw = Array.isArray(record.constituents) ? record.constituents : [];
833
+ const constituents = constituentsRaw.map((constituent) => {
834
+ const constituentRecord = getRecord(constituent);
835
+ return {
836
+ id: toIntOrNull(constituentRecord.id),
837
+ name: typeof constituentRecord.name === "string" ? constituentRecord.name : null,
838
+ symbol: typeof constituentRecord.symbol === "string" ? constituentRecord.symbol : null,
839
+ url: typeof constituentRecord.url === "string" ? constituentRecord.url : null,
840
+ weightPct: toCmcIndexWeightPct(constituentRecord.weight),
841
+ priceUsd: toFiniteNumberOrNull(constituentRecord.priceUsd),
842
+ units: toFiniteNumberOrNull(constituentRecord.units)
843
+ };
844
+ });
845
+ const topConstituent = constituents.reduce((top, constituent) => {
846
+ if (top == null) return constituent;
847
+ return (constituent.weightPct ?? Number.NEGATIVE_INFINITY) > (top.weightPct ?? Number.NEGATIVE_INFINITY) ? constituent : top;
848
+ }, null);
849
+ return {
850
+ source: SOURCE_INDEX,
851
+ indexSlug,
852
+ interval,
853
+ ts,
854
+ value,
855
+ constituentsCount: constituents.length,
856
+ topConstituentSymbol: topConstituent?.symbol ?? null,
857
+ topConstituentWeightPct: topConstituent?.weightPct ?? null,
858
+ constituents
859
+ };
860
+ }).filter((row) => row != null);
861
+ };
862
+ var skippedResult = (cached = false) => ({
863
+ skipped: true,
864
+ globalRows: 0,
865
+ referenceRows: 0,
866
+ breadthRows: 0,
867
+ exchangeLiquidityRows: 0,
868
+ fearGreedRows: 0,
869
+ indexRows: 0,
870
+ cached
871
+ });
872
+ var backfillCoinMarketCapContext = async (params) => {
873
+ if (!Number.isFinite(params.startMs) || !Number.isFinite(params.endMs)) {
874
+ return skippedResult();
875
+ }
876
+ const { fromMs, toMs } = resolveCoinMarketCapBackfillWindow(params);
877
+ if (toMs <= fromMs) return skippedResult();
878
+ const exchangeLiquidityEnabled = isExchangeLiquidityBackfillEnabled();
879
+ const fearGreedEnabled = isFearGreedBackfillEnabled();
880
+ await (0, import_client.waitForDbReady)();
881
+ const [
882
+ globalDailyCoverage,
883
+ referenceDailyCoverage,
884
+ exchangeLiquidityCoverage,
885
+ fearGreedCoverage,
886
+ indexCoverage,
887
+ globalDailyBackfillCoverage,
888
+ referenceDailyBackfillCoverage,
889
+ exchangeLiquidityBackfillCoverage,
890
+ fearGreedBackfillCoverage,
891
+ indexBackfillCoverage
892
+ ] = await Promise.all([
893
+ (0, import_marketContext.getMarketGlobalContextCoverage)({
894
+ source: SOURCE_GLOBAL_DAILY,
895
+ startMs: fromMs,
896
+ endMs: toMs
897
+ }),
898
+ (0, import_marketContext.getMarketReferenceAssetContextCoverage)({
899
+ source: SOURCE_REFERENCE,
900
+ symbols: REFERENCE_ASSETS.map((item) => item.symbol),
901
+ interval: "1d",
902
+ startMs: fromMs,
903
+ endMs: toMs
904
+ }),
905
+ exchangeLiquidityEnabled ? (0, import_marketContext.getMarketCmcExchangeLiquidityContextCoverage)({
906
+ source: SOURCE_EXCHANGE_LIQUIDITY,
907
+ interval: "1d",
908
+ startMs: fromMs,
909
+ endMs: toMs
910
+ }) : Promise.resolve(null),
911
+ fearGreedEnabled ? (0, import_marketContext.getMarketCmcFearGreedContextCoverage)({
912
+ source: SOURCE_FEAR_GREED,
913
+ interval: "1d",
914
+ startMs: fromMs,
915
+ endMs: toMs
916
+ }) : Promise.resolve(null),
917
+ (0, import_marketContext.getMarketCmcIndexContextCoverage)({
918
+ source: SOURCE_INDEX,
919
+ indexSlugs: CMC_INDEXES.map((index) => index.slug),
920
+ interval: "1d",
921
+ startMs: fromMs,
922
+ endMs: toMs
923
+ }),
924
+ (0, import_marketContext.getMarketContextBackfillCoverage)({
925
+ source: SOURCE_GLOBAL_DAILY,
926
+ scopes: [COVERAGE_SCOPE_ALL],
927
+ interval: "1d",
928
+ fromMs,
929
+ toMs
930
+ }),
931
+ (0, import_marketContext.getMarketContextBackfillCoverage)({
932
+ source: SOURCE_REFERENCE,
933
+ scopes: REFERENCE_ASSETS.map((item) => item.symbol),
934
+ interval: "1d",
935
+ fromMs,
936
+ toMs
937
+ }),
938
+ exchangeLiquidityEnabled ? (0, import_marketContext.getMarketContextBackfillCoverage)({
939
+ source: SOURCE_EXCHANGE_LIQUIDITY,
940
+ scopes: [COVERAGE_SCOPE_ALL],
941
+ interval: "1d",
942
+ fromMs,
943
+ toMs
944
+ }) : Promise.resolve([]),
945
+ fearGreedEnabled ? (0, import_marketContext.getMarketContextBackfillCoverage)({
946
+ source: SOURCE_FEAR_GREED,
947
+ scopes: [COVERAGE_SCOPE_ALL],
948
+ interval: "1d",
949
+ fromMs,
950
+ toMs
951
+ }) : Promise.resolve([]),
952
+ (0, import_marketContext.getMarketContextBackfillCoverage)({
953
+ source: SOURCE_INDEX,
954
+ scopes: CMC_INDEXES.map((index) => index.slug),
955
+ interval: "1d",
956
+ fromMs,
957
+ toMs
958
+ })
959
+ ]);
960
+ const globalDailyBackfillKeys = coverageRowsToKeySet(
961
+ globalDailyBackfillCoverage
962
+ );
963
+ const referenceDailyBackfillKeys = coverageRowsToKeySet(
964
+ referenceDailyBackfillCoverage
965
+ );
966
+ const exchangeLiquidityBackfillKeys = coverageRowsToKeySet(
967
+ exchangeLiquidityBackfillCoverage
968
+ );
969
+ const fearGreedBackfillKeys = coverageRowsToKeySet(fearGreedBackfillCoverage);
970
+ const indexBackfillKeys = coverageRowsToKeySet(indexBackfillCoverage);
971
+ const globalDailyCached = hasDailyCoverage(globalDailyCoverage, fromMs, toMs);
972
+ const referencesDailyCached = REFERENCE_ASSETS.every(
973
+ (asset) => hasDailyCoverage(referenceDailyCoverage.get(asset.symbol), fromMs, toMs)
974
+ );
975
+ const exchangeLiquidityCached = !exchangeLiquidityEnabled || hasDailyCoverage(exchangeLiquidityCoverage, fromMs, toMs);
976
+ const fearGreedCached = !fearGreedEnabled || hasDailyCoverage(fearGreedCoverage, fromMs, toMs);
977
+ const indexesCached = CMC_INDEXES.every(
978
+ (index) => hasDailyCoverage(indexCoverage.get(index.slug), fromMs, toMs)
979
+ );
980
+ const globalDailyBackfillCovered = hasBackfillCoverage(
981
+ globalDailyBackfillKeys,
982
+ {
983
+ source: SOURCE_GLOBAL_DAILY,
984
+ scope: COVERAGE_SCOPE_ALL,
985
+ interval: "1d",
986
+ fromMs,
987
+ toMs
988
+ }
989
+ );
990
+ const referencesDailyBackfillCovered = REFERENCE_ASSETS.every(
991
+ (asset) => hasBackfillCoverage(referenceDailyBackfillKeys, {
992
+ source: SOURCE_REFERENCE,
993
+ scope: asset.symbol,
994
+ interval: "1d",
995
+ fromMs,
996
+ toMs
997
+ })
998
+ );
999
+ const exchangeLiquidityBackfillCovered = hasBackfillCoverage(
1000
+ exchangeLiquidityBackfillKeys,
1001
+ {
1002
+ source: SOURCE_EXCHANGE_LIQUIDITY,
1003
+ scope: COVERAGE_SCOPE_ALL,
1004
+ interval: "1d",
1005
+ fromMs,
1006
+ toMs
1007
+ }
1008
+ );
1009
+ const fearGreedBackfillParams = {
1010
+ source: SOURCE_FEAR_GREED,
1011
+ scope: COVERAGE_SCOPE_ALL,
1012
+ interval: "1d",
1013
+ fromMs,
1014
+ toMs
1015
+ };
1016
+ const fearGreedBackfillWasCovered = hasBackfillCoverage(
1017
+ fearGreedBackfillKeys,
1018
+ fearGreedBackfillParams
1019
+ );
1020
+ const fearGreedLatestAgeMs = fearGreedCoverage == null ? Number.POSITIVE_INFINITY : Math.max(0, params.endMs - fearGreedCoverage.lastMs);
1021
+ const refreshStaleFearGreed = params.refreshStaleFearGreed === true && fearGreedLatestAgeMs > getContextMaxAgeMs();
1022
+ const fearGreedBackfillCovered = fearGreedBackfillWasCovered && (!refreshStaleFearGreed || hasRecentBackfillCoverage(fearGreedBackfillCoverage, {
1023
+ ...fearGreedBackfillParams,
1024
+ atMs: params.endMs,
1025
+ maxAgeMs: getFearGreedStaleRetryMs()
1026
+ }));
1027
+ const indexesBackfillCovered = CMC_INDEXES.every(
1028
+ (index) => hasBackfillCoverage(indexBackfillKeys, {
1029
+ source: SOURCE_INDEX,
1030
+ scope: index.slug,
1031
+ interval: "1d",
1032
+ fromMs,
1033
+ toMs
1034
+ })
1035
+ );
1036
+ const ready = (dataset, existing) => params.repairDataset ? params.repairDataset !== dataset : existing;
1037
+ const globalDailyReady = ready(
1038
+ "cmc-global-1d",
1039
+ globalDailyCached || globalDailyBackfillCovered
1040
+ );
1041
+ const referencesDailyReady = ready(
1042
+ "cmc-reference_asset-1d",
1043
+ referencesDailyCached || referencesDailyBackfillCovered
1044
+ );
1045
+ const exchangeLiquidityReady = ready(
1046
+ "cmc-exchange_liquidity-1d",
1047
+ exchangeLiquidityCached || !exchangeLiquidityEnabled || exchangeLiquidityBackfillCovered
1048
+ );
1049
+ const fearGreedReady = ready(
1050
+ "cmc-fear_greed-1d",
1051
+ fearGreedCached || !fearGreedEnabled || fearGreedBackfillCovered
1052
+ );
1053
+ const indexesReady = ready(
1054
+ "cmc-index-1d",
1055
+ indexesCached || indexesBackfillCovered
1056
+ );
1057
+ let globalStatus = resolveBackfillStatus({
1058
+ cached: globalDailyCached,
1059
+ covered: globalDailyBackfillCovered
1060
+ });
1061
+ let referenceStatus = resolveBackfillStatus({
1062
+ cached: referencesDailyCached,
1063
+ covered: referencesDailyBackfillCovered
1064
+ });
1065
+ let exchangeLiquidityStatus = resolveBackfillStatus({
1066
+ enabled: exchangeLiquidityEnabled,
1067
+ cached: exchangeLiquidityCached,
1068
+ covered: exchangeLiquidityBackfillCovered
1069
+ });
1070
+ let fearGreedStatus = resolveBackfillStatus({
1071
+ enabled: fearGreedEnabled,
1072
+ cached: fearGreedCached,
1073
+ covered: fearGreedBackfillCovered
1074
+ });
1075
+ let indexStatus = resolveBackfillStatus({
1076
+ cached: indexesCached,
1077
+ covered: indexesBackfillCovered
1078
+ });
1079
+ console.log(
1080
+ import_chalk.default.gray(
1081
+ [
1082
+ "coinmarketcap context status:",
1083
+ formatSourceStatus({
1084
+ name: "global",
1085
+ status: globalStatus,
1086
+ coverage: formatCoverageRange(globalDailyCoverage)
1087
+ }),
1088
+ formatSourceStatus({
1089
+ name: "reference",
1090
+ status: referenceStatus,
1091
+ coverage: formatReferenceCoverage(referenceDailyCoverage)
1092
+ }),
1093
+ formatSourceStatus({
1094
+ name: "exchangeLiquidity",
1095
+ status: exchangeLiquidityStatus,
1096
+ coverage: formatCoverageRange(exchangeLiquidityCoverage)
1097
+ }),
1098
+ formatSourceStatus({
1099
+ name: "fearGreed",
1100
+ status: fearGreedStatus,
1101
+ coverage: formatCoverageRange(fearGreedCoverage)
1102
+ }),
1103
+ formatSourceStatus({
1104
+ name: "indexes",
1105
+ status: indexStatus,
1106
+ coverage: CMC_INDEXES.map(
1107
+ (index) => `${index.slug}:${formatCoverageRange(indexCoverage.get(index.slug))}`
1108
+ ).join(" ")
1109
+ })
1110
+ ].join(" ")
1111
+ )
1112
+ );
1113
+ if (globalDailyReady && referencesDailyReady && exchangeLiquidityReady && fearGreedReady && indexesReady) {
1114
+ console.log(
1115
+ import_chalk.default.gray(
1116
+ `coinmarketcap context cached: ${new Date(fromMs).toISOString()}..${new Date(toMs).toISOString()}`
1117
+ )
1118
+ );
1119
+ return skippedResult(true);
1120
+ }
1121
+ const settings = await (0, import_userSettings.getUserSettings)(params.userName);
1122
+ const apiKey = settings.COINMARKETCAP_API_KEY.trim();
1123
+ if (!apiKey) {
1124
+ throw new Error(
1125
+ `Missing COINMARKETCAP_API_KEY for historical context backfill (user=${params.userName})`
1126
+ );
1127
+ }
1128
+ let globalRows = 0;
1129
+ let referenceRows = 0;
1130
+ let exchangeLiquidityRows = 0;
1131
+ let fearGreedRows = 0;
1132
+ let indexRows = 0;
1133
+ const fetchStats = createCoinMarketCapFetchStats();
1134
+ const pendingSources = [
1135
+ !globalDailyReady ? "global" : null,
1136
+ !referencesDailyReady ? "reference" : null,
1137
+ exchangeLiquidityEnabled && !exchangeLiquidityReady ? "exchangeLiquidity" : null,
1138
+ fearGreedEnabled && !fearGreedReady ? "fearGreed" : null,
1139
+ !indexesReady ? "indexes" : null
1140
+ ].filter((source) => source != null);
1141
+ console.log(
1142
+ import_chalk.default.cyan(
1143
+ `coinmarketcap context backfill: window=${new Date(fromMs).toISOString()}..${new Date(toMs).toISOString()} days=${Math.floor((toMs - fromMs) / DAY_MS) + 1} fetch=${pendingSources.join(",")}`
1144
+ )
1145
+ );
1146
+ if (!globalDailyReady) {
1147
+ const payload = await coinMarketCapFetch({
1148
+ path: "/v1/global-metrics/quotes/historical",
1149
+ apiKey,
1150
+ stats: fetchStats,
1151
+ searchParams: {
1152
+ time_start: toIsoDate(fromMs),
1153
+ time_end: toIsoDate(toMs),
1154
+ interval: "1d",
1155
+ convert: "USD"
1156
+ }
1157
+ });
1158
+ const rows = coinMarketCapGlobalPayloadToRows(payload, SOURCE_GLOBAL_DAILY);
1159
+ await (0, import_marketContext.upsertMarketGlobalContextRows)(rows);
1160
+ await markBackfillCoverage([
1161
+ {
1162
+ source: SOURCE_GLOBAL_DAILY,
1163
+ scope: COVERAGE_SCOPE_ALL,
1164
+ interval: "1d",
1165
+ fromMs,
1166
+ toMs,
1167
+ rowsCount: rows.length
1168
+ }
1169
+ ]);
1170
+ globalRows += rows.length;
1171
+ globalStatus = "fetched";
1172
+ logBackfillSourceDone({
1173
+ name: "global",
1174
+ status: globalStatus,
1175
+ rows: rows.length,
1176
+ details: formatRowTimeRange(rows)
1177
+ });
1178
+ }
1179
+ if (!referencesDailyReady) {
1180
+ const payload = await coinMarketCapFetch({
1181
+ path: "/v3/cryptocurrency/quotes/historical",
1182
+ apiKey,
1183
+ stats: fetchStats,
1184
+ searchParams: {
1185
+ id: REFERENCE_ASSETS.map((item) => item.cmcId).join(","),
1186
+ time_start: toIsoDate(fromMs),
1187
+ time_end: toIsoDate(toMs),
1188
+ interval: "1d",
1189
+ convert: "USD"
1190
+ }
1191
+ });
1192
+ const rows = coinMarketCapHistoricalQuotesPayloadToRows(payload, "1d");
1193
+ await (0, import_marketContext.upsertMarketReferenceAssetContextRows)(rows);
1194
+ await markBackfillCoverage(
1195
+ REFERENCE_ASSETS.map((asset) => ({
1196
+ source: SOURCE_REFERENCE,
1197
+ scope: asset.symbol,
1198
+ interval: "1d",
1199
+ fromMs,
1200
+ toMs,
1201
+ rowsCount: rows.filter((row) => row.symbol === asset.symbol).length
1202
+ }))
1203
+ );
1204
+ referenceRows += rows.length;
1205
+ referenceStatus = "fetched";
1206
+ logBackfillSourceDone({
1207
+ name: "reference",
1208
+ status: referenceStatus,
1209
+ rows: rows.length,
1210
+ details: `${REFERENCE_ASSETS.map(
1211
+ (asset) => `${asset.symbol}:${rows.filter((row) => row.symbol === asset.symbol).length}`
1212
+ ).join(" ")} ${formatRowTimeRange(rows)}`
1213
+ });
1214
+ }
1215
+ if (exchangeLiquidityEnabled && !exchangeLiquidityReady) {
1216
+ const payload = await coinMarketCapFetch({
1217
+ path: "/v1/exchange/quotes/historical",
1218
+ apiKey,
1219
+ stats: fetchStats,
1220
+ searchParams: {
1221
+ slug: getExchangeSlugs().join(","),
1222
+ time_start: toIsoDate(fromMs),
1223
+ time_end: toIsoDate(toMs),
1224
+ interval: "1d",
1225
+ convert: "USD"
1226
+ }
1227
+ });
1228
+ const rows = coinMarketCapExchangeQuotesPayloadToLiquidityRows(
1229
+ payload,
1230
+ "1d"
1231
+ );
1232
+ await (0, import_marketContext.upsertMarketCmcExchangeLiquidityContextRows)(rows);
1233
+ await markBackfillCoverage([
1234
+ {
1235
+ source: SOURCE_EXCHANGE_LIQUIDITY,
1236
+ scope: COVERAGE_SCOPE_ALL,
1237
+ interval: "1d",
1238
+ fromMs,
1239
+ toMs,
1240
+ rowsCount: rows.length
1241
+ }
1242
+ ]);
1243
+ exchangeLiquidityRows = rows.length;
1244
+ exchangeLiquidityStatus = "fetched";
1245
+ logBackfillSourceDone({
1246
+ name: "exchangeLiquidity",
1247
+ status: exchangeLiquidityStatus,
1248
+ rows: rows.length,
1249
+ details: formatRowTimeRange(rows)
1250
+ });
1251
+ }
1252
+ if (fearGreedEnabled && !fearGreedReady) {
1253
+ const rows = [];
1254
+ const pageSize = getFearGreedPageSize();
1255
+ let pages = 0;
1256
+ for (let start = 1; ; start += pageSize) {
1257
+ const payload = await coinMarketCapFetch({
1258
+ path: "/v3/fear-and-greed/historical",
1259
+ apiKey,
1260
+ stats: fetchStats,
1261
+ searchParams: {
1262
+ start: String(start),
1263
+ limit: String(pageSize)
1264
+ }
1265
+ });
1266
+ pages += 1;
1267
+ const pageRows = coinMarketCapFearGreedPayloadToRows(payload);
1268
+ if (!pageRows.length) break;
1269
+ const matchingRows = pageRows.filter((row) => {
1270
+ const ts = row.ts.getTime();
1271
+ return ts >= fromMs && ts <= toMs;
1272
+ });
1273
+ rows.push(...matchingRows);
1274
+ const oldestPageTs = Math.min(...pageRows.map((row) => row.ts.getTime()));
1275
+ if (oldestPageTs < fromMs || pageRows.length < pageSize) break;
1276
+ }
1277
+ await (0, import_marketContext.upsertMarketCmcFearGreedContextRows)(rows);
1278
+ await markBackfillCoverage([
1279
+ {
1280
+ source: SOURCE_FEAR_GREED,
1281
+ scope: COVERAGE_SCOPE_ALL,
1282
+ interval: "1d",
1283
+ fromMs,
1284
+ toMs,
1285
+ rowsCount: rows.length
1286
+ }
1287
+ ]);
1288
+ fearGreedRows = rows.length;
1289
+ fearGreedStatus = "fetched";
1290
+ logBackfillSourceDone({
1291
+ name: "fearGreed",
1292
+ status: fearGreedStatus,
1293
+ rows: rows.length,
1294
+ details: `pages=${pages} ${formatRowTimeRange(rows)}`
1295
+ });
1296
+ }
1297
+ if (!indexesReady) {
1298
+ const rowsByIndex = await Promise.all(
1299
+ CMC_INDEXES.map(async (index) => ({
1300
+ index,
1301
+ result: await fetchCoinMarketCapIndexRows({
1302
+ apiKey,
1303
+ index,
1304
+ fromMs,
1305
+ toMs,
1306
+ stats: fetchStats
1307
+ })
1308
+ }))
1309
+ );
1310
+ const rows = rowsByIndex.flatMap((item) => item.result.rows);
1311
+ await (0, import_marketContext.upsertMarketCmcIndexContextRows)(rows);
1312
+ await markBackfillCoverage(
1313
+ rowsByIndex.map(({ index, result }) => ({
1314
+ source: SOURCE_INDEX,
1315
+ scope: index.slug,
1316
+ interval: "1d",
1317
+ fromMs,
1318
+ toMs,
1319
+ rowsCount: result.rows.length
1320
+ }))
1321
+ );
1322
+ indexRows = rows.length;
1323
+ indexStatus = "fetched";
1324
+ logBackfillSourceDone({
1325
+ name: "indexes",
1326
+ status: indexStatus,
1327
+ rows: rows.length,
1328
+ details: `${rowsByIndex.map(
1329
+ ({ index, result }) => `${index.slug}:${result.rows.length} rows/${result.chunks} chunks`
1330
+ ).join(" ")} ${formatRowTimeRange(rows)}`
1331
+ });
1332
+ }
1333
+ console.log(
1334
+ import_chalk.default.green(
1335
+ [
1336
+ "coinmarketcap context backfill done:",
1337
+ `global=${globalStatus} rows=${globalRows}`,
1338
+ `reference=${referenceStatus} rows=${referenceRows}`,
1339
+ `exchangeLiquidity=${exchangeLiquidityStatus} rows=${exchangeLiquidityRows}`,
1340
+ `fearGreed=${fearGreedStatus} rows=${fearGreedRows}`,
1341
+ `indexes=${indexStatus} rows=${indexRows}`,
1342
+ formatCoinMarketCapFetchStats(fetchStats)
1343
+ ].join(" ")
1344
+ )
1345
+ );
1346
+ return {
1347
+ skipped: false,
1348
+ globalRows,
1349
+ referenceRows,
1350
+ breadthRows: 0,
1351
+ exchangeLiquidityRows,
1352
+ fearGreedRows,
1353
+ indexRows,
1354
+ cached: false
1355
+ };
1356
+ };
1357
+
1358
+ // src/lib/derivativesContextBackfill.ts
1359
+ var import_chalk2 = __toESM(require("chalk"));
1360
+ var import_progress = __toESM(require("progress"));
1361
+ var import_async2 = require("@tradejs/core/async");
1362
+ var import_constants = require("@tradejs/core/constants");
1363
+ var import_indicators = require("@tradejs/core/indicators");
1364
+ var import_derivatives = require("@tradejs/infra/timescale/derivatives");
1365
+ var import_client2 = require("@tradejs/infra/timescale/client");
1366
+ var import_userSettings2 = require("@tradejs/infra/userSettings");
1367
+ var HOUR_MS = 60 * 60 * 1e3;
1368
+ var DAY_MS2 = 24 * HOUR_MS;
1369
+ var lastRequestTs2 = 0;
1370
+ var asInt2 = (value, fallback) => {
1371
+ const parsed = Number.parseInt(String(value ?? ""), 10);
1372
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
1373
+ };
1374
+ var parseList = (value) => String(value ?? "").split(",").map((item) => item.trim().toUpperCase()).filter(Boolean);
1375
+ var getCoinalyzeApiKey = async (userName) => {
1376
+ const settings = await (0, import_userSettings2.getUserSettings)(userName);
1377
+ return settings.COINALYZE_API_KEY.trim() || String(process.env.COINALYZE_API_KEY ?? "").trim();
1378
+ };
1379
+ var getCoinalyzeBaseUrl = () => process.env.COINALYZE_BASE_URL?.trim() || "https://api.coinalyze.net/v1";
1380
+ var getRequestDelayMs2 = () => Math.max(
1381
+ 100,
1382
+ asInt2(
1383
+ process.env.DERIVATIVES_CONTEXT_BACKFILL_REQUEST_DELAY_MS ?? process.env.COINALYZE_MIN_REQUEST_DELAY_MS,
1384
+ 200
1385
+ )
1386
+ );
1387
+ var getRequestTimeoutMs = () => Math.max(5e3, asInt2(process.env.COINALYZE_REQUEST_TIMEOUT_MS, 45e3));
1388
+ var networkErrorCodes = /* @__PURE__ */ new Set([
1389
+ "UND_ERR_CONNECT_TIMEOUT",
1390
+ "ETIMEDOUT",
1391
+ "ECONNRESET",
1392
+ "ECONNREFUSED",
1393
+ "ENOTFOUND",
1394
+ "EAI_AGAIN",
1395
+ "EHOSTUNREACH"
1396
+ ]);
1397
+ var getNestedErrorValue = (error, key) => {
1398
+ let current = error;
1399
+ const seen = /* @__PURE__ */ new Set();
1400
+ while (current && typeof current === "object" && !seen.has(current)) {
1401
+ seen.add(current);
1402
+ const record = current;
1403
+ const value = record[key];
1404
+ if (typeof value === "string" && value.trim()) {
1405
+ return value.trim();
1406
+ }
1407
+ current = record.cause;
1408
+ }
1409
+ return null;
1410
+ };
1411
+ var getCoinalyzeErrorCause = (error) => {
1412
+ const code = getNestedErrorValue(error, "code");
1413
+ const message = getNestedErrorValue(error, "message") ?? String(error || "unknown error");
1414
+ return code ? `${code}: ${message}` : message;
1415
+ };
1416
+ var isRetryableCoinalyzeFetchError = (error) => {
1417
+ const name = getNestedErrorValue(error, "name");
1418
+ const code = getNestedErrorValue(error, "code");
1419
+ const message = getNestedErrorValue(error, "message")?.toLowerCase() ?? "";
1420
+ return name === "AbortError" || code != null && networkErrorCodes.has(code) || message.includes("aborted") || message.includes("fetch failed");
1421
+ };
1422
+ var formatCoinalyzeRequestError = ({
1423
+ url,
1424
+ error,
1425
+ attempts,
1426
+ timeoutMs
1427
+ }) => {
1428
+ const endpoint = (() => {
1429
+ try {
1430
+ const parsed = new URL(url);
1431
+ return `${parsed.origin}${parsed.pathname}`;
1432
+ } catch {
1433
+ return url;
1434
+ }
1435
+ })();
1436
+ return [
1437
+ `Coinalyze request failed: ${endpoint}`,
1438
+ `cause=${getCoinalyzeErrorCause(error)}`,
1439
+ `attempts=${attempts}`,
1440
+ `timeout=${timeoutMs}ms`,
1441
+ "Set COINALYZE_REQUEST_TIMEOUT_MS/COINALYZE_MAX_RETRIES to retry longer, or DERIVATIVES_CONTEXT_ENABLED=live to skip derivatives backfill during backtests."
1442
+ ].join("; ");
1443
+ };
1444
+ var fetchJsonWithRateLimit = async (url, apiKey) => {
1445
+ const requestDelayMs = getRequestDelayMs2();
1446
+ const requestTimeoutMs = getRequestTimeoutMs();
1447
+ const maxRetries = asInt2(process.env.COINALYZE_MAX_RETRIES, 4);
1448
+ let lastError = null;
1449
+ for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
1450
+ const waitMs = Math.max(0, lastRequestTs2 + requestDelayMs - Date.now());
1451
+ if (waitMs > 0) {
1452
+ await (0, import_async2.delay)(waitMs);
1453
+ }
1454
+ lastRequestTs2 = Date.now();
1455
+ const controller = new AbortController();
1456
+ const timer = setTimeout(() => controller.abort(), requestTimeoutMs);
1457
+ let response;
1458
+ try {
1459
+ response = await fetch(url, {
1460
+ headers: {
1461
+ api_key: apiKey,
1462
+ "x-api-key": apiKey,
1463
+ Authorization: `Bearer ${apiKey}`
1464
+ },
1465
+ signal: controller.signal
1466
+ });
1467
+ } catch (error) {
1468
+ clearTimeout(timer);
1469
+ lastError = error;
1470
+ if (attempt < maxRetries && isRetryableCoinalyzeFetchError(error)) {
1471
+ await (0, import_async2.delay)(Math.min(12e3, 800 * 2 ** attempt));
1472
+ continue;
1473
+ }
1474
+ throw new Error(
1475
+ formatCoinalyzeRequestError({
1476
+ url,
1477
+ error,
1478
+ attempts: attempt + 1,
1479
+ timeoutMs: requestTimeoutMs
1480
+ })
1481
+ );
1482
+ } finally {
1483
+ clearTimeout(timer);
1484
+ }
1485
+ if (response.ok) {
1486
+ return response.json();
1487
+ }
1488
+ const text = await response.text();
1489
+ const retryAfterRaw = Number(response.headers.get("retry-after") ?? "");
1490
+ const retryAfterMs = Number.isFinite(retryAfterRaw) && retryAfterRaw > 0 ? retryAfterRaw * 1e3 : null;
1491
+ const transient = response.status === 429 || response.status >= 500;
1492
+ if (attempt < maxRetries && transient) {
1493
+ await (0, import_async2.delay)(retryAfterMs ?? Math.min(12e3, 800 * 2 ** attempt));
1494
+ continue;
1495
+ }
1496
+ throw new Error(`Coinalyze ${response.status}: ${text}`);
1497
+ }
1498
+ throw new Error(
1499
+ formatCoinalyzeRequestError({
1500
+ url,
1501
+ error: lastError,
1502
+ attempts: maxRetries + 1,
1503
+ timeoutMs: requestTimeoutMs
1504
+ })
1505
+ );
1506
+ };
1507
+ var fetchCoinalyzeMarkets = async (apiKey) => {
1508
+ const raw = await fetchJsonWithRateLimit(
1509
+ `${getCoinalyzeBaseUrl()}/future-markets`,
1510
+ apiKey
1511
+ );
1512
+ return Array.isArray(raw) ? raw : [];
1513
+ };
1514
+ var selectBestMarket = (candidates, exchangePriority) => {
1515
+ const scored = candidates.map((item) => {
1516
+ let score = 0;
1517
+ if (item.is_perpetual) score += 50;
1518
+ if (String(item.quote_asset || "").toUpperCase() === "USDT") score += 25;
1519
+ if (item.symbol.includes("_PERP")) score += 10;
1520
+ const exchange = String(item.exchange || "").toUpperCase();
1521
+ const idx = exchangePriority.indexOf(exchange);
1522
+ if (idx >= 0) {
1523
+ score += (exchangePriority.length - idx) * 10;
1524
+ }
1525
+ return { item, score };
1526
+ });
1527
+ scored.sort((a, b) => {
1528
+ if (b.score !== a.score) return b.score - a.score;
1529
+ return a.item.symbol.localeCompare(b.item.symbol);
1530
+ });
1531
+ return scored[0]?.item ?? null;
1532
+ };
1533
+ var buildMatches = (tickers, markets, exchangePriority) => {
1534
+ const marketByTicker = /* @__PURE__ */ new Map();
1535
+ for (const market of markets) {
1536
+ const key = String(market.symbol_on_exchange || "").trim().toUpperCase();
1537
+ if (!key) continue;
1538
+ const list = marketByTicker.get(key) ?? [];
1539
+ list.push(market);
1540
+ marketByTicker.set(key, list);
1541
+ }
1542
+ const matches = [];
1543
+ const unmatched = [];
1544
+ for (const ticker of tickers) {
1545
+ const candidates = marketByTicker.get(ticker) ?? [];
1546
+ const selected = selectBestMarket(candidates, exchangePriority);
1547
+ if (!selected) {
1548
+ unmatched.push(ticker);
1549
+ continue;
1550
+ }
1551
+ matches.push({
1552
+ symbol: ticker,
1553
+ marketSymbol: selected.symbol,
1554
+ exchange: selected.exchange
1555
+ });
1556
+ }
1557
+ return { matches, unmatched };
1558
+ };
1559
+ async function resolveCoinalyzeBackfillMarket(userName, symbol) {
1560
+ const apiKey = await getCoinalyzeApiKey(userName);
1561
+ if (!apiKey) throw new Error("Missing Coinalyze API key.");
1562
+ const markets = (await fetchCoinalyzeMarkets(apiKey)).filter(
1563
+ (market) => market.quote_asset?.toUpperCase() === "USDT" && market.is_perpetual
1564
+ );
1565
+ const priority = parseList(
1566
+ process.env.COINALYZE_EXCHANGE_PRIORITY || process.env.DERIVATIVES_CONTEXT_EXCHANGE_PRIORITY || "A,6,0"
1567
+ );
1568
+ const { matches } = buildMatches([symbol], markets, priority);
1569
+ return { apiKey, marketSymbol: matches[0]?.marketSymbol };
1570
+ }
1571
+
1572
+ // src/scripts/coverageBackfill.ts
1573
+ var DAY_MS3 = 864e5;
1574
+ async function fetchTarget(userName, request, target, loaded) {
1575
+ const { dataset, end } = target;
1576
+ const { series, day } = request;
1577
+ const endMs = end - 1;
1578
+ const interval = dataset.interval;
1579
+ const missing = (time) => time >= day && time < end && !loaded.has(time);
1580
+ if (dataset.provider === "CoinMarketCap") {
1581
+ await backfillCoinMarketCapContext({
1582
+ userName,
1583
+ startMs: day,
1584
+ preloadStartMs: day,
1585
+ endMs: end,
1586
+ repairDataset: dataset.id
1587
+ });
1588
+ return;
1589
+ }
1590
+ if (dataset.provider === "Coinalyze") {
1591
+ if (interval !== "15m" && interval !== "1h") return;
1592
+ const { apiKey, marketSymbol } = await resolveCoinalyzeBackfillMarket(
1593
+ userName,
1594
+ series
1595
+ );
1596
+ if (!marketSymbol) return;
1597
+ const { derivativesRows = [] } = await import_connectors.marketDataProviders.coinalyze.fetchWindow({
1598
+ symbol: series,
1599
+ marketSymbol,
1600
+ apiKey,
1601
+ interval,
1602
+ fromMs: day,
1603
+ toMs: endMs
1604
+ });
1605
+ await (0, import_derivatives2.upsertDerivatives)(
1606
+ derivativesRows.filter(({ ts }) => missing(ts.getTime())).flatMap((row) => {
1607
+ const { symbol, ts, interval: interval2, source } = row;
1608
+ if (dataset.id.startsWith("coinalyze-open-interest-")) {
1609
+ return row.openInterest == null ? [] : [
1610
+ {
1611
+ symbol,
1612
+ ts,
1613
+ interval: interval2,
1614
+ source,
1615
+ openInterest: row.openInterest
1616
+ }
1617
+ ];
1618
+ }
1619
+ if (dataset.id.startsWith("coinalyze-funding-")) {
1620
+ return row.fundingRate == null ? [] : [
1621
+ {
1622
+ symbol,
1623
+ ts,
1624
+ interval: interval2,
1625
+ source,
1626
+ fundingRate: row.fundingRate
1627
+ }
1628
+ ];
1629
+ }
1630
+ const { liqLong, liqShort, liqTotal } = row;
1631
+ return liqLong == null && liqShort == null && liqTotal == null ? [] : [{ symbol, ts, interval: interval2, source, liqLong, liqShort, liqTotal }];
1632
+ })
1633
+ );
1634
+ return;
1635
+ }
1636
+ if (dataset.id === "binance-coinbase-spread-15m") {
1637
+ const { spreadRows = [] } = await import_connectors.marketDataProviders.binance_coinbase_spread.fetchWindow({
1638
+ symbol: series,
1639
+ interval: "15m",
1640
+ fromMs: day,
1641
+ toMs: endMs
1642
+ });
1643
+ await (0, import_spread.upsertSpreadRows)(
1644
+ spreadRows.filter(({ ts }) => missing(ts.getTime()))
1645
+ );
1646
+ return;
1647
+ }
1648
+ const provider = dataset.provider === "Bybit" ? "bybit" : "binance";
1649
+ const connector = await import_connectors.connectors[provider === "bybit" ? import_connectors.ConnectorNames.ByBit : import_connectors.ConnectorNames.Binance]({ userName });
1650
+ const connectorInterval = interval === "5m" ? "5" : interval === "1h" ? "60" : "15";
1651
+ const fetchCandles = async (symbol, start) => {
1652
+ const candles = await connector.kline({
1653
+ symbol,
1654
+ interval: connectorInterval,
1655
+ start,
1656
+ end: endMs,
1657
+ silent: true
1658
+ });
1659
+ return candles.filter(
1660
+ ({ timestamp }) => timestamp >= start && timestamp < end
1661
+ );
1662
+ };
1663
+ if (dataset.id.includes("-candles-")) {
1664
+ const candles = (await fetchCandles(series, day)).filter(
1665
+ ({ timestamp }) => missing(timestamp)
1666
+ );
1667
+ await (0, import_candles.upsertCandles)(
1668
+ (0, import_candles.toRows)(provider, series, Number(connectorInterval), candles)
1669
+ );
1670
+ return;
1671
+ }
1672
+ if (interval === "1d") return;
1673
+ if (dataset.id.startsWith("binance-trade-flow-")) {
1674
+ await (0, import_marketContext2.upsertMarketTradeFlowRows)(
1675
+ buildKlineTradeFlowRows({
1676
+ symbol: series,
1677
+ interval,
1678
+ candles: (await fetchCandles(series, day)).filter(
1679
+ ({ timestamp }) => missing(timestamp)
1680
+ )
1681
+ })
1682
+ );
1683
+ return;
1684
+ }
1685
+ const universe = (0, import_strategies.getBinanceBreadthUniverses)().find(
1686
+ (item) => item.universe === series
1687
+ );
1688
+ if (!universe)
1689
+ throw new Error("The historical Binance basket is unavailable.");
1690
+ const candlesBySymbol = {};
1691
+ const warmup = day - 2 * DAY_MS3;
1692
+ for (let offset = 0; offset < universe.symbols.length; offset += 4) {
1693
+ const entries = await Promise.all(
1694
+ universe.symbols.slice(offset, offset + 4).map(
1695
+ async (symbol) => [symbol, await fetchCandles(symbol, warmup)]
1696
+ )
1697
+ );
1698
+ Object.assign(candlesBySymbol, Object.fromEntries(entries));
1699
+ }
1700
+ const rows = buildMarketBreadthRows({
1701
+ universe: series,
1702
+ interval,
1703
+ candlesBySymbol,
1704
+ btcCandles: candlesBySymbol.BTCUSDT ?? await fetchCandles("BTCUSDT", warmup)
1705
+ });
1706
+ await (0, import_marketContext2.upsertMarketBreadthRows)(rows.filter(({ ts }) => missing(ts.getTime())));
1707
+ }
1708
+ async function main() {
1709
+ const [userName, dataset, series, day] = process.argv.slice(2);
1710
+ const request = { dataset, series, day: Number(day) };
1711
+ const target = (0, import_dataCoverage.resolveDataBackfill)(request);
1712
+ if (!target || !userName)
1713
+ throw new Error(
1714
+ "Expected user, dataset, series and UTC day in milliseconds."
1715
+ );
1716
+ await (0, import_client3.waitForDbReady)();
1717
+ const before = await (0, import_dataCoverage.readDataBackfillSlots)(request, target.end);
1718
+ if (before.loaded.size < before.expected)
1719
+ await fetchTarget(userName, request, target, before.loaded);
1720
+ const after = await (0, import_dataCoverage.readDataBackfillSlots)(request, target.end);
1721
+ const result = {
1722
+ loaded: after.loaded.size,
1723
+ added: Math.max(0, after.loaded.size - before.loaded.size),
1724
+ missing: Math.max(0, after.expected - after.loaded.size)
1725
+ };
1726
+ console.log(`TRADEJS_DATA_BACKFILL ${JSON.stringify(result)}`);
1727
+ }
1728
+ // Annotate the CommonJS export names for ESM import in node:
1729
+ 0 && (module.exports = {
1730
+ main
1731
+ });