@tradejs/cli 1.0.12 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2495,7 +2495,7 @@ var import_userSettings = require("@tradejs/infra/userSettings");
2495
2495
  var HOUR_MS = 60 * 60 * 1e3;
2496
2496
  var DAY_MS2 = 24 * HOUR_MS;
2497
2497
  var DEFAULT_LOOKBACK_HOURS = 48;
2498
- var DEFAULT_INTERVALS = ["15m", "1h"];
2498
+ var SOURCE_INTERVALS = ["15m"];
2499
2499
  var coinalyzeIntervalMap = {
2500
2500
  "15m": "15min",
2501
2501
  "1h": "1hour"
@@ -2556,10 +2556,7 @@ var isDerivativesContextBackfillEnabled = (env) => {
2556
2556
  var shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
2557
2557
  var shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
2558
2558
  var resolveDerivativesContextIntervals = () => {
2559
- const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
2560
- process.env.DERIVATIVES_CONTEXT_INTERVALS
2561
- );
2562
- return intervals.length ? intervals : DEFAULT_INTERVALS;
2559
+ return [...SOURCE_INTERVALS];
2563
2560
  };
2564
2561
  var resolveDerivativesContextLookbackMs = () => {
2565
2562
  const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
@@ -2594,9 +2591,10 @@ var resolveDerivativesContextBackfillWindow = (params) => {
2594
2591
  };
2595
2592
  var resolveDerivativesContextIntervalBackfillWindow = (params) => {
2596
2593
  const intervalMs = derivativesIntervalMs(params.interval);
2594
+ const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs) * intervalMs;
2597
2595
  return {
2598
2596
  fromMs: Math.floor(params.fromMs / intervalMs) * intervalMs,
2599
- toMs: Math.floor(params.toMs / intervalMs) * intervalMs,
2597
+ toMs: alignedToMs,
2600
2598
  intervalMs
2601
2599
  };
2602
2600
  };
@@ -2614,6 +2612,7 @@ var resolveDerivativesContextMissingFetchFromMs = (params) => {
2614
2612
  }
2615
2613
  return null;
2616
2614
  };
2615
+ var resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
2617
2616
  var countBackfillWindows = (params) => {
2618
2617
  let count = 0;
2619
2618
  let cursor = params.fromMs;
@@ -2677,6 +2676,14 @@ var groupDerivativesContextMissingFetchRanges = (ranges) => {
2677
2676
  }
2678
2677
  return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
2679
2678
  };
2679
+ var getMissingClosedDerivativesSymbols = (params) => {
2680
+ const availableSymbols = new Set(
2681
+ params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
2682
+ );
2683
+ return params.symbols.filter(
2684
+ (symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
2685
+ );
2686
+ };
2680
2687
  var coverageKey = (params) => [
2681
2688
  params.symbol.trim().toUpperCase(),
2682
2689
  params.interval,
@@ -2956,7 +2963,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
2956
2963
  ...resolveDerivativesContextIntervalBackfillWindow({
2957
2964
  fromMs,
2958
2965
  toMs: safeEndMs,
2959
- interval: interval2
2966
+ interval: interval2,
2967
+ closedOnly: mode === "signals"
2960
2968
  })
2961
2969
  })).filter((item) => item.toMs > item.fromMs);
2962
2970
  if (!intervalWindows.length) {
@@ -3015,7 +3023,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3015
3023
  }),
3016
3024
  0
3017
3025
  );
3018
- const allBackfillWindowsCached = intervalWindows.every((window2) => {
3026
+ const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window2) => {
3019
3027
  const edgesBySymbol = edgesByInterval.get(window2.interval);
3020
3028
  const coverageKeys = coverageKeysByInterval.get(window2.interval);
3021
3029
  const backfillWindows = buildBackfillWindows({
@@ -3145,7 +3153,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3145
3153
  return null;
3146
3154
  }
3147
3155
  const normalizedSymbol = item.symbol.toUpperCase();
3148
- const coverageFromMs = resolveDerivativesContextMissingCoverageFetchFromMs({
3156
+ const coverageFromMs = mode === "signals" ? cursor : resolveDerivativesContextMissingCoverageFetchFromMs({
3149
3157
  ranges: coverageRangesBySymbol.get(normalizedSymbol),
3150
3158
  fromMs: cursor,
3151
3159
  toMs,
@@ -3155,16 +3163,20 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3155
3163
  return null;
3156
3164
  }
3157
3165
  const edges = edgesBySymbol.get(normalizedSymbol);
3158
- const edgesFromMs = resolveDerivativesContextMissingFetchFromMs({
3166
+ const edgesFromMs = resolveDerivativesContextFetchFromMs({
3159
3167
  edges,
3160
3168
  fromMs: cursor,
3161
3169
  toMs,
3162
- intervalMs
3170
+ intervalMs,
3171
+ refreshClosedTail: mode === "signals"
3163
3172
  });
3164
3173
  if (edgesFromMs == null) {
3165
3174
  return null;
3166
3175
  }
3167
- return { item, fromMs: Math.max(coverageFromMs, edgesFromMs) };
3176
+ return {
3177
+ item,
3178
+ fromMs: Math.max(coverageFromMs, edgesFromMs)
3179
+ };
3168
3180
  }).filter(
3169
3181
  (item) => item != null
3170
3182
  );
@@ -3178,85 +3190,113 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3178
3190
  const marketSymbols = missingBatch.map(
3179
3191
  (item) => item.marketSymbol
3180
3192
  );
3181
- const oiMap = await fetchMetricBatch({
3182
- endpoint: oiPath,
3183
- metric: "oi",
3184
- marketSymbols,
3185
- apiKey,
3186
- interval: interval2,
3187
- fromMs: group.fromMs,
3188
- toMs
3189
- });
3190
- const fundingMap = await fetchMetricBatch({
3191
- endpoint: fundingPath,
3192
- metric: "funding",
3193
- marketSymbols,
3194
- apiKey,
3195
- interval: interval2,
3196
- fromMs: group.fromMs,
3197
- toMs
3198
- });
3199
- const liqMap = await fetchMetricBatch({
3200
- endpoint: liqPath,
3201
- metric: "liq",
3202
- marketSymbols,
3203
- apiKey,
3204
- interval: interval2,
3205
- fromMs: group.fromMs,
3206
- toMs
3207
- });
3208
- const rows = missingBatch.flatMap((item) => {
3209
- const marketSymbol = item.marketSymbol.toUpperCase();
3210
- const points = (0, import_indicators.mergeCoinalyzeMetrics)({
3211
- symbol: item.symbol,
3212
- oiRaw: oiMap.get(marketSymbol) ?? [],
3213
- fundingRaw: fundingMap.get(marketSymbol) ?? [],
3214
- liqRaw: liqMap.get(marketSymbol) ?? []
3193
+ const maxClosedBarAttempts = mode === "signals" ? asInt2(
3194
+ process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
3195
+ 3
3196
+ ) : 1;
3197
+ const closedBarRetryDelayMs = asInt2(
3198
+ process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
3199
+ 2e3
3200
+ );
3201
+ let rows = [];
3202
+ let missingClosedSymbols = [];
3203
+ for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
3204
+ const oiMap = await fetchMetricBatch({
3205
+ endpoint: oiPath,
3206
+ metric: "oi",
3207
+ marketSymbols,
3208
+ apiKey,
3209
+ interval: interval2,
3210
+ fromMs: group.fromMs,
3211
+ toMs
3215
3212
  });
3216
- return (0, import_indicators.coinalyzePointsToRows)(points, interval2, "coinalyze");
3217
- });
3213
+ const fundingMap = await fetchMetricBatch({
3214
+ endpoint: fundingPath,
3215
+ metric: "funding",
3216
+ marketSymbols,
3217
+ apiKey,
3218
+ interval: interval2,
3219
+ fromMs: group.fromMs,
3220
+ toMs
3221
+ });
3222
+ const liqMap = await fetchMetricBatch({
3223
+ endpoint: liqPath,
3224
+ metric: "liq",
3225
+ marketSymbols,
3226
+ apiKey,
3227
+ interval: interval2,
3228
+ fromMs: group.fromMs,
3229
+ toMs
3230
+ });
3231
+ rows = missingBatch.flatMap((item) => {
3232
+ const marketSymbol = item.marketSymbol.toUpperCase();
3233
+ const points = (0, import_indicators.mergeCoinalyzeMetrics)({
3234
+ symbol: item.symbol,
3235
+ oiRaw: oiMap.get(marketSymbol) ?? [],
3236
+ fundingRaw: fundingMap.get(marketSymbol) ?? [],
3237
+ liqRaw: liqMap.get(marketSymbol) ?? []
3238
+ });
3239
+ return (0, import_indicators.coinalyzePointsToRows)(points, interval2, "coinalyze");
3240
+ });
3241
+ missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
3242
+ symbols: missingBatch.map((item) => item.symbol),
3243
+ rows,
3244
+ expectedTimestamp: toMs
3245
+ }) : [];
3246
+ if (!missingClosedSymbols.length) break;
3247
+ if (attempt < maxClosedBarAttempts) {
3248
+ await (0, import_async.delay)(closedBarRetryDelayMs);
3249
+ }
3250
+ }
3251
+ if (missingClosedSymbols.length) {
3252
+ throw new Error(
3253
+ `Coinalyze closed ${interval2} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
3254
+ );
3255
+ }
3218
3256
  if (rows.length) {
3219
3257
  await (0, import_timescale2.upsertDerivatives)(rows);
3220
3258
  totalRows += rows.length;
3221
3259
  }
3222
- const rowsCountBySymbol = /* @__PURE__ */ new Map();
3223
- for (const row of rows) {
3224
- const symbol = row.symbol.toUpperCase();
3225
- rowsCountBySymbol.set(
3226
- symbol,
3227
- (rowsCountBySymbol.get(symbol) ?? 0) + 1
3228
- );
3229
- }
3230
- const coverageRows = missingBatch.map((item) => {
3231
- const normalizedSymbol = item.symbol.toUpperCase();
3232
- const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
3233
- return {
3234
- source: "coinalyze",
3235
- symbol: item.symbol,
3236
- interval: interval2,
3237
- fromMs: cursor,
3238
- toMs,
3239
- rowsCount
3240
- };
3241
- });
3242
- await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
3243
- for (const coverageRow of coverageRows) {
3244
- const symbol = coverageRow.symbol.toUpperCase();
3245
- edgesBySymbol.set(
3246
- symbol,
3247
- extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
3248
- );
3249
- coverageKeys.add(
3250
- coverageKey({
3260
+ if (mode === "backtest") {
3261
+ const rowsCountBySymbol = /* @__PURE__ */ new Map();
3262
+ for (const row of rows) {
3263
+ const symbol = row.symbol.toUpperCase();
3264
+ rowsCountBySymbol.set(
3251
3265
  symbol,
3266
+ (rowsCountBySymbol.get(symbol) ?? 0) + 1
3267
+ );
3268
+ }
3269
+ const coverageRows = missingBatch.map((item) => {
3270
+ const normalizedSymbol = item.symbol.toUpperCase();
3271
+ const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
3272
+ return {
3273
+ source: "coinalyze",
3274
+ symbol: item.symbol,
3252
3275
  interval: interval2,
3253
3276
  fromMs: cursor,
3254
- toMs
3255
- })
3256
- );
3257
- const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
3258
- coverageRanges.push({ fromMs: cursor, toMs });
3259
- coverageRangesBySymbol.set(symbol, coverageRanges);
3277
+ toMs,
3278
+ rowsCount
3279
+ };
3280
+ });
3281
+ await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
3282
+ for (const coverageRow of coverageRows) {
3283
+ const symbol = coverageRow.symbol.toUpperCase();
3284
+ edgesBySymbol.set(
3285
+ symbol,
3286
+ extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
3287
+ );
3288
+ coverageKeys.add(
3289
+ coverageKey({
3290
+ symbol,
3291
+ interval: interval2,
3292
+ fromMs: cursor,
3293
+ toMs
3294
+ })
3295
+ );
3296
+ const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
3297
+ coverageRanges.push({ fromMs: cursor, toMs });
3298
+ coverageRangesBySymbol.set(symbol, coverageRanges);
3299
+ }
3260
3300
  }
3261
3301
  }
3262
3302
  }
@@ -41,11 +41,11 @@ var import_timescale = require("@tradejs/infra/timescale");
41
41
  var import_userSettings = require("@tradejs/infra/userSettings");
42
42
  var import_connectors = require("@tradejs/connectors");
43
43
  import_args.default.example(
44
- "yarn ts-node ./src/scripts/derivativesIngest --provider coinalyze --symbols BTCUSDT,ETHUSDT --intervals 15m,1h --days 120",
44
+ "yarn ts-node ./src/scripts/derivativesIngest --provider coinalyze --symbols BTCUSDT,ETHUSDT --intervals 15m --days 120",
45
45
  "Ingest market features (derivatives/spread) into Timescale by provider"
46
46
  );
47
47
  import_args.default.option(["s", "symbols"], "Comma-separated symbols", "BTCUSDT,ETHUSDT");
48
- import_args.default.option(["t", "intervals"], "Comma-separated intervals: 15m,1h", "15m,1h");
48
+ import_args.default.option(["t", "intervals"], "Comma-separated intervals: 15m,1h", "15m");
49
49
  import_args.default.option(["d", "days"], "Lookback in days", 120);
50
50
  import_args.default.option(
51
51
  ["p", "provider"],
@@ -82,9 +82,16 @@ var main = async () => {
82
82
  let totalSpreadRows = 0;
83
83
  for (const symbol of symbols) {
84
84
  for (const interval of intervals) {
85
+ const lastClosedStartMs = (0, import_indicators.getLastClosedDerivativesBarStartMs)(
86
+ now,
87
+ interval
88
+ );
85
89
  let cursor = fromMs;
86
- while (cursor < now) {
87
- const toMs = Math.min(now, cursor + batchDays * 24 * 60 * 60 * 1e3);
90
+ while (cursor < lastClosedStartMs) {
91
+ const toMs = Math.min(
92
+ lastClosedStartMs,
93
+ cursor + batchDays * 24 * 60 * 60 * 1e3
94
+ );
88
95
  process.stdout.write(
89
96
  `\r${import_chalk.default.cyan(providerName)} ${import_chalk.default.yellow(symbol)} ${interval} ${new Date(cursor).toISOString()} .. ${new Date(toMs).toISOString()} `
90
97
  );
@@ -49,7 +49,7 @@ var coinalyzeIntervalMap = {
49
49
  "1h": "1hour"
50
50
  };
51
51
  import_args.default.example(
52
- "yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m,1h",
52
+ "yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m",
53
53
  "Fetch derivatives for all getTickers symbols matched to Coinalyze markets"
54
54
  );
55
55
  import_args.default.option(["U", "user"], "User settings profile name from Redis", "root");
@@ -57,7 +57,7 @@ import_args.default.option(["t", "tickers"], "Comma-separated include symbols");
57
57
  import_args.default.option(["e", "exclude"], "Comma-separated exclude symbols");
58
58
  import_args.default.option(["l", "tickersLimit"], "Tickers limit");
59
59
  import_args.default.option(["c", "chunk"], "Chunk selector, e.g. 1/4");
60
- import_args.default.option(["i", "intervals"], "Intervals: 15m,1h", "15m,1h");
60
+ import_args.default.option(["i", "intervals"], "Intervals: 15m,1h", "15m");
61
61
  import_args.default.option(["d", "days"], "Lookback in days", 120);
62
62
  import_args.default.option(["b", "batchDays"], "Request chunk size in days", 120);
63
63
  import_args.default.option(
@@ -326,12 +326,16 @@ var main = async () => {
326
326
  let totalRows = 0;
327
327
  let failedWindows = 0;
328
328
  for (const interval of intervals) {
329
+ const lastClosedStartMs = (0, import_indicators.getLastClosedDerivativesBarStartMs)(now, interval);
329
330
  for (let batchIdx = 0; batchIdx < symbolBatches.length; batchIdx += 1) {
330
331
  const batch = symbolBatches[batchIdx];
331
332
  const marketSymbols = batch.map((item) => item.marketSymbol);
332
333
  let cursor = fromMs;
333
- while (cursor < now) {
334
- const toMs = Math.min(now, cursor + batchDays * 24 * 60 * 60 * 1e3);
334
+ while (cursor < lastClosedStartMs) {
335
+ const toMs = Math.min(
336
+ lastClosedStartMs,
337
+ cursor + batchDays * 24 * 60 * 60 * 1e3
338
+ );
335
339
  try {
336
340
  const oiMap = await fetchMetricBatch({
337
341
  endpoint: oiPath,
@@ -3106,7 +3106,7 @@ var import_userSettings = require("@tradejs/infra/userSettings");
3106
3106
  var HOUR_MS = 60 * 60 * 1e3;
3107
3107
  var DAY_MS2 = 24 * HOUR_MS;
3108
3108
  var DEFAULT_LOOKBACK_HOURS = 48;
3109
- var DEFAULT_INTERVALS = ["15m", "1h"];
3109
+ var SOURCE_INTERVALS = ["15m"];
3110
3110
  var coinalyzeIntervalMap = {
3111
3111
  "15m": "15min",
3112
3112
  "1h": "1hour"
@@ -3167,10 +3167,7 @@ var isDerivativesContextBackfillEnabled = (env) => {
3167
3167
  var shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
3168
3168
  var shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
3169
3169
  var resolveDerivativesContextIntervals = () => {
3170
- const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
3171
- process.env.DERIVATIVES_CONTEXT_INTERVALS
3172
- );
3173
- return intervals.length ? intervals : DEFAULT_INTERVALS;
3170
+ return [...SOURCE_INTERVALS];
3174
3171
  };
3175
3172
  var resolveDerivativesContextLookbackMs = () => {
3176
3173
  const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
@@ -3205,9 +3202,10 @@ var resolveDerivativesContextBackfillWindow = (params) => {
3205
3202
  };
3206
3203
  var resolveDerivativesContextIntervalBackfillWindow = (params) => {
3207
3204
  const intervalMs = derivativesIntervalMs(params.interval);
3205
+ const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs) * intervalMs;
3208
3206
  return {
3209
3207
  fromMs: Math.floor(params.fromMs / intervalMs) * intervalMs,
3210
- toMs: Math.floor(params.toMs / intervalMs) * intervalMs,
3208
+ toMs: alignedToMs,
3211
3209
  intervalMs
3212
3210
  };
3213
3211
  };
@@ -3225,6 +3223,7 @@ var resolveDerivativesContextMissingFetchFromMs = (params) => {
3225
3223
  }
3226
3224
  return null;
3227
3225
  };
3226
+ var resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
3228
3227
  var countBackfillWindows = (params) => {
3229
3228
  let count = 0;
3230
3229
  let cursor = params.fromMs;
@@ -3288,6 +3287,14 @@ var groupDerivativesContextMissingFetchRanges = (ranges) => {
3288
3287
  }
3289
3288
  return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
3290
3289
  };
3290
+ var getMissingClosedDerivativesSymbols = (params) => {
3291
+ const availableSymbols = new Set(
3292
+ params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
3293
+ );
3294
+ return params.symbols.filter(
3295
+ (symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
3296
+ );
3297
+ };
3291
3298
  var coverageKey = (params) => [
3292
3299
  params.symbol.trim().toUpperCase(),
3293
3300
  params.interval,
@@ -3567,7 +3574,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3567
3574
  ...resolveDerivativesContextIntervalBackfillWindow({
3568
3575
  fromMs,
3569
3576
  toMs: safeEndMs,
3570
- interval
3577
+ interval,
3578
+ closedOnly: mode === "signals"
3571
3579
  })
3572
3580
  })).filter((item) => item.toMs > item.fromMs);
3573
3581
  if (!intervalWindows.length) {
@@ -3626,7 +3634,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3626
3634
  }),
3627
3635
  0
3628
3636
  );
3629
- const allBackfillWindowsCached = intervalWindows.every((window2) => {
3637
+ const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window2) => {
3630
3638
  const edgesBySymbol = edgesByInterval.get(window2.interval);
3631
3639
  const coverageKeys = coverageKeysByInterval.get(window2.interval);
3632
3640
  const backfillWindows = buildBackfillWindows({
@@ -3756,7 +3764,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3756
3764
  return null;
3757
3765
  }
3758
3766
  const normalizedSymbol = item.symbol.toUpperCase();
3759
- const coverageFromMs = resolveDerivativesContextMissingCoverageFetchFromMs({
3767
+ const coverageFromMs = mode === "signals" ? cursor : resolveDerivativesContextMissingCoverageFetchFromMs({
3760
3768
  ranges: coverageRangesBySymbol.get(normalizedSymbol),
3761
3769
  fromMs: cursor,
3762
3770
  toMs,
@@ -3766,16 +3774,20 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3766
3774
  return null;
3767
3775
  }
3768
3776
  const edges = edgesBySymbol.get(normalizedSymbol);
3769
- const edgesFromMs = resolveDerivativesContextMissingFetchFromMs({
3777
+ const edgesFromMs = resolveDerivativesContextFetchFromMs({
3770
3778
  edges,
3771
3779
  fromMs: cursor,
3772
3780
  toMs,
3773
- intervalMs
3781
+ intervalMs,
3782
+ refreshClosedTail: mode === "signals"
3774
3783
  });
3775
3784
  if (edgesFromMs == null) {
3776
3785
  return null;
3777
3786
  }
3778
- return { item, fromMs: Math.max(coverageFromMs, edgesFromMs) };
3787
+ return {
3788
+ item,
3789
+ fromMs: Math.max(coverageFromMs, edgesFromMs)
3790
+ };
3779
3791
  }).filter(
3780
3792
  (item) => item != null
3781
3793
  );
@@ -3789,85 +3801,113 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
3789
3801
  const marketSymbols = missingBatch.map(
3790
3802
  (item) => item.marketSymbol
3791
3803
  );
3792
- const oiMap = await fetchMetricBatch({
3793
- endpoint: oiPath,
3794
- metric: "oi",
3795
- marketSymbols,
3796
- apiKey,
3797
- interval,
3798
- fromMs: group.fromMs,
3799
- toMs
3800
- });
3801
- const fundingMap = await fetchMetricBatch({
3802
- endpoint: fundingPath,
3803
- metric: "funding",
3804
- marketSymbols,
3805
- apiKey,
3806
- interval,
3807
- fromMs: group.fromMs,
3808
- toMs
3809
- });
3810
- const liqMap = await fetchMetricBatch({
3811
- endpoint: liqPath,
3812
- metric: "liq",
3813
- marketSymbols,
3814
- apiKey,
3815
- interval,
3816
- fromMs: group.fromMs,
3817
- toMs
3818
- });
3819
- const rows = missingBatch.flatMap((item) => {
3820
- const marketSymbol = item.marketSymbol.toUpperCase();
3821
- const points = (0, import_indicators.mergeCoinalyzeMetrics)({
3822
- symbol: item.symbol,
3823
- oiRaw: oiMap.get(marketSymbol) ?? [],
3824
- fundingRaw: fundingMap.get(marketSymbol) ?? [],
3825
- liqRaw: liqMap.get(marketSymbol) ?? []
3804
+ const maxClosedBarAttempts = mode === "signals" ? asInt2(
3805
+ process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
3806
+ 3
3807
+ ) : 1;
3808
+ const closedBarRetryDelayMs = asInt2(
3809
+ process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
3810
+ 2e3
3811
+ );
3812
+ let rows = [];
3813
+ let missingClosedSymbols = [];
3814
+ for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
3815
+ const oiMap = await fetchMetricBatch({
3816
+ endpoint: oiPath,
3817
+ metric: "oi",
3818
+ marketSymbols,
3819
+ apiKey,
3820
+ interval,
3821
+ fromMs: group.fromMs,
3822
+ toMs
3823
+ });
3824
+ const fundingMap = await fetchMetricBatch({
3825
+ endpoint: fundingPath,
3826
+ metric: "funding",
3827
+ marketSymbols,
3828
+ apiKey,
3829
+ interval,
3830
+ fromMs: group.fromMs,
3831
+ toMs
3826
3832
  });
3827
- return (0, import_indicators.coinalyzePointsToRows)(points, interval, "coinalyze");
3828
- });
3833
+ const liqMap = await fetchMetricBatch({
3834
+ endpoint: liqPath,
3835
+ metric: "liq",
3836
+ marketSymbols,
3837
+ apiKey,
3838
+ interval,
3839
+ fromMs: group.fromMs,
3840
+ toMs
3841
+ });
3842
+ rows = missingBatch.flatMap((item) => {
3843
+ const marketSymbol = item.marketSymbol.toUpperCase();
3844
+ const points = (0, import_indicators.mergeCoinalyzeMetrics)({
3845
+ symbol: item.symbol,
3846
+ oiRaw: oiMap.get(marketSymbol) ?? [],
3847
+ fundingRaw: fundingMap.get(marketSymbol) ?? [],
3848
+ liqRaw: liqMap.get(marketSymbol) ?? []
3849
+ });
3850
+ return (0, import_indicators.coinalyzePointsToRows)(points, interval, "coinalyze");
3851
+ });
3852
+ missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
3853
+ symbols: missingBatch.map((item) => item.symbol),
3854
+ rows,
3855
+ expectedTimestamp: toMs
3856
+ }) : [];
3857
+ if (!missingClosedSymbols.length) break;
3858
+ if (attempt < maxClosedBarAttempts) {
3859
+ await (0, import_async.delay)(closedBarRetryDelayMs);
3860
+ }
3861
+ }
3862
+ if (missingClosedSymbols.length) {
3863
+ throw new Error(
3864
+ `Coinalyze closed ${interval} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
3865
+ );
3866
+ }
3829
3867
  if (rows.length) {
3830
3868
  await (0, import_timescale2.upsertDerivatives)(rows);
3831
3869
  totalRows += rows.length;
3832
3870
  }
3833
- const rowsCountBySymbol = /* @__PURE__ */ new Map();
3834
- for (const row of rows) {
3835
- const symbol = row.symbol.toUpperCase();
3836
- rowsCountBySymbol.set(
3837
- symbol,
3838
- (rowsCountBySymbol.get(symbol) ?? 0) + 1
3839
- );
3840
- }
3841
- const coverageRows = missingBatch.map((item) => {
3842
- const normalizedSymbol = item.symbol.toUpperCase();
3843
- const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
3844
- return {
3845
- source: "coinalyze",
3846
- symbol: item.symbol,
3847
- interval,
3848
- fromMs: cursor,
3849
- toMs,
3850
- rowsCount
3851
- };
3852
- });
3853
- await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
3854
- for (const coverageRow of coverageRows) {
3855
- const symbol = coverageRow.symbol.toUpperCase();
3856
- edgesBySymbol.set(
3857
- symbol,
3858
- extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
3859
- );
3860
- coverageKeys.add(
3861
- coverageKey({
3871
+ if (mode === "backtest") {
3872
+ const rowsCountBySymbol = /* @__PURE__ */ new Map();
3873
+ for (const row of rows) {
3874
+ const symbol = row.symbol.toUpperCase();
3875
+ rowsCountBySymbol.set(
3862
3876
  symbol,
3877
+ (rowsCountBySymbol.get(symbol) ?? 0) + 1
3878
+ );
3879
+ }
3880
+ const coverageRows = missingBatch.map((item) => {
3881
+ const normalizedSymbol = item.symbol.toUpperCase();
3882
+ const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
3883
+ return {
3884
+ source: "coinalyze",
3885
+ symbol: item.symbol,
3863
3886
  interval,
3864
3887
  fromMs: cursor,
3865
- toMs
3866
- })
3867
- );
3868
- const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
3869
- coverageRanges.push({ fromMs: cursor, toMs });
3870
- coverageRangesBySymbol.set(symbol, coverageRanges);
3888
+ toMs,
3889
+ rowsCount
3890
+ };
3891
+ });
3892
+ await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
3893
+ for (const coverageRow of coverageRows) {
3894
+ const symbol = coverageRow.symbol.toUpperCase();
3895
+ edgesBySymbol.set(
3896
+ symbol,
3897
+ extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
3898
+ );
3899
+ coverageKeys.add(
3900
+ coverageKey({
3901
+ symbol,
3902
+ interval,
3903
+ fromMs: cursor,
3904
+ toMs
3905
+ })
3906
+ );
3907
+ const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
3908
+ coverageRanges.push({ fromMs: cursor, toMs });
3909
+ coverageRangesBySymbol.set(symbol, coverageRanges);
3910
+ }
3871
3911
  }
3872
3912
  }
3873
3913
  }