@tradejs/cli 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +384 -122
- package/dist/lib/aiTrainCharts.js +49 -2
- package/dist/lib/aiTrainMetrics.js +49 -2
- package/dist/lib/aiTrainResearch.js +58 -4
- package/dist/lib/derivativesContextBackfill.js +262 -107
- package/dist/lib/marketContextPrepare.js +256 -107
- package/dist/scripts/aiPocketSearch.js +49 -2
- package/dist/scripts/aiTrain.js +76 -4
- package/dist/scripts/backtest.js +256 -107
- package/dist/scripts/derivativesIngest.js +11 -4
- package/dist/scripts/derivativesIngestCoinalyzeAll.js +38 -4
- package/dist/scripts/replay.js +256 -107
- package/dist/scripts/replayRunner.js +256 -107
- package/dist/scripts/runtimeParity.js +256 -107
- package/dist/scripts/signals.js +256 -107
- package/dist/scripts/signalsDaemon.js +256 -107
- package/package.json +9 -9
package/dist/scripts/aiTrain.js
CHANGED
|
@@ -1653,6 +1653,7 @@ var divideOrNull = (num, denom) => {
|
|
|
1653
1653
|
var DAY_MS = 24 * 60 * 60 * 1e3;
|
|
1654
1654
|
var DAYS_PER_WEEK = 7;
|
|
1655
1655
|
var DAYS_PER_MONTH = 30.4375;
|
|
1656
|
+
var DAYS_PER_YEAR = 365;
|
|
1656
1657
|
var getEvaluationPeriodDays = (evaluations) => {
|
|
1657
1658
|
let minTimestamp = null;
|
|
1658
1659
|
let maxTimestamp = null;
|
|
@@ -1674,6 +1675,40 @@ var getEvaluationPeriodDays = (evaluations) => {
|
|
|
1674
1675
|
return Math.max((maxTimestamp - minTimestamp) / DAY_MS, 1);
|
|
1675
1676
|
};
|
|
1676
1677
|
var qualitySortKey = (quality) => quality == null ? Number.POSITIVE_INFINITY : quality;
|
|
1678
|
+
var calculateTradePnlRiskRatios = ({
|
|
1679
|
+
profits,
|
|
1680
|
+
totalProfit,
|
|
1681
|
+
maxDrawdown,
|
|
1682
|
+
periodDays
|
|
1683
|
+
}) => {
|
|
1684
|
+
if (!profits.length || periodDays == null || periodDays <= 0) {
|
|
1685
|
+
return {
|
|
1686
|
+
sharpeRatio: null,
|
|
1687
|
+
sortinoRatio: null,
|
|
1688
|
+
calmarRatio: null
|
|
1689
|
+
};
|
|
1690
|
+
}
|
|
1691
|
+
const meanProfit = totalProfit / profits.length;
|
|
1692
|
+
const variance = profits.reduce((sum, profit) => {
|
|
1693
|
+
const diff = profit - meanProfit;
|
|
1694
|
+
return sum + diff * diff;
|
|
1695
|
+
}, 0) / profits.length;
|
|
1696
|
+
const stdDev = Math.sqrt(variance);
|
|
1697
|
+
const downsideDeviation = Math.sqrt(
|
|
1698
|
+
profits.reduce(
|
|
1699
|
+
(sum, profit) => profit < 0 ? sum + profit * profit : sum,
|
|
1700
|
+
0
|
|
1701
|
+
) / profits.length
|
|
1702
|
+
);
|
|
1703
|
+
const annualizationScale = Math.sqrt(
|
|
1704
|
+
profits.length / periodDays * DAYS_PER_YEAR
|
|
1705
|
+
);
|
|
1706
|
+
return {
|
|
1707
|
+
sharpeRatio: stdDev > 0 && annualizationScale > 0 ? meanProfit / stdDev * annualizationScale : null,
|
|
1708
|
+
sortinoRatio: downsideDeviation > 0 && annualizationScale > 0 ? meanProfit / downsideDeviation * annualizationScale : null,
|
|
1709
|
+
calmarRatio: maxDrawdown > 0 ? totalProfit / periodDays * DAYS_PER_YEAR / maxDrawdown : null
|
|
1710
|
+
};
|
|
1711
|
+
};
|
|
1677
1712
|
var emptyRiskSummary = () => ({
|
|
1678
1713
|
trades: 0,
|
|
1679
1714
|
totalProfit: 0,
|
|
@@ -1689,12 +1724,15 @@ var emptyRiskSummary = () => ({
|
|
|
1689
1724
|
maxDrawdown: 0,
|
|
1690
1725
|
maxDrawdownPctOfGrossProfit: null,
|
|
1691
1726
|
maxDrawdownPctOfTotalProfit: null,
|
|
1727
|
+
sharpeRatio: null,
|
|
1728
|
+
sortinoRatio: null,
|
|
1729
|
+
calmarRatio: null,
|
|
1692
1730
|
recoveryFactor: null,
|
|
1693
1731
|
ulcerIndex: null,
|
|
1694
1732
|
maxConsecutiveWins: 0,
|
|
1695
1733
|
maxConsecutiveLosses: 0
|
|
1696
1734
|
});
|
|
1697
|
-
var summarizeApprovedRisk = (evaluations) => {
|
|
1735
|
+
var summarizeApprovedRisk = (evaluations, periodDays = getEvaluationPeriodDays(evaluations)) => {
|
|
1698
1736
|
const approvedEvaluations = evaluations.filter((evaluation) => evaluation.aiApproved).sort((left, right) => {
|
|
1699
1737
|
const leftTimestamp = typeof left.timestamp === "number" && Number.isFinite(left.timestamp) ? left.timestamp : Number.POSITIVE_INFINITY;
|
|
1700
1738
|
const rightTimestamp = typeof right.timestamp === "number" && Number.isFinite(right.timestamp) ? right.timestamp : Number.POSITIVE_INFINITY;
|
|
@@ -1717,8 +1755,10 @@ var summarizeApprovedRisk = (evaluations) => {
|
|
|
1717
1755
|
let currentLossStreak = 0;
|
|
1718
1756
|
let maxConsecutiveWins = 0;
|
|
1719
1757
|
let maxConsecutiveLosses = 0;
|
|
1758
|
+
const approvedProfits = [];
|
|
1720
1759
|
for (const evaluation of approvedEvaluations) {
|
|
1721
1760
|
const profit = evaluation.profit;
|
|
1761
|
+
approvedProfits.push(profit);
|
|
1722
1762
|
if (profit > 0) {
|
|
1723
1763
|
grossProfit += profit;
|
|
1724
1764
|
wins += 1;
|
|
@@ -1746,6 +1786,12 @@ var summarizeApprovedRisk = (evaluations) => {
|
|
|
1746
1786
|
const totalProfit = grossProfit - grossLoss;
|
|
1747
1787
|
const avgWin = divideOrNull(grossProfit, wins);
|
|
1748
1788
|
const avgLoss = divideOrNull(grossLoss, losses);
|
|
1789
|
+
const riskRatios = calculateTradePnlRiskRatios({
|
|
1790
|
+
profits: approvedProfits,
|
|
1791
|
+
totalProfit,
|
|
1792
|
+
maxDrawdown,
|
|
1793
|
+
periodDays
|
|
1794
|
+
});
|
|
1749
1795
|
return {
|
|
1750
1796
|
trades: approvedEvaluations.length,
|
|
1751
1797
|
totalProfit,
|
|
@@ -1761,6 +1807,7 @@ var summarizeApprovedRisk = (evaluations) => {
|
|
|
1761
1807
|
maxDrawdown,
|
|
1762
1808
|
maxDrawdownPctOfGrossProfit: grossProfit > 0 ? maxDrawdown / grossProfit : null,
|
|
1763
1809
|
maxDrawdownPctOfTotalProfit: totalProfit > 0 ? maxDrawdown / totalProfit : null,
|
|
1810
|
+
...riskRatios,
|
|
1764
1811
|
recoveryFactor: maxDrawdown > 0 ? totalProfit / maxDrawdown : null,
|
|
1765
1812
|
ulcerIndex: Math.sqrt(drawdownSquares / approvedEvaluations.length),
|
|
1766
1813
|
maxConsecutiveWins,
|
|
@@ -1859,7 +1906,7 @@ var summarizeAiTrainEvaluations = (evaluations) => {
|
|
|
1859
1906
|
avgApprovedTradesPerDay,
|
|
1860
1907
|
avgApprovedTradesPerWeek,
|
|
1861
1908
|
expectancyDelta,
|
|
1862
|
-
approvedRisk: summarizeApprovedRisk(evaluations),
|
|
1909
|
+
approvedRisk: summarizeApprovedRisk(evaluations, periodDays),
|
|
1863
1910
|
qualityBuckets: [...bucketMap.values()].sort(
|
|
1864
1911
|
(a, b) => qualitySortKey(a.quality) - qualitySortKey(b.quality)
|
|
1865
1912
|
)
|
|
@@ -2510,7 +2557,6 @@ var RESEARCH_CONTEXT_ENV_KEYS = [
|
|
|
2510
2557
|
"INTERVAL",
|
|
2511
2558
|
"DERIVATIVES_CONTEXT_ENABLED",
|
|
2512
2559
|
"DERIVATIVES_CONTEXT_TARGET_ENABLED",
|
|
2513
|
-
"DERIVATIVES_CONTEXT_INTERVALS",
|
|
2514
2560
|
"DERIVATIVES_CONTEXT_LOOKBACK_HOURS",
|
|
2515
2561
|
"DERIVATIVES_CONTEXT_EXTRA_REFERENCE_SYMBOLS",
|
|
2516
2562
|
"DERIVATIVES_CONTEXT_EXCHANGE_PRIORITY",
|
|
@@ -2524,6 +2570,12 @@ var RESEARCH_CONTEXT_ENV_KEYS = [
|
|
|
2524
2570
|
"COINMARKETCAP_CONTEXT_BACKFILL_MAX_DAYS",
|
|
2525
2571
|
"COINMARKETCAP_CONTEXT_BACKFILL_WARMUP_DAYS"
|
|
2526
2572
|
];
|
|
2573
|
+
var DERIVATIVES_CONTEXT_DATA_MODEL = {
|
|
2574
|
+
derivativesSourceIntervals: "15m",
|
|
2575
|
+
derivativesDerivedIntervals: "1h",
|
|
2576
|
+
derivativesHourlyFallback: "stored-1h",
|
|
2577
|
+
derivativesDataModelVersion: 2
|
|
2578
|
+
};
|
|
2527
2579
|
var normalizeForStableJson = (value) => {
|
|
2528
2580
|
if (Array.isArray(value)) {
|
|
2529
2581
|
return value.map(normalizeForStableJson);
|
|
@@ -2636,6 +2688,7 @@ var resolveGateFingerprint = async (projectRoot, strategyName, gitSha) => {
|
|
|
2636
2688
|
const relativeCandidates = [
|
|
2637
2689
|
`packages/strategies/src/${strategyName}/adapters/ai.ts`,
|
|
2638
2690
|
`packages/strategies/src/${strategyName}/guardrails.ts`,
|
|
2691
|
+
`packages/strategies/src/${strategyName}/pockets.ts`,
|
|
2639
2692
|
`packages/strategies/src/${strategyName}/config.ts`,
|
|
2640
2693
|
"packages/node/src/ai.ts"
|
|
2641
2694
|
];
|
|
@@ -2682,7 +2735,8 @@ var buildAiTrainLineage = async ({
|
|
|
2682
2735
|
...Object.fromEntries(
|
|
2683
2736
|
RESEARCH_CONTEXT_ENV_KEYS.map((key) => [key, env[key] ?? null])
|
|
2684
2737
|
),
|
|
2685
|
-
...runContext
|
|
2738
|
+
...runContext,
|
|
2739
|
+
...DERIVATIVES_CONTEXT_DATA_MODEL
|
|
2686
2740
|
};
|
|
2687
2741
|
return {
|
|
2688
2742
|
gitSha,
|
|
@@ -3742,6 +3796,9 @@ var main = async () => {
|
|
|
3742
3796
|
import_chalk.default.gray("CALENDAR/D"),
|
|
3743
3797
|
import_chalk.default.gray("WR"),
|
|
3744
3798
|
import_chalk.default.gray("PF"),
|
|
3799
|
+
import_chalk.default.gray("SHARPE"),
|
|
3800
|
+
import_chalk.default.gray("SORTINO"),
|
|
3801
|
+
import_chalk.default.gray("CALMAR"),
|
|
3745
3802
|
import_chalk.default.gray("PNL"),
|
|
3746
3803
|
import_chalk.default.gray("MAX_DD"),
|
|
3747
3804
|
import_chalk.default.gray("TOP REJECT")
|
|
@@ -3756,6 +3813,9 @@ var main = async () => {
|
|
|
3756
3813
|
colorizeMetricNumber(window2.approvedPerCalendarDay),
|
|
3757
3814
|
colorizeRatio(window2.outcome.approvedRisk.winRate),
|
|
3758
3815
|
colorizeMetricNumber(window2.outcome.approvedRisk.profitFactor),
|
|
3816
|
+
colorizeMetricNumber(window2.outcome.approvedRisk.sharpeRatio),
|
|
3817
|
+
colorizeMetricNumber(window2.outcome.approvedRisk.sortinoRatio),
|
|
3818
|
+
colorizeMetricNumber(window2.outcome.approvedRisk.calmarRatio),
|
|
3759
3819
|
colorizeProfit(window2.outcome.approvedRisk.totalProfit),
|
|
3760
3820
|
colorizeProfit(-window2.outcome.approvedRisk.maxDrawdown),
|
|
3761
3821
|
import_chalk.default.gray(
|
|
@@ -3804,6 +3864,18 @@ var main = async () => {
|
|
|
3804
3864
|
"profit_factor",
|
|
3805
3865
|
colorizeMetricNumber(summary.approvedRisk.profitFactor)
|
|
3806
3866
|
],
|
|
3867
|
+
[
|
|
3868
|
+
"sharpe_ratio",
|
|
3869
|
+
colorizeMetricNumber(summary.approvedRisk.sharpeRatio)
|
|
3870
|
+
],
|
|
3871
|
+
[
|
|
3872
|
+
"sortino_ratio",
|
|
3873
|
+
colorizeMetricNumber(summary.approvedRisk.sortinoRatio)
|
|
3874
|
+
],
|
|
3875
|
+
[
|
|
3876
|
+
"calmar_ratio",
|
|
3877
|
+
colorizeMetricNumber(summary.approvedRisk.calmarRatio)
|
|
3878
|
+
],
|
|
3807
3879
|
[
|
|
3808
3880
|
"payoff_ratio",
|
|
3809
3881
|
colorizeMetricNumber(summary.approvedRisk.payoffRatio)
|
package/dist/scripts/backtest.js
CHANGED
|
@@ -2495,7 +2495,7 @@ var import_userSettings = require("@tradejs/infra/userSettings");
|
|
|
2495
2495
|
var HOUR_MS = 60 * 60 * 1e3;
|
|
2496
2496
|
var DAY_MS2 = 24 * HOUR_MS;
|
|
2497
2497
|
var DEFAULT_LOOKBACK_HOURS = 48;
|
|
2498
|
-
var
|
|
2498
|
+
var SOURCE_INTERVALS = ["15m"];
|
|
2499
2499
|
var coinalyzeIntervalMap = {
|
|
2500
2500
|
"15m": "15min",
|
|
2501
2501
|
"1h": "1hour"
|
|
@@ -2556,10 +2556,7 @@ var isDerivativesContextBackfillEnabled = (env) => {
|
|
|
2556
2556
|
var shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
|
|
2557
2557
|
var shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
|
|
2558
2558
|
var resolveDerivativesContextIntervals = () => {
|
|
2559
|
-
|
|
2560
|
-
process.env.DERIVATIVES_CONTEXT_INTERVALS
|
|
2561
|
-
);
|
|
2562
|
-
return intervals.length ? intervals : DEFAULT_INTERVALS;
|
|
2559
|
+
return [...SOURCE_INTERVALS];
|
|
2563
2560
|
};
|
|
2564
2561
|
var resolveDerivativesContextLookbackMs = () => {
|
|
2565
2562
|
const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
|
|
@@ -2594,9 +2591,10 @@ var resolveDerivativesContextBackfillWindow = (params) => {
|
|
|
2594
2591
|
};
|
|
2595
2592
|
var resolveDerivativesContextIntervalBackfillWindow = (params) => {
|
|
2596
2593
|
const intervalMs = derivativesIntervalMs(params.interval);
|
|
2594
|
+
const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs) * intervalMs;
|
|
2597
2595
|
return {
|
|
2598
2596
|
fromMs: Math.floor(params.fromMs / intervalMs) * intervalMs,
|
|
2599
|
-
toMs:
|
|
2597
|
+
toMs: alignedToMs,
|
|
2600
2598
|
intervalMs
|
|
2601
2599
|
};
|
|
2602
2600
|
};
|
|
@@ -2614,6 +2612,39 @@ var resolveDerivativesContextMissingFetchFromMs = (params) => {
|
|
|
2614
2612
|
}
|
|
2615
2613
|
return null;
|
|
2616
2614
|
};
|
|
2615
|
+
var resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
|
|
2616
|
+
var resolveDerivativesContextRequiredFetchFromMs = (params) => {
|
|
2617
|
+
const dataCoverageFromMs = params.dataCoverageKeyExists ? null : params.mode === "signals" ? params.fromMs : resolveDerivativesContextMissingCoverageFetchFromMs({
|
|
2618
|
+
ranges: params.dataCoverageRanges,
|
|
2619
|
+
fromMs: params.fromMs,
|
|
2620
|
+
toMs: params.toMs,
|
|
2621
|
+
intervalMs: params.intervalMs
|
|
2622
|
+
});
|
|
2623
|
+
const edgesFromMs = dataCoverageFromMs == null ? null : resolveDerivativesContextFetchFromMs({
|
|
2624
|
+
edges: params.edges,
|
|
2625
|
+
fromMs: params.fromMs,
|
|
2626
|
+
toMs: params.toMs,
|
|
2627
|
+
intervalMs: params.intervalMs,
|
|
2628
|
+
refreshClosedTail: params.mode === "signals"
|
|
2629
|
+
});
|
|
2630
|
+
const dataFetchFromMs = dataCoverageFromMs != null && edgesFromMs != null ? Math.max(dataCoverageFromMs, edgesFromMs) : null;
|
|
2631
|
+
const confirmedLiquidationWindow = (0, import_indicators.resolveCoinalyzeConfirmedIntradayCoverage)({
|
|
2632
|
+
interval: params.interval,
|
|
2633
|
+
fromMs: params.fromMs,
|
|
2634
|
+
toMs: params.toMs,
|
|
2635
|
+
nowMs: params.nowMs
|
|
2636
|
+
});
|
|
2637
|
+
const liquidationFetchFromMs = confirmedLiquidationWindow ? resolveDerivativesContextMissingCoverageFetchFromMs({
|
|
2638
|
+
ranges: params.liquidationCoverageRanges,
|
|
2639
|
+
fromMs: confirmedLiquidationWindow.fromMs,
|
|
2640
|
+
toMs: confirmedLiquidationWindow.toMs,
|
|
2641
|
+
intervalMs: params.intervalMs
|
|
2642
|
+
}) : null;
|
|
2643
|
+
const fetchStarts = [dataFetchFromMs, liquidationFetchFromMs].filter(
|
|
2644
|
+
(value) => value != null
|
|
2645
|
+
);
|
|
2646
|
+
return fetchStarts.length ? Math.min(...fetchStarts) : null;
|
|
2647
|
+
};
|
|
2617
2648
|
var countBackfillWindows = (params) => {
|
|
2618
2649
|
let count = 0;
|
|
2619
2650
|
let cursor = params.fromMs;
|
|
@@ -2677,6 +2708,14 @@ var groupDerivativesContextMissingFetchRanges = (ranges) => {
|
|
|
2677
2708
|
}
|
|
2678
2709
|
return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
|
|
2679
2710
|
};
|
|
2711
|
+
var getMissingClosedDerivativesSymbols = (params) => {
|
|
2712
|
+
const availableSymbols = new Set(
|
|
2713
|
+
params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
|
|
2714
|
+
);
|
|
2715
|
+
return params.symbols.filter(
|
|
2716
|
+
(symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
|
|
2717
|
+
);
|
|
2718
|
+
};
|
|
2680
2719
|
var coverageKey = (params) => [
|
|
2681
2720
|
params.symbol.trim().toUpperCase(),
|
|
2682
2721
|
params.interval,
|
|
@@ -2941,7 +2980,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
2941
2980
|
mode,
|
|
2942
2981
|
startMs,
|
|
2943
2982
|
endMs,
|
|
2944
|
-
preloadStartMs: params.preloadStartMs
|
|
2983
|
+
preloadStartMs: params.preloadStartMs,
|
|
2984
|
+
nowMs: Date.now()
|
|
2945
2985
|
});
|
|
2946
2986
|
if (safeEndMs <= fromMs) {
|
|
2947
2987
|
return skippedBackfillResult2();
|
|
@@ -2956,7 +2996,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
2956
2996
|
...resolveDerivativesContextIntervalBackfillWindow({
|
|
2957
2997
|
fromMs,
|
|
2958
2998
|
toMs: safeEndMs,
|
|
2959
|
-
interval: interval2
|
|
2999
|
+
interval: interval2,
|
|
3000
|
+
closedOnly: mode === "signals"
|
|
2960
3001
|
})
|
|
2961
3002
|
})).filter((item) => item.toMs > item.fromMs);
|
|
2962
3003
|
if (!intervalWindows.length) {
|
|
@@ -2974,15 +3015,33 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
2974
3015
|
);
|
|
2975
3016
|
const coverageKeysByInterval = /* @__PURE__ */ new Map();
|
|
2976
3017
|
const coverageRangesByInterval = /* @__PURE__ */ new Map();
|
|
3018
|
+
const liquidationCoverageRangesByInterval = /* @__PURE__ */ new Map();
|
|
3019
|
+
const coverageNowMs = Date.now();
|
|
2977
3020
|
await Promise.all(
|
|
2978
3021
|
intervalWindows.map(async (window2) => {
|
|
2979
|
-
const
|
|
2980
|
-
source: "coinalyze",
|
|
2981
|
-
symbols,
|
|
3022
|
+
const confirmedLiquidationWindow = (0, import_indicators.resolveCoinalyzeConfirmedIntradayCoverage)({
|
|
2982
3023
|
interval: window2.interval,
|
|
2983
3024
|
fromMs: window2.fromMs,
|
|
2984
|
-
toMs: window2.toMs
|
|
3025
|
+
toMs: window2.toMs,
|
|
3026
|
+
nowMs: coverageNowMs
|
|
2985
3027
|
});
|
|
3028
|
+
const [coverageRows, liquidationCoverageRows] = await Promise.all([
|
|
3029
|
+
(0, import_timescale2.getDerivativesBackfillCoverage)({
|
|
3030
|
+
source: "coinalyze",
|
|
3031
|
+
symbols,
|
|
3032
|
+
interval: window2.interval,
|
|
3033
|
+
fromMs: window2.fromMs,
|
|
3034
|
+
toMs: window2.toMs
|
|
3035
|
+
}),
|
|
3036
|
+
confirmedLiquidationWindow ? (0, import_timescale2.getDerivativesMetricCoverage)({
|
|
3037
|
+
source: "coinalyze",
|
|
3038
|
+
metric: "liquidation",
|
|
3039
|
+
symbols,
|
|
3040
|
+
interval: window2.interval,
|
|
3041
|
+
fromMs: confirmedLiquidationWindow.fromMs,
|
|
3042
|
+
toMs: confirmedLiquidationWindow.toMs
|
|
3043
|
+
}) : Promise.resolve([])
|
|
3044
|
+
]);
|
|
2986
3045
|
coverageKeysByInterval.set(
|
|
2987
3046
|
window2.interval,
|
|
2988
3047
|
new Set(
|
|
@@ -3004,6 +3063,17 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3004
3063
|
rangesBySymbol.set(symbol, ranges);
|
|
3005
3064
|
}
|
|
3006
3065
|
coverageRangesByInterval.set(window2.interval, rangesBySymbol);
|
|
3066
|
+
const liquidationRangesBySymbol = /* @__PURE__ */ new Map();
|
|
3067
|
+
for (const row of liquidationCoverageRows) {
|
|
3068
|
+
const symbol = row.symbol.toUpperCase();
|
|
3069
|
+
const ranges = liquidationRangesBySymbol.get(symbol) ?? [];
|
|
3070
|
+
ranges.push({ fromMs: row.fromMs, toMs: row.toMs });
|
|
3071
|
+
liquidationRangesBySymbol.set(symbol, ranges);
|
|
3072
|
+
}
|
|
3073
|
+
liquidationCoverageRangesByInterval.set(
|
|
3074
|
+
window2.interval,
|
|
3075
|
+
liquidationRangesBySymbol
|
|
3076
|
+
);
|
|
3007
3077
|
})
|
|
3008
3078
|
);
|
|
3009
3079
|
const cachedWindows = intervalWindows.reduce(
|
|
@@ -3015,7 +3085,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3015
3085
|
}),
|
|
3016
3086
|
0
|
|
3017
3087
|
);
|
|
3018
|
-
const allBackfillWindowsCached = intervalWindows.every((window2) => {
|
|
3088
|
+
const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window2) => {
|
|
3019
3089
|
const edgesBySymbol = edgesByInterval.get(window2.interval);
|
|
3020
3090
|
const coverageKeys = coverageKeysByInterval.get(window2.interval);
|
|
3021
3091
|
const backfillWindows = buildBackfillWindows({
|
|
@@ -3028,7 +3098,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3028
3098
|
(symbol) => backfillWindows.every((backfillWindow) => {
|
|
3029
3099
|
const normalizedSymbol = symbol.toUpperCase();
|
|
3030
3100
|
const coverageRanges = coverageRangesByInterval.get(window2.interval)?.get(normalizedSymbol) ?? [];
|
|
3031
|
-
|
|
3101
|
+
const dataCovered = hasDerivativesWindowCoverage({
|
|
3032
3102
|
edges: edgesBySymbol?.get(normalizedSymbol),
|
|
3033
3103
|
fromMs: backfillWindow.fromMs,
|
|
3034
3104
|
toMs: backfillWindow.toMs
|
|
@@ -3045,6 +3115,19 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3045
3115
|
toMs: backfillWindow.toMs
|
|
3046
3116
|
})
|
|
3047
3117
|
);
|
|
3118
|
+
const confirmedLiquidationWindow = (0, import_indicators.resolveCoinalyzeConfirmedIntradayCoverage)({
|
|
3119
|
+
interval: window2.interval,
|
|
3120
|
+
fromMs: backfillWindow.fromMs,
|
|
3121
|
+
toMs: backfillWindow.toMs,
|
|
3122
|
+
nowMs: coverageNowMs
|
|
3123
|
+
});
|
|
3124
|
+
const liquidationCovered = confirmedLiquidationWindow == null || resolveDerivativesContextMissingCoverageFetchFromMs({
|
|
3125
|
+
ranges: liquidationCoverageRangesByInterval.get(window2.interval)?.get(normalizedSymbol) ?? [],
|
|
3126
|
+
fromMs: confirmedLiquidationWindow.fromMs,
|
|
3127
|
+
toMs: confirmedLiquidationWindow.toMs,
|
|
3128
|
+
intervalMs: window2.intervalMs
|
|
3129
|
+
}) == null;
|
|
3130
|
+
return dataCovered && liquidationCovered;
|
|
3048
3131
|
})
|
|
3049
3132
|
);
|
|
3050
3133
|
});
|
|
@@ -3129,6 +3212,11 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3129
3212
|
coverageKeysByInterval.set(interval2, coverageKeys);
|
|
3130
3213
|
const coverageRangesBySymbol = coverageRangesByInterval.get(interval2) ?? /* @__PURE__ */ new Map();
|
|
3131
3214
|
coverageRangesByInterval.set(interval2, coverageRangesBySymbol);
|
|
3215
|
+
const liquidationCoverageRangesBySymbol = liquidationCoverageRangesByInterval.get(interval2) ?? /* @__PURE__ */ new Map();
|
|
3216
|
+
liquidationCoverageRangesByInterval.set(
|
|
3217
|
+
interval2,
|
|
3218
|
+
liquidationCoverageRangesBySymbol
|
|
3219
|
+
);
|
|
3132
3220
|
for (let batchIdx = 0; batchIdx < symbolBatches.length; batchIdx += 1) {
|
|
3133
3221
|
const batch = symbolBatches[batchIdx];
|
|
3134
3222
|
let cursor = window2.fromMs;
|
|
@@ -3141,30 +3229,26 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3141
3229
|
fromMs: cursor,
|
|
3142
3230
|
toMs
|
|
3143
3231
|
});
|
|
3144
|
-
if (coverageKeys.has(key)) {
|
|
3145
|
-
return null;
|
|
3146
|
-
}
|
|
3147
3232
|
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3148
|
-
const
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
}
|
|
3167
|
-
return { item, fromMs: Math.max(coverageFromMs, edgesFromMs) };
|
|
3233
|
+
const requiredFromMs = resolveDerivativesContextRequiredFetchFromMs(
|
|
3234
|
+
{
|
|
3235
|
+
mode,
|
|
3236
|
+
interval: interval2,
|
|
3237
|
+
intervalMs,
|
|
3238
|
+
fromMs: cursor,
|
|
3239
|
+
toMs,
|
|
3240
|
+
nowMs: coverageNowMs,
|
|
3241
|
+
dataCoverageKeyExists: coverageKeys.has(key),
|
|
3242
|
+
dataCoverageRanges: coverageRangesBySymbol.get(normalizedSymbol),
|
|
3243
|
+
liquidationCoverageRanges: liquidationCoverageRangesBySymbol.get(normalizedSymbol),
|
|
3244
|
+
edges: edgesBySymbol.get(normalizedSymbol)
|
|
3245
|
+
}
|
|
3246
|
+
);
|
|
3247
|
+
if (requiredFromMs == null) return null;
|
|
3248
|
+
return {
|
|
3249
|
+
item,
|
|
3250
|
+
fromMs: requiredFromMs
|
|
3251
|
+
};
|
|
3168
3252
|
}).filter(
|
|
3169
3253
|
(item) => item != null
|
|
3170
3254
|
);
|
|
@@ -3178,85 +3262,150 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
|
|
|
3178
3262
|
const marketSymbols = missingBatch.map(
|
|
3179
3263
|
(item) => item.marketSymbol
|
|
3180
3264
|
);
|
|
3181
|
-
const
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
metric: "liq",
|
|
3202
|
-
marketSymbols,
|
|
3203
|
-
apiKey,
|
|
3204
|
-
interval: interval2,
|
|
3205
|
-
fromMs: group.fromMs,
|
|
3206
|
-
toMs
|
|
3207
|
-
});
|
|
3208
|
-
const rows = missingBatch.flatMap((item) => {
|
|
3209
|
-
const marketSymbol = item.marketSymbol.toUpperCase();
|
|
3210
|
-
const points = (0, import_indicators.mergeCoinalyzeMetrics)({
|
|
3211
|
-
symbol: item.symbol,
|
|
3212
|
-
oiRaw: oiMap.get(marketSymbol) ?? [],
|
|
3213
|
-
fundingRaw: fundingMap.get(marketSymbol) ?? [],
|
|
3214
|
-
liqRaw: liqMap.get(marketSymbol) ?? []
|
|
3265
|
+
const maxClosedBarAttempts = mode === "signals" ? asInt2(
|
|
3266
|
+
process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
|
|
3267
|
+
3
|
|
3268
|
+
) : 1;
|
|
3269
|
+
const closedBarRetryDelayMs = asInt2(
|
|
3270
|
+
process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
|
|
3271
|
+
2e3
|
|
3272
|
+
);
|
|
3273
|
+
let rows = [];
|
|
3274
|
+
let missingClosedSymbols = [];
|
|
3275
|
+
let liquidationRowsByMarket = /* @__PURE__ */ new Map();
|
|
3276
|
+
for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
|
|
3277
|
+
const oiMap = await fetchMetricBatch({
|
|
3278
|
+
endpoint: oiPath,
|
|
3279
|
+
metric: "oi",
|
|
3280
|
+
marketSymbols,
|
|
3281
|
+
apiKey,
|
|
3282
|
+
interval: interval2,
|
|
3283
|
+
fromMs: group.fromMs,
|
|
3284
|
+
toMs
|
|
3215
3285
|
});
|
|
3216
|
-
|
|
3217
|
-
|
|
3286
|
+
const fundingMap = await fetchMetricBatch({
|
|
3287
|
+
endpoint: fundingPath,
|
|
3288
|
+
metric: "funding",
|
|
3289
|
+
marketSymbols,
|
|
3290
|
+
apiKey,
|
|
3291
|
+
interval: interval2,
|
|
3292
|
+
fromMs: group.fromMs,
|
|
3293
|
+
toMs
|
|
3294
|
+
});
|
|
3295
|
+
liquidationRowsByMarket = await fetchMetricBatch({
|
|
3296
|
+
endpoint: liqPath,
|
|
3297
|
+
metric: "liq",
|
|
3298
|
+
marketSymbols,
|
|
3299
|
+
apiKey,
|
|
3300
|
+
interval: interval2,
|
|
3301
|
+
fromMs: group.fromMs,
|
|
3302
|
+
toMs
|
|
3303
|
+
});
|
|
3304
|
+
rows = missingBatch.flatMap((item) => {
|
|
3305
|
+
const marketSymbol = item.marketSymbol.toUpperCase();
|
|
3306
|
+
const points = (0, import_indicators.mergeCoinalyzeMetrics)({
|
|
3307
|
+
symbol: item.symbol,
|
|
3308
|
+
oiRaw: oiMap.get(marketSymbol) ?? [],
|
|
3309
|
+
fundingRaw: fundingMap.get(marketSymbol) ?? [],
|
|
3310
|
+
liqRaw: liquidationRowsByMarket.get(marketSymbol) ?? []
|
|
3311
|
+
});
|
|
3312
|
+
return (0, import_indicators.coinalyzePointsToRows)(points, interval2, "coinalyze");
|
|
3313
|
+
});
|
|
3314
|
+
missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
|
|
3315
|
+
symbols: missingBatch.map((item) => item.symbol),
|
|
3316
|
+
rows,
|
|
3317
|
+
expectedTimestamp: toMs
|
|
3318
|
+
}) : [];
|
|
3319
|
+
if (!missingClosedSymbols.length) break;
|
|
3320
|
+
if (attempt < maxClosedBarAttempts) {
|
|
3321
|
+
await (0, import_async.delay)(closedBarRetryDelayMs);
|
|
3322
|
+
}
|
|
3323
|
+
}
|
|
3324
|
+
if (missingClosedSymbols.length) {
|
|
3325
|
+
throw new Error(
|
|
3326
|
+
`Coinalyze closed ${interval2} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
|
|
3327
|
+
);
|
|
3328
|
+
}
|
|
3218
3329
|
if (rows.length) {
|
|
3219
3330
|
await (0, import_timescale2.upsertDerivatives)(rows);
|
|
3220
3331
|
totalRows += rows.length;
|
|
3221
3332
|
}
|
|
3222
|
-
const
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
(rowsCountBySymbol.get(symbol) ?? 0) + 1
|
|
3228
|
-
);
|
|
3229
|
-
}
|
|
3230
|
-
const coverageRows = missingBatch.map((item) => {
|
|
3231
|
-
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3232
|
-
const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
|
|
3233
|
-
return {
|
|
3234
|
-
source: "coinalyze",
|
|
3235
|
-
symbol: item.symbol,
|
|
3236
|
-
interval: interval2,
|
|
3237
|
-
fromMs: cursor,
|
|
3238
|
-
toMs,
|
|
3239
|
-
rowsCount
|
|
3240
|
-
};
|
|
3333
|
+
const confirmedLiquidationWindow = (0, import_indicators.resolveCoinalyzeConfirmedIntradayCoverage)({
|
|
3334
|
+
interval: interval2,
|
|
3335
|
+
fromMs: group.fromMs,
|
|
3336
|
+
toMs,
|
|
3337
|
+
nowMs: coverageNowMs
|
|
3241
3338
|
});
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3339
|
+
if (confirmedLiquidationWindow) {
|
|
3340
|
+
const metricCoverageRows = missingBatch.map((item) => {
|
|
3341
|
+
const marketSymbol = item.marketSymbol.toUpperCase();
|
|
3342
|
+
const eventRowsCount = (liquidationRowsByMarket.get(marketSymbol) ?? []).filter((point) => {
|
|
3343
|
+
const timestamp = (0, import_indicators.toCoinalyzeTimestampMs)(
|
|
3344
|
+
point.t ?? point.ts ?? point.time ?? point.timestamp
|
|
3345
|
+
);
|
|
3346
|
+
return timestamp != null && timestamp >= confirmedLiquidationWindow.fromMs && timestamp <= confirmedLiquidationWindow.toMs;
|
|
3347
|
+
}).length;
|
|
3348
|
+
return {
|
|
3349
|
+
source: "coinalyze",
|
|
3350
|
+
metric: "liquidation",
|
|
3351
|
+
symbol: item.symbol,
|
|
3352
|
+
interval: interval2,
|
|
3353
|
+
fromMs: confirmedLiquidationWindow.fromMs,
|
|
3354
|
+
toMs: confirmedLiquidationWindow.toMs,
|
|
3355
|
+
eventRowsCount
|
|
3356
|
+
};
|
|
3357
|
+
});
|
|
3358
|
+
await (0, import_timescale2.applyDerivativesMetricCoverage)(metricCoverageRows);
|
|
3359
|
+
for (const coverageRow of metricCoverageRows) {
|
|
3360
|
+
const symbol = coverageRow.symbol.toUpperCase();
|
|
3361
|
+
const ranges = liquidationCoverageRangesBySymbol.get(symbol) ?? [];
|
|
3362
|
+
ranges.push({
|
|
3363
|
+
fromMs: coverageRow.fromMs,
|
|
3364
|
+
toMs: coverageRow.toMs
|
|
3365
|
+
});
|
|
3366
|
+
liquidationCoverageRangesBySymbol.set(symbol, ranges);
|
|
3367
|
+
}
|
|
3368
|
+
}
|
|
3369
|
+
if (mode === "backtest") {
|
|
3370
|
+
const rowsCountBySymbol = /* @__PURE__ */ new Map();
|
|
3371
|
+
for (const row of rows) {
|
|
3372
|
+
const symbol = row.symbol.toUpperCase();
|
|
3373
|
+
rowsCountBySymbol.set(
|
|
3251
3374
|
symbol,
|
|
3375
|
+
(rowsCountBySymbol.get(symbol) ?? 0) + 1
|
|
3376
|
+
);
|
|
3377
|
+
}
|
|
3378
|
+
const coverageRows = missingBatch.map((item) => {
|
|
3379
|
+
const normalizedSymbol = item.symbol.toUpperCase();
|
|
3380
|
+
const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
|
|
3381
|
+
return {
|
|
3382
|
+
source: "coinalyze",
|
|
3383
|
+
symbol: item.symbol,
|
|
3252
3384
|
interval: interval2,
|
|
3253
3385
|
fromMs: cursor,
|
|
3254
|
-
toMs
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3386
|
+
toMs,
|
|
3387
|
+
rowsCount
|
|
3388
|
+
};
|
|
3389
|
+
});
|
|
3390
|
+
await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
|
|
3391
|
+
for (const coverageRow of coverageRows) {
|
|
3392
|
+
const symbol = coverageRow.symbol.toUpperCase();
|
|
3393
|
+
edgesBySymbol.set(
|
|
3394
|
+
symbol,
|
|
3395
|
+
extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
|
|
3396
|
+
);
|
|
3397
|
+
coverageKeys.add(
|
|
3398
|
+
coverageKey({
|
|
3399
|
+
symbol,
|
|
3400
|
+
interval: interval2,
|
|
3401
|
+
fromMs: cursor,
|
|
3402
|
+
toMs
|
|
3403
|
+
})
|
|
3404
|
+
);
|
|
3405
|
+
const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
|
|
3406
|
+
coverageRanges.push({ fromMs: cursor, toMs });
|
|
3407
|
+
coverageRangesBySymbol.set(symbol, coverageRanges);
|
|
3408
|
+
}
|
|
3260
3409
|
}
|
|
3261
3410
|
}
|
|
3262
3411
|
}
|