dexbot 1.6.12 → 1.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -1
- package/README.md +2 -2
- package/analysis/README.md +9 -11
- package/analysis/ama_fitting/README.md +5 -3
- package/claw/README.md +1 -0
- package/claw/docs/AI_BOT_LIBRARY_API.md +3 -1
- package/claw/docs/DEXBOT2_TUNING_CHEAT_SHEET.md +4 -4
- package/claw/modules/claw_bridge.ts +0 -3
- package/claw/modules/claw_catalog.ts +8 -4
- package/claw/modules/claw_skill_md.ts +1 -0
- package/claw/modules/dexbot_profiles.ts +59 -15
- package/claw/modules/types.ts +2 -1
- package/claw/package.json +3 -3
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/claw/skills/margin-trading/SKILL.md +1 -1
- package/claw/skills/margin-trading/references/position-management.md +1 -1
- package/claw/tsconfig.json +2 -2
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +1 -14
- package/dist/analysis/ama_fitting/fetch_lp_candles.js +51 -41
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -24
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +36 -18
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js +18 -66
- package/dist/analysis/analyze_args.d.ts +12 -0
- package/dist/analysis/analyze_args.js +20 -0
- package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +1 -16
- package/dist/analysis/bot_fitting/backtest_ama_sweep.js +20 -101
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js +26 -92
- package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
- package/dist/analysis/bot_fitting/shared_utils.d.ts +71 -3
- package/dist/analysis/bot_fitting/shared_utils.js +124 -8
- package/dist/analysis/chart_ui.d.ts +30 -1
- package/dist/analysis/chart_ui.js +37 -1
- package/dist/analysis/fills_cache.d.ts +3 -23
- package/dist/analysis/fills_cache.js +3 -84
- package/dist/analysis/fills_source.d.ts +31 -1
- package/dist/analysis/fills_source.js +69 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -23
- package/dist/analysis/grid_correction_check.js +15 -58
- package/dist/analysis/math_utils.d.ts +85 -1
- package/dist/analysis/math_utils.js +116 -1
- package/dist/analysis/price_sources.d.ts +22 -1
- package/dist/analysis/price_sources.js +23 -0
- package/dist/analysis/resolve_source.js +31 -10
- package/dist/analysis/trade_profitability.d.ts +6 -0
- package/dist/analysis/trade_profitability.js +23 -97
- package/dist/analysis/tradingview/analyze_tradingview.d.ts +1 -17
- package/dist/analysis/tradingview/analyze_tradingview.js +1 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +27 -40
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +22 -80
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +11 -37
- package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
- package/dist/analysis/worker_pool.d.ts +38 -0
- package/dist/analysis/worker_pool.js +67 -0
- package/dist/dexbot.js +67 -48
- package/dist/market_adapter/ama_signal_runner.js +21 -54
- package/dist/market_adapter/candle_utils.d.ts +7 -1
- package/dist/market_adapter/candle_utils.js +11 -1
- package/dist/market_adapter/cli_args.d.ts +22 -0
- package/dist/market_adapter/cli_args.js +30 -0
- package/dist/market_adapter/core/config_normalizers.js +11 -6
- package/dist/market_adapter/core/kibana_candles.d.ts +3 -1
- package/dist/market_adapter/core/kibana_candles.js +3 -2
- package/dist/market_adapter/core/market_adapter_service.d.ts +85 -85
- package/dist/market_adapter/core/market_adapter_service.js +93 -158
- package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
- package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
- package/dist/market_adapter/inputs/fetch_book_data.js +0 -1
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +63 -107
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +8 -1
- package/dist/market_adapter/inputs/fetch_lp_data.js +25 -7
- package/dist/market_adapter/inputs/kibana_feed_source.js +4 -26
- package/dist/market_adapter/inputs/window_cache.d.ts +4 -72
- package/dist/market_adapter/inputs/window_cache.js +52 -177
- package/dist/market_adapter/interval_utils.d.ts +11 -1
- package/dist/market_adapter/interval_utils.js +16 -1
- package/dist/market_adapter/lp_chart_runner.d.ts +3 -0
- package/dist/market_adapter/lp_chart_runner.js +37 -19
- package/dist/market_adapter/market_adapter.js +27 -72
- package/dist/market_adapter/utils/data_discovery.d.ts +45 -1
- package/dist/market_adapter/utils/data_discovery.js +220 -27
- package/dist/market_adapter/utils/month_shards.d.ts +84 -0
- package/dist/market_adapter/utils/month_shards.js +237 -0
- package/dist/modules/account_bots.d.ts +8 -6
- package/dist/modules/account_bots.js +112 -131
- package/dist/modules/account_orders.d.ts +49 -4
- package/dist/modules/account_orders.js +114 -49
- package/dist/modules/bitshares-native/chain_client.d.ts +1 -1
- package/dist/modules/bitshares-native/chain_client.js +14 -13
- package/dist/modules/bitshares-native/crypto/ecc_core.d.ts +1 -2
- package/dist/modules/bitshares-native/crypto/ecc_core.js +1 -1
- package/dist/modules/bitshares-native/resolvers.js +22 -25
- package/dist/modules/bitshares-native/serial/chain_constants.d.ts +1 -2
- package/dist/modules/bitshares-native/serial/chain_constants.js +1 -2
- package/dist/modules/bitshares-native/serial/types.js +22 -50
- package/dist/modules/bitshares-native/signing_client.d.ts +1 -1
- package/dist/modules/bot_defaults.d.ts +18 -3
- package/dist/modules/bot_defaults.js +34 -8
- package/dist/modules/bot_settings.d.ts +5 -1
- package/dist/modules/bot_settings.js +55 -21
- package/dist/modules/broadcast_failure.d.ts +2 -1
- package/dist/modules/chain_orders.d.ts +1 -1
- package/dist/modules/chain_orders.js +51 -96
- package/dist/modules/cli_start_onboarding.d.ts +2 -1
- package/dist/modules/config.js +6 -1
- package/dist/modules/constants.d.ts +14 -6
- package/dist/modules/constants.js +32 -6
- package/dist/modules/cr_planner.d.ts +1 -1
- package/dist/modules/cr_planner.js +1 -5
- package/dist/modules/credit_pricing.d.ts +2 -1
- package/dist/modules/credit_pricing.js +36 -40
- package/dist/modules/credit_runtime.d.ts +7 -0
- package/dist/modules/credit_runtime.js +45 -73
- package/dist/modules/daemon_node_health.d.ts +4 -3
- package/dist/modules/dexbot_class.d.ts +33 -17
- package/dist/modules/dexbot_class.js +113 -61
- package/dist/modules/dexbot_cow_runtime.d.ts +62 -181
- package/dist/modules/dexbot_cow_runtime.js +380 -69
- package/dist/modules/dexbot_credential_client.d.ts +1 -23
- package/dist/modules/dexbot_credential_client.js +2 -3
- package/dist/modules/dexbot_fill_runtime.d.ts +2 -3
- package/dist/modules/dexbot_fill_runtime.js +17 -7
- package/dist/modules/dexbot_maintenance_runtime.d.ts +36 -81
- package/dist/modules/dexbot_maintenance_runtime.js +137 -133
- package/dist/modules/dexbot_startup_runtime.d.ts +0 -1
- package/dist/modules/dexbot_startup_runtime.js +25 -14
- package/dist/modules/dexbot_state_recovery.js +16 -0
- package/dist/modules/general_settings.d.ts +1 -1
- package/dist/modules/graceful_shutdown.js +17 -11
- package/dist/modules/key_store.d.ts +0 -5
- package/dist/modules/key_store.js +1 -2
- package/dist/modules/lazy_shim.d.ts +11 -0
- package/dist/modules/lazy_shim.js +16 -0
- package/dist/modules/node_connect_policy.d.ts +3 -2
- package/dist/modules/node_failure_ledger.d.ts +6 -5
- package/dist/modules/order/accounting.js +11 -11
- package/dist/modules/order/export.js +2 -2
- package/dist/modules/order/format.d.ts +26 -3
- package/dist/modules/order/format.js +33 -3
- package/dist/modules/order/genesis_policy.d.ts +4 -4
- package/dist/modules/order/genesis_policy.js +6 -3
- package/dist/modules/order/grid.d.ts +26 -12
- package/dist/modules/order/grid.js +468 -80
- package/dist/modules/order/grid_reconcile.d.ts +1 -1
- package/dist/modules/order/grid_reconcile.js +5 -5
- package/dist/modules/order/grid_reconcile_internal.js +48 -40
- package/dist/modules/order/index.d.ts +35 -8
- package/dist/modules/order/logger.d.ts +1 -8
- package/dist/modules/order/logger.js +2 -11
- package/dist/modules/order/manager.d.ts +18 -15
- package/dist/modules/order/manager.js +91 -74
- package/dist/modules/order/strategy.d.ts +2 -2
- package/dist/modules/order/strategy.js +51 -21
- package/dist/modules/order/sync_engine.js +4 -0
- package/dist/modules/order/utils/math.d.ts +3 -3
- package/dist/modules/order/utils/math.js +38 -27
- package/dist/modules/order/utils/order.d.ts +58 -58
- package/dist/modules/order/utils/order.js +158 -137
- package/dist/modules/order/utils/system.d.ts +202 -35
- package/dist/modules/order/utils/system.js +409 -31
- package/dist/modules/order/utils/validate.js +28 -6
- package/dist/modules/order/utils/withPoolRef.d.ts +2 -1
- package/dist/modules/paths.js +32 -34
- package/dist/modules/runtime_settings.d.ts +3 -5
- package/dist/modules/runtime_settings.js +3 -3
- package/dist/modules/settings_merge.d.ts +2 -2
- package/dist/modules/socket_json_client.d.ts +2 -1
- package/dist/modules/types.d.ts +89 -30
- package/dist/modules/utils/errors.js +1 -3
- package/dist/modules/version_notice.d.ts +11 -10
- package/dist/pm2.d.ts +0 -1
- package/dist/pm2.js +56 -35
- package/dist/scripts/analyze-git.js +2 -15
- package/dist/scripts/analyze-orders.d.ts +1 -1
- package/dist/scripts/analyze-orders.js +4 -4
- package/dist/scripts/chart_command.js +2 -2
- package/dist/scripts/check_any.d.ts +4 -0
- package/dist/scripts/check_any.js +53 -21
- package/dist/scripts/diagnose-pool-history.js +1 -6
- package/dist/scripts/divergence-calc.js +6 -6
- package/dist/scripts/run-tests.js +0 -1
- package/dist/scripts/sync-version.js +0 -10
- package/dist/scripts/validate_bots.d.ts +1 -1
- package/dist/scripts/validate_bots.js +3 -3
- package/dist/unlock.js +7 -2
- package/docs/BITSHARES_ONBOARDING.md +8 -12
- package/docs/COPY_ON_WRITE_MASTER_PLAN.md +2 -3
- package/docs/COW_INVARIANTS.md +6 -6
- package/docs/DEXBOT_COMPARISON.md +7 -7
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +8 -6
- package/docs/GRID_PRICE_INVARIANT.md +3 -3
- package/docs/GRID_RECALCULATION.md +4 -1
- package/docs/GRID_RECONCILE.md +4 -4
- package/docs/LIFECYCLE.md +11 -11
- package/docs/LOGGING.md +14 -0
- package/docs/MPA_CREDIT_USAGE.md +1 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +12 -4
- package/docs/README.md +3 -3
- package/docs/WORKFLOW.md +1 -0
- package/docs/architecture.md +25 -16
- package/docs/developer_guide.md +7 -2
- package/market_adapter/README.md +6 -4
- package/package.json +6 -9
- package/scripts/README.md +43 -38
- package/tests/README.md +26 -11
- package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
- package/dist/scripts/diagnose-kibana-candles.js +0 -82
- package/scripts/git-viewer.sh +0 -489
- package/scripts/postinstall.js +0 -16
|
@@ -38,11 +38,18 @@ declare function applyPrecisionOverrides(assetA: AssetRef, assetB: AssetRef, pre
|
|
|
38
38
|
assetA: AssetRef;
|
|
39
39
|
assetB: AssetRef;
|
|
40
40
|
};
|
|
41
|
+
declare function loadCachedFetchContext(bot: BotConfig, intervalSeconds: number, dir?: string): {
|
|
42
|
+
poolId: string;
|
|
43
|
+
assetA: AssetRef;
|
|
44
|
+
assetB: AssetRef;
|
|
45
|
+
source: string;
|
|
46
|
+
path: string;
|
|
47
|
+
} | null;
|
|
41
48
|
declare function fetchCandlesSequentially(fullPoolId: string, assetA: AssetRef, assetB: AssetRef, config: LpConfig, outPath: string): Promise<number[][]>;
|
|
42
49
|
declare function parseBotsConfig(raw: string, sourceLabel?: string): BotConfig[];
|
|
43
50
|
/**
|
|
44
51
|
* Pick a bot: by name if given, otherwise first active bot with startPrice: "pool".
|
|
45
52
|
*/
|
|
46
53
|
declare function selectBot(bots: BotConfig[], botName: string | null | undefined): BotConfig;
|
|
47
|
-
export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath };
|
|
54
|
+
export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath, loadCachedFetchContext };
|
|
48
55
|
//# sourceMappingURL=fetch_lp_data.d.ts.map
|
|
@@ -35,6 +35,8 @@ import { normalizePoolId, resolveAsset, findPoolByAssets } from '../utils/chain.
|
|
|
35
35
|
import { writeJsonAtomic } from '../utils/atomic_write.js';
|
|
36
36
|
import { buildFetchWindowsFromRange, formatWindowLine, runCachedWindows, } from './window_cache.js';
|
|
37
37
|
import { PATHS } from '../../modules/paths.js';
|
|
38
|
+
import { loadLpSeriesFromBasePath } from '../utils/data_discovery.js';
|
|
39
|
+
import { shardStemFromName } from '../utils/month_shards.js';
|
|
38
40
|
import * as bitsharesClient from '../../modules/bitshares_client.js';
|
|
39
41
|
import { getErrorMessage, resolveSeamMs } from '../../modules/utils/errors.js';
|
|
40
42
|
import { isSameBotName } from '../../modules/utils/sanitize_key.js';
|
|
@@ -185,13 +187,24 @@ function buildRequestKey(config, fullPoolId, assetA, assetB, timeRange, outPath)
|
|
|
185
187
|
chunkMonths,
|
|
186
188
|
};
|
|
187
189
|
}
|
|
188
|
-
function loadCachedFetchContext(bot, intervalSeconds) {
|
|
189
|
-
const dir = pairFolderPath(bot.assetA, bot.assetB);
|
|
190
|
+
function loadCachedFetchContext(bot, intervalSeconds, dir = pairFolderPath(bot.assetA, bot.assetB)) {
|
|
190
191
|
if (!storage.exists(dir))
|
|
191
192
|
return null;
|
|
192
193
|
const label = toIntervalLabel(intervalSeconds);
|
|
194
|
+
// Accept the whole-history export (`….json`) AND month shards
|
|
195
|
+
// (`….shard_YYYY-MM.json`). After the shard refactor `dexbot tv`/`dw` only
|
|
196
|
+
// write shards, so a shard-only pair had no cached context and fell back to
|
|
197
|
+
// a full blockchain connect. A shard's meta carries the same
|
|
198
|
+
// pool/asset/precision identity, and reading one shard is cheaper than
|
|
199
|
+
// parsing a multi-MB whole-history export.
|
|
200
|
+
const matchesInterval = (name) => {
|
|
201
|
+
if (name.endsWith(`${label}.json`))
|
|
202
|
+
return true;
|
|
203
|
+
const stem = shardStemFromName(name);
|
|
204
|
+
return stem ? stem.endsWith(`_${label}`) : false;
|
|
205
|
+
};
|
|
193
206
|
const dataFiles = storage.readdir(dir)
|
|
194
|
-
.filter(
|
|
207
|
+
.filter(matchesInterval)
|
|
195
208
|
.sort();
|
|
196
209
|
for (const file of dataFiles) {
|
|
197
210
|
try {
|
|
@@ -307,7 +320,6 @@ async function fetchCandlesSequentially(fullPoolId, assetA, assetB, config, outP
|
|
|
307
320
|
assetA: requestKey.assetA,
|
|
308
321
|
assetB: requestKey.assetB,
|
|
309
322
|
intervalSeconds: requestKey.intervalSeconds,
|
|
310
|
-
chunkIndex: window.index,
|
|
311
323
|
timeRange: { gte: window.gte, lte: window.lte },
|
|
312
324
|
format: '[timestamp_ms, open, high, low, close, volume_A]',
|
|
313
325
|
}),
|
|
@@ -545,6 +557,12 @@ async function run() {
|
|
|
545
557
|
keyBySymbols: `${assetA.symbol}|${assetB.symbol}`,
|
|
546
558
|
keyByIds: `${assetA.id}|${assetB.id}`,
|
|
547
559
|
};
|
|
560
|
+
// Derive the whole-history export from the assembled shard cache rather
|
|
561
|
+
// than from this run's fetched range: a later narrow run must not overwrite
|
|
562
|
+
// a full-history file with a subset. Falls back to the fetched candles when
|
|
563
|
+
// the shards were withheld (e.g. a partial fetch).
|
|
564
|
+
const assembled = loadLpSeriesFromBasePath(outPath);
|
|
565
|
+
const exportCandles = assembled && assembled.candles.length > 0 ? assembled.candles : candles;
|
|
548
566
|
const output = {
|
|
549
567
|
meta: {
|
|
550
568
|
fetchedAt: new Date().toISOString(),
|
|
@@ -555,7 +573,7 @@ async function run() {
|
|
|
555
573
|
pair,
|
|
556
574
|
intervalSeconds: config.intervalSeconds,
|
|
557
575
|
lookbackHours: config.lookbackHours ?? 0,
|
|
558
|
-
candleCount:
|
|
576
|
+
candleCount: exportCandles.length,
|
|
559
577
|
priceUnit: `${assetB.symbol} per ${assetA.symbol}`,
|
|
560
578
|
// Candle format: [timestamp_ms, open, high, low, close, volume_in_assetA]
|
|
561
579
|
// Raw Kibana LP exchange documents are ordered and converted to true OHLC.
|
|
@@ -564,7 +582,7 @@ async function run() {
|
|
|
564
582
|
// Actual received used (operation_result_object), not min_to_receive.
|
|
565
583
|
format: '[timestamp_ms, open, high, low, close, volume_A]',
|
|
566
584
|
},
|
|
567
|
-
candles,
|
|
585
|
+
candles: exportCandles,
|
|
568
586
|
};
|
|
569
587
|
writeJsonAtomic(outPath, output);
|
|
570
588
|
const kb = (storage.stat(outPath).size / 1024).toFixed(1);
|
|
@@ -578,5 +596,5 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
|
|
|
578
596
|
process.exit(1);
|
|
579
597
|
});
|
|
580
598
|
}
|
|
581
|
-
export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath };
|
|
599
|
+
export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath, loadCachedFetchContext };
|
|
582
600
|
//# sourceMappingURL=fetch_lp_data.js.map
|
|
@@ -30,7 +30,8 @@
|
|
|
30
30
|
* - Volume is the feed publish count per bucket, NOT traded asset volume.
|
|
31
31
|
*/
|
|
32
32
|
import { fillCandleGaps } from '../candle_utils.js';
|
|
33
|
-
import { resolveRequestedFillRange } from '../core/kibana_candles.js';
|
|
33
|
+
import { resolveRequestedFillRange, hitSortKey, hitSequence } from '../core/kibana_candles.js';
|
|
34
|
+
import { parseChainTimeToMs } from '../interval_utils.js';
|
|
34
35
|
import { kibanaSearch, DEFAULT_CONFIG as BASE_CONFIG } from '../core/kibana_client.js';
|
|
35
36
|
import { path } from '../../modules/path_api.js';
|
|
36
37
|
import { PATHS } from '../../modules/paths.js';
|
|
@@ -110,31 +111,9 @@ function parseFeedTimestamp(source) {
|
|
|
110
111
|
const raw = source?.block_data?.block_time;
|
|
111
112
|
if (raw == null)
|
|
112
113
|
return null;
|
|
113
|
-
const
|
|
114
|
-
const tsMs = Date.parse(text.endsWith('Z') ? text : `${text}Z`);
|
|
114
|
+
const tsMs = parseChainTimeToMs(String(raw));
|
|
115
115
|
return Number.isFinite(tsMs) ? tsMs : null;
|
|
116
116
|
}
|
|
117
|
-
function hitSortKey(hit) {
|
|
118
|
-
const h = hit;
|
|
119
|
-
const sort = Array.isArray(h?.sort) ? h.sort : [];
|
|
120
|
-
return sort.map((v) => String(v)).join('|') || String(h?._id || '');
|
|
121
|
-
}
|
|
122
|
-
function hitSequence(source) {
|
|
123
|
-
const src = source;
|
|
124
|
-
const candidates = [
|
|
125
|
-
src?.operation_id_num,
|
|
126
|
-
src?.account_history?.operation_id,
|
|
127
|
-
src?.account_history?.sequence,
|
|
128
|
-
];
|
|
129
|
-
for (const value of candidates) {
|
|
130
|
-
if (typeof value === 'number' && Number.isFinite(value))
|
|
131
|
-
return value;
|
|
132
|
-
const m = String(value || '').match(/(\d+)$/);
|
|
133
|
-
if (m)
|
|
134
|
-
return Number(m[1]);
|
|
135
|
-
}
|
|
136
|
-
return Number.NaN;
|
|
137
|
-
}
|
|
138
117
|
/**
|
|
139
118
|
* Kibana hit → { tsMs, price, sequence, kibanaSortKey } in backing-per-MPA,
|
|
140
119
|
* or null when the hit carries no usable settlement price for the pair.
|
|
@@ -151,7 +130,7 @@ function hitToFeedPrice(hit, { mpaAsset, backingAsset }) {
|
|
|
151
130
|
return {
|
|
152
131
|
tsMs,
|
|
153
132
|
price: price,
|
|
154
|
-
sequence: hitSequence(source),
|
|
133
|
+
sequence: hitSequence(source, 'account_history.operation_id'),
|
|
155
134
|
kibanaSortKey: hitSortKey(hit),
|
|
156
135
|
};
|
|
157
136
|
}
|
|
@@ -562,7 +541,6 @@ async function fetchFeedCandlesSequentially(feedCtx, assetA, assetB, opts = {})
|
|
|
562
541
|
assetA: requestKey.assetA,
|
|
563
542
|
assetB: requestKey.assetB,
|
|
564
543
|
intervalSeconds,
|
|
565
|
-
chunkIndex: window.index,
|
|
566
544
|
timeRange: { gte: window.gte, lte: window.lte },
|
|
567
545
|
format: '[timestamp_ms, open, high, low, close, feed_publish_count]',
|
|
568
546
|
}),
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges } from '../utils/month_shards.js';
|
|
2
|
+
import type { QueriedRange } from '../utils/month_shards.js';
|
|
1
3
|
type Candle = number[];
|
|
2
4
|
interface FileCover {
|
|
3
5
|
gte: number | null;
|
|
@@ -75,68 +77,12 @@ interface RunCachedWindowsOptions {
|
|
|
75
77
|
declare const TAIL_REFRESH_HOURS = 48;
|
|
76
78
|
declare const TAIL_SETTLE_LAG_MS: number;
|
|
77
79
|
declare const GAP_SETTLE_HORIZON_MS: number;
|
|
78
|
-
declare const GAP_MERGE_TOLERANCE_BUCKETS = 2;
|
|
79
|
-
declare const MAX_SUBFETCH_RANGES = 8;
|
|
80
|
-
declare const MAX_SUBFETCH_SPAN_RATIO = 0.5;
|
|
81
80
|
declare const MAX_COVERAGE_SPANS = 64;
|
|
82
|
-
declare function shardKeyForTimestamp(tsMs: unknown): string;
|
|
83
|
-
declare function shardBoundsForKey(key: unknown): {
|
|
84
|
-
start: number;
|
|
85
|
-
end: number;
|
|
86
|
-
};
|
|
87
|
-
declare function shardKeysForRange(gteMs: unknown, lteMs: unknown): string[];
|
|
88
|
-
declare function shardPathFor(outPath: string, shardKey: string): string;
|
|
89
81
|
declare function readCacheChunk(chunkFile: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean): CacheChunk | null;
|
|
90
82
|
declare function loadBucketCache(outPath: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean, range?: {
|
|
91
83
|
gte: number;
|
|
92
84
|
lte: number;
|
|
93
85
|
} | null): BucketCache;
|
|
94
|
-
declare function cachedCandlesInRange(localCache: BucketCache, gteMs: number, lteMs: number): Candle[];
|
|
95
|
-
/** Span of time actually queried, plus when the query ran (null = unknown). */
|
|
96
|
-
type QueriedRange = {
|
|
97
|
-
gte: number;
|
|
98
|
-
lte: number;
|
|
99
|
-
at: number | null;
|
|
100
|
-
};
|
|
101
|
-
/** Parse a verification timestamp: epoch ms or ISO. Anything else is unknown. */
|
|
102
|
-
declare function timestampMs(raw: unknown): number | null;
|
|
103
|
-
declare function verifiedAt(range: unknown): number | null;
|
|
104
|
-
/**
|
|
105
|
-
* Normalize recorded spans: sort, then merge overlapping / bucket-adjacent
|
|
106
|
-
* ones — but ONLY when they assert the same verification time. Merging spans
|
|
107
|
-
* with different `at` would have to pick one verdict for the whole union, and
|
|
108
|
-
* either pick loses (min → the merged span vouches for buckets the older
|
|
109
|
-
* query never saw; max → buckets verified long ago look freshly verified).
|
|
110
|
-
* Keeping them separate preserves exact per-bucket freshness, which is the
|
|
111
|
-
* whole point of `at`. Spans therefore accumulate ~1 per tail run, each a few
|
|
112
|
-
* dozen bytes — negligible next to the candles in the same file, and the
|
|
113
|
-
* same-`at` case (bulk fetches, and every pre-`at` shard) still collapses to
|
|
114
|
-
* one span as before.
|
|
115
|
-
*/
|
|
116
|
-
declare function unionQueriedRanges(ranges: {
|
|
117
|
-
gte: number;
|
|
118
|
-
lte: number;
|
|
119
|
-
at?: number | null;
|
|
120
|
-
}[], bucketMs: number): QueriedRange[];
|
|
121
|
-
declare function rangesCoveredBy(have: {
|
|
122
|
-
gte: number;
|
|
123
|
-
lte: number;
|
|
124
|
-
}[], want: {
|
|
125
|
-
gte: number;
|
|
126
|
-
lte: number;
|
|
127
|
-
}[]): boolean;
|
|
128
|
-
/**
|
|
129
|
-
* Change detection for shard writes: `have` already accounts for every span in
|
|
130
|
-
* `want`, in extent AND in verification time. Without the `at` half, a run
|
|
131
|
-
* whose only news is "these buckets were re-verified 1h later" would skip the
|
|
132
|
-
* write and the next run would fall back to the wide fixed refresh window.
|
|
133
|
-
* An unknown `at` on either side is never treated as fresh.
|
|
134
|
-
*/
|
|
135
|
-
declare function coverageSatisfied(have: QueriedRange[], want: QueriedRange[]): boolean;
|
|
136
|
-
/** Every recorded span across the loaded files, newest verification included. */
|
|
137
|
-
declare function allQueriedRanges(fileCover: {
|
|
138
|
-
queried?: QueriedRange[] | null;
|
|
139
|
-
}[]): QueriedRange[];
|
|
140
86
|
/**
|
|
141
87
|
* How settled a window is, given the recorded coverage.
|
|
142
88
|
*
|
|
@@ -158,20 +104,6 @@ declare function settleCoverage(opts: {
|
|
|
158
104
|
hasTimestamps: boolean;
|
|
159
105
|
firstUnsettled: number | null;
|
|
160
106
|
};
|
|
161
|
-
/**
|
|
162
|
-
* Bound the persisted span list. Because spans merge only when they assert
|
|
163
|
-
* the same `at`, a shard that is re-verified on every run gains one span per
|
|
164
|
-
* run and would grow without limit (and `settleCoverage` scans the list once
|
|
165
|
-
* per tail window). Dropping the OLDEST spans never invents emptiness, but it
|
|
166
|
-
* is not free: coverage is consulted by BOTH `pruneImmutableGaps` (to skip a
|
|
167
|
-
* query) and `settleCoverage` (to decide a bucket is fresh). A dropped span
|
|
168
|
-
* makes its buckets look unsettled, and the tail refresh would otherwise start
|
|
169
|
-
* at the earliest of them. That is why the refresh is capped at
|
|
170
|
-
* TAIL_REFRESH_HOURS: the worst a dropped span can cost is the legacy 48h
|
|
171
|
-
* overlap, never weeks. The input is already sorted by extent, so the tail of
|
|
172
|
-
* the list is the newest.
|
|
173
|
-
*/
|
|
174
|
-
declare function compactCoverage(spans: QueriedRange[], maxSpans: number): QueriedRange[];
|
|
175
107
|
declare function buildFetchWindowsFromRange(timeRange: {
|
|
176
108
|
gte?: unknown;
|
|
177
109
|
lte?: unknown;
|
|
@@ -229,6 +161,6 @@ declare function formatWindowLine(unit: string, index: number, total: number, gt
|
|
|
229
161
|
* touch nothing on disk.
|
|
230
162
|
*/
|
|
231
163
|
declare function runCachedWindows(opts: RunCachedWindowsOptions): Promise<Candle[]>;
|
|
232
|
-
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS,
|
|
233
|
-
export type { QueriedRange };
|
|
164
|
+
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
|
|
165
|
+
export type { QueriedRange } from '../utils/month_shards.js';
|
|
234
166
|
//# sourceMappingURL=window_cache.d.ts.map
|
|
@@ -27,8 +27,7 @@
|
|
|
27
27
|
* - `isMatch(meta, requestKey)` — same-pool/feed/interval/assets check,
|
|
28
28
|
* - `fetchRange(gteIso, lteIso)` — query one (sub-)range, gap-filled grid,
|
|
29
29
|
* - `metaForWindow(window)` — identity meta fields (source/feed/pool/...);
|
|
30
|
-
* the runner overrides timeRange with the shard bounds
|
|
31
|
-
* chunkIndex, which is meaningless for stable shards.
|
|
30
|
+
* the runner overrides timeRange with the shard bounds.
|
|
32
31
|
*
|
|
33
32
|
* Node-only (disk I/O via storage). Browser-safe code must not import this.
|
|
34
33
|
*/
|
|
@@ -36,7 +35,8 @@ import { path } from '../../modules/path_api.js';
|
|
|
36
35
|
import { getStorage } from '../../modules/storage/index.js';
|
|
37
36
|
import { writeJsonAtomic } from '../utils/atomic_write.js';
|
|
38
37
|
import { sleepMs, getErrorName, getErrorMessage } from '../../modules/utils/errors.js';
|
|
39
|
-
import { mergeCandles } from '../candle_utils.js';
|
|
38
|
+
import { mergeCandles, higherVolumeWins } from '../candle_utils.js';
|
|
39
|
+
import { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, shardKeyFromName, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges, clipRangeTo, } from '../utils/month_shards.js';
|
|
40
40
|
const storage = getStorage();
|
|
41
41
|
const { readJSON } = storage;
|
|
42
42
|
// Trailing overlap for the newest window. It is the refresh when the cache
|
|
@@ -87,49 +87,18 @@ const MAX_SUBFETCH_SPAN_RATIO = 0.5;
|
|
|
87
87
|
// current month accumulates.
|
|
88
88
|
const MAX_COVERAGE_SPANS = 64;
|
|
89
89
|
// ─── Month-shard naming ───────────────────────────────────────────────────────
|
|
90
|
-
// Shard key
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
function shardKeyForTimestamp(tsMs) {
|
|
94
|
-
const d = new Date(Number(tsMs));
|
|
95
|
-
return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
|
|
96
|
-
}
|
|
97
|
-
function shardBoundsForKey(key) {
|
|
98
|
-
const parts = String(key).split('-').map(Number);
|
|
99
|
-
const start = Date.UTC(parts[0], parts[1] - 1, 1);
|
|
100
|
-
const end = parts[1] === 12 ? Date.UTC(parts[0] + 1, 0, 1) : Date.UTC(parts[0], parts[1], 1);
|
|
101
|
-
return { start, end };
|
|
102
|
-
}
|
|
103
|
-
function shardKeysForRange(gteMs, lteMs) {
|
|
104
|
-
const keys = [];
|
|
105
|
-
let cursor = new Date(Date.UTC(new Date(Number(gteMs)).getUTCFullYear(), new Date(Number(gteMs)).getUTCMonth(), 1));
|
|
106
|
-
const last = new Date(Number(lteMs));
|
|
107
|
-
while (cursor <= last) {
|
|
108
|
-
keys.push(shardKeyForTimestamp(cursor.getTime()));
|
|
109
|
-
cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth() + 1, 1));
|
|
110
|
-
}
|
|
111
|
-
return keys;
|
|
112
|
-
}
|
|
113
|
-
function shardRangeOverlaps(shardKey, gteMs, lteMs) {
|
|
114
|
-
const { start, end } = shardBoundsForKey(shardKey);
|
|
115
|
-
return start <= Number(lteMs) && end > Number(gteMs);
|
|
116
|
-
}
|
|
117
|
-
function shardPathFor(outPath, shardKey) {
|
|
118
|
-
const parsed = path.parse(outPath);
|
|
119
|
-
return path.join(parsed.dir, `${parsed.name}.shard_${shardKey}${parsed.ext}`);
|
|
120
|
-
}
|
|
90
|
+
// Shard key, bounds, path and coverage helpers live in
|
|
91
|
+
// `market_adapter/utils/month_shards.ts` so the candle and fills caches share
|
|
92
|
+
// one contract and cannot disagree on a month boundary.
|
|
121
93
|
function siblingCacheFiles(outPath) {
|
|
122
94
|
const resolved = path.resolve(outPath);
|
|
123
95
|
const parsed = path.parse(resolved);
|
|
124
96
|
if (!storage.exists(parsed.dir))
|
|
125
97
|
return [];
|
|
126
|
-
const shardPrefix = `${parsed.name}.shard_`;
|
|
127
98
|
const out = [];
|
|
128
99
|
for (const name of storage.readdir(parsed.dir)) {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
const key = name.slice(shardPrefix.length, name.length - parsed.ext.length);
|
|
132
|
-
if (!/^\d{4}-\d{2}$/.test(key))
|
|
100
|
+
const key = shardKeyFromName(name, parsed.name, parsed.ext);
|
|
101
|
+
if (!key)
|
|
133
102
|
continue;
|
|
134
103
|
out.push({ file: path.join(parsed.dir, name), shardKey: key });
|
|
135
104
|
}
|
|
@@ -188,11 +157,16 @@ function loadBucketCache(outPath, requestKey, isMatch, range) {
|
|
|
188
157
|
const shards = [];
|
|
189
158
|
let files = 0;
|
|
190
159
|
const scoped = range && Number.isFinite(range.gte) && Number.isFinite(range.lte);
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
160
|
+
// Scoped load: enumerate only the calendar-month shards that overlap the
|
|
161
|
+
// run and open exactly those — no directory listing, and out-of-range
|
|
162
|
+
// months are never even stat'd. Unscoped callers (no finite range) fall
|
|
163
|
+
// back to the full listing.
|
|
164
|
+
const entries = scoped
|
|
165
|
+
? shardKeysForRange(range.gte, range.lte)
|
|
166
|
+
.map((shardKey) => ({ file: shardPathFor(outPath, shardKey), shardKey }))
|
|
167
|
+
.filter((entry) => storage.exists(entry.file))
|
|
168
|
+
: siblingCacheFiles(outPath);
|
|
169
|
+
for (const entry of entries) {
|
|
196
170
|
const chunk = readCacheChunk(entry.file, requestKey, isMatch);
|
|
197
171
|
if (!chunk)
|
|
198
172
|
continue;
|
|
@@ -221,109 +195,12 @@ function cachedCandlesInRange(localCache, gteMs, lteMs) {
|
|
|
221
195
|
out.sort((a, b) => Number(a[0]) - Number(b[0]));
|
|
222
196
|
return out;
|
|
223
197
|
}
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
if (raw == null || raw === '')
|
|
231
|
-
return null;
|
|
232
|
-
if (typeof raw === 'string') {
|
|
233
|
-
const parsed = Date.parse(raw);
|
|
234
|
-
return Number.isFinite(parsed) ? parsed : null;
|
|
235
|
-
}
|
|
236
|
-
const at = Number(raw);
|
|
237
|
-
return Number.isFinite(at) ? at : null;
|
|
238
|
-
}
|
|
239
|
-
function verifiedAt(range) {
|
|
240
|
-
return timestampMs(range?.at);
|
|
241
|
-
}
|
|
242
|
-
/**
|
|
243
|
-
* Normalize recorded spans: sort, then merge overlapping / bucket-adjacent
|
|
244
|
-
* ones — but ONLY when they assert the same verification time. Merging spans
|
|
245
|
-
* with different `at` would have to pick one verdict for the whole union, and
|
|
246
|
-
* either pick loses (min → the merged span vouches for buckets the older
|
|
247
|
-
* query never saw; max → buckets verified long ago look freshly verified).
|
|
248
|
-
* Keeping them separate preserves exact per-bucket freshness, which is the
|
|
249
|
-
* whole point of `at`. Spans therefore accumulate ~1 per tail run, each a few
|
|
250
|
-
* dozen bytes — negligible next to the candles in the same file, and the
|
|
251
|
-
* same-`at` case (bulk fetches, and every pre-`at` shard) still collapses to
|
|
252
|
-
* one span as before.
|
|
253
|
-
*/
|
|
254
|
-
function unionQueriedRanges(ranges, bucketMs) {
|
|
255
|
-
const clean = (ranges || [])
|
|
256
|
-
.filter((q) => q && Number.isFinite(Number(q.gte)) && Number.isFinite(Number(q.lte)) && Number(q.lte) >= Number(q.gte))
|
|
257
|
-
.map((q) => ({ gte: Number(q.gte), lte: Number(q.lte), at: verifiedAt(q) }))
|
|
258
|
-
.sort((a, b) => a.gte - b.gte || a.lte - b.lte);
|
|
259
|
-
const merged = [];
|
|
260
|
-
const gap = Number.isFinite(Number(bucketMs)) && Number(bucketMs) > 0 ? Number(bucketMs) : 0;
|
|
261
|
-
for (const q of clean) {
|
|
262
|
-
const top = merged[merged.length - 1];
|
|
263
|
-
if (top && q.gte <= top.lte + gap && top.at === q.at) {
|
|
264
|
-
if (q.lte > top.lte)
|
|
265
|
-
top.lte = q.lte;
|
|
266
|
-
}
|
|
267
|
-
else {
|
|
268
|
-
merged.push({ gte: q.gte, lte: q.lte, at: q.at });
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
return merged;
|
|
272
|
-
}
|
|
273
|
-
function rangesCoveredBy(have, want) {
|
|
274
|
-
for (const w of want || []) {
|
|
275
|
-
let covered = false;
|
|
276
|
-
for (const h of have || []) {
|
|
277
|
-
if (h.gte <= w.gte && h.lte >= w.lte) {
|
|
278
|
-
covered = true;
|
|
279
|
-
break;
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
if (!covered)
|
|
283
|
-
return false;
|
|
284
|
-
}
|
|
285
|
-
return true;
|
|
286
|
-
}
|
|
287
|
-
/**
|
|
288
|
-
* Change detection for shard writes: `have` already accounts for every span in
|
|
289
|
-
* `want`, in extent AND in verification time. Without the `at` half, a run
|
|
290
|
-
* whose only news is "these buckets were re-verified 1h later" would skip the
|
|
291
|
-
* write and the next run would fall back to the wide fixed refresh window.
|
|
292
|
-
* An unknown `at` on either side is never treated as fresh.
|
|
293
|
-
*/
|
|
294
|
-
function coverageSatisfied(have, want) {
|
|
295
|
-
if (!rangesCoveredBy(have, want))
|
|
296
|
-
return false;
|
|
297
|
-
for (const w of want || []) {
|
|
298
|
-
const wAt = verifiedAt(w);
|
|
299
|
-
if (wAt == null)
|
|
300
|
-
continue;
|
|
301
|
-
let fresh = false;
|
|
302
|
-
for (const h of have || []) {
|
|
303
|
-
if (h.gte > w.gte || h.lte < w.lte)
|
|
304
|
-
continue;
|
|
305
|
-
const hAt = verifiedAt(h);
|
|
306
|
-
if (hAt != null && hAt >= wAt) {
|
|
307
|
-
fresh = true;
|
|
308
|
-
break;
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
if (!fresh)
|
|
312
|
-
return false;
|
|
313
|
-
}
|
|
314
|
-
return true;
|
|
315
|
-
}
|
|
316
|
-
/** Every recorded span across the loaded files, newest verification included. */
|
|
317
|
-
function allQueriedRanges(fileCover) {
|
|
318
|
-
const out = [];
|
|
319
|
-
for (const f of fileCover || []) {
|
|
320
|
-
for (const q of f?.queried || []) {
|
|
321
|
-
if (q && Number.isFinite(Number(q.gte)) && Number.isFinite(Number(q.lte)))
|
|
322
|
-
out.push(q);
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
return out;
|
|
326
|
-
}
|
|
198
|
+
// ─── Queried-range set ops ────────────────────────────────────────────────────
|
|
199
|
+
// Coverage provenance: normalize recorded spans (sort, merge overlapping or
|
|
200
|
+
// bucket-adjacent) so growth checks and absorption decisions are exact.
|
|
201
|
+
// A span also carries the time its covering query RAN (`at`), which is what
|
|
202
|
+
// makes the tail refresh incremental: extent says "we asked", `at` says "and
|
|
203
|
+
// the answer is still current".
|
|
327
204
|
/**
|
|
328
205
|
* How settled a window is, given the recorded coverage.
|
|
329
206
|
*
|
|
@@ -359,29 +236,6 @@ function settleCoverage(opts) {
|
|
|
359
236
|
}
|
|
360
237
|
return { hasTimestamps: true, firstUnsettled: null };
|
|
361
238
|
}
|
|
362
|
-
/**
|
|
363
|
-
* Bound the persisted span list. Because spans merge only when they assert
|
|
364
|
-
* the same `at`, a shard that is re-verified on every run gains one span per
|
|
365
|
-
* run and would grow without limit (and `settleCoverage` scans the list once
|
|
366
|
-
* per tail window). Dropping the OLDEST spans never invents emptiness, but it
|
|
367
|
-
* is not free: coverage is consulted by BOTH `pruneImmutableGaps` (to skip a
|
|
368
|
-
* query) and `settleCoverage` (to decide a bucket is fresh). A dropped span
|
|
369
|
-
* makes its buckets look unsettled, and the tail refresh would otherwise start
|
|
370
|
-
* at the earliest of them. That is why the refresh is capped at
|
|
371
|
-
* TAIL_REFRESH_HOURS: the worst a dropped span can cost is the legacy 48h
|
|
372
|
-
* overlap, never weeks. The input is already sorted by extent, so the tail of
|
|
373
|
-
* the list is the newest.
|
|
374
|
-
*/
|
|
375
|
-
function compactCoverage(spans, maxSpans) {
|
|
376
|
-
const list = spans || [];
|
|
377
|
-
const max = Number.isFinite(Number(maxSpans)) && Number(maxSpans) >= 1 ? Math.floor(Number(maxSpans)) : list.length;
|
|
378
|
-
return list.length <= max ? list : list.slice(list.length - max);
|
|
379
|
-
}
|
|
380
|
-
function clipRangeTo(q, gteMs, lteMs) {
|
|
381
|
-
const gte = Math.max(q.gte, gteMs);
|
|
382
|
-
const lte = Math.min(q.lte, lteMs);
|
|
383
|
-
return lte >= gte ? { gte, lte, at: verifiedAt(q) } : null;
|
|
384
|
-
}
|
|
385
239
|
function addUtcMonths(date, months) {
|
|
386
240
|
const result = new Date(date.getTime());
|
|
387
241
|
const day = result.getUTCDate();
|
|
@@ -391,6 +245,18 @@ function addUtcMonths(date, months) {
|
|
|
391
245
|
result.setUTCDate(Math.min(day, lastDay));
|
|
392
246
|
return result;
|
|
393
247
|
}
|
|
248
|
+
/** First instant of the UTC month after `date`. */
|
|
249
|
+
function nextUtcMonthStart(date) {
|
|
250
|
+
return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 1));
|
|
251
|
+
}
|
|
252
|
+
/** True when `date` sits exactly on a UTC month start (midnight, day 1). */
|
|
253
|
+
function isUtcMonthStart(date) {
|
|
254
|
+
return date.getUTCDate() === 1
|
|
255
|
+
&& date.getUTCHours() === 0
|
|
256
|
+
&& date.getUTCMinutes() === 0
|
|
257
|
+
&& date.getUTCSeconds() === 0
|
|
258
|
+
&& date.getUTCMilliseconds() === 0;
|
|
259
|
+
}
|
|
394
260
|
function normalizeDateInput(raw, label) {
|
|
395
261
|
const ms = Date.parse(String(raw || ''));
|
|
396
262
|
if (!Number.isFinite(ms)) {
|
|
@@ -404,10 +270,20 @@ function buildFetchWindowsFromRange(timeRange, chunkMonths) {
|
|
|
404
270
|
if (start >= end) {
|
|
405
271
|
throw new Error(`Invalid fetch range: ${start.toISOString()} must be earlier than ${end.toISOString()}`);
|
|
406
272
|
}
|
|
273
|
+
const step = Number.isFinite(Number(chunkMonths)) && Number(chunkMonths) >= 1
|
|
274
|
+
? Math.floor(Number(chunkMonths))
|
|
275
|
+
: 1;
|
|
276
|
+
// Interior window boundaries snap to UTC month starts — the same grid the
|
|
277
|
+
// storage shards use — so a window maps onto whole calendar months instead
|
|
278
|
+
// of straddling two of them. The requested start/end stay exact: only the
|
|
279
|
+
// first window can be short (a mid-month start), and the last is clipped
|
|
280
|
+
// to `end`.
|
|
407
281
|
const windows = [];
|
|
408
282
|
let cursor = start;
|
|
409
283
|
while (cursor < end) {
|
|
410
|
-
const next =
|
|
284
|
+
const next = isUtcMonthStart(cursor)
|
|
285
|
+
? addUtcMonths(cursor, step)
|
|
286
|
+
: nextUtcMonthStart(cursor);
|
|
411
287
|
const windowEnd = next < end ? next : end;
|
|
412
288
|
windows.push({
|
|
413
289
|
gte: cursor.toISOString(),
|
|
@@ -674,9 +550,6 @@ async function fetchRangeLogged(fetchRange, opts) {
|
|
|
674
550
|
console.log(formatWindowLine(opts.unit, opts.index, opts.total, opts.gte, opts.lte, `-> ${candles.length} candles (${((Date.now() - startMs) / 1000).toFixed(1)}s)${attemptNote}${partialNote}${note}`));
|
|
675
551
|
return { candles, complete };
|
|
676
552
|
}
|
|
677
|
-
function higherVolumeWins(existing, incoming) {
|
|
678
|
-
return incoming[5] > existing[5] ? incoming : existing;
|
|
679
|
-
}
|
|
680
553
|
function sortedCandles(byTs) {
|
|
681
554
|
return [...byTs.values()].sort((a, b) => Number(a[0]) - Number(b[0]));
|
|
682
555
|
}
|
|
@@ -872,6 +745,9 @@ async function runCachedWindows(opts) {
|
|
|
872
745
|
// else on disk is already current.
|
|
873
746
|
if (overall) {
|
|
874
747
|
const fetchedAt = new Date().toISOString();
|
|
748
|
+
// O(1) lookup of each shard's loaded candles for the write-on-change
|
|
749
|
+
// comparison below (instead of a linear scan per shard state).
|
|
750
|
+
const loadedShardCandles = new Map(localCache.shards.map((s) => [s.shardKey, s.candles]));
|
|
875
751
|
for (const state of shardStates.values()) {
|
|
876
752
|
const { start, end } = shardBoundsForKey(state.key);
|
|
877
753
|
// Fold every loaded bucket into its home shard. Presence on disk
|
|
@@ -884,7 +760,7 @@ async function runCachedWindows(opts) {
|
|
|
884
760
|
if (!prev || Number(c[5] || 0) > Number(prev[5] || 0))
|
|
885
761
|
state.candles.set(ts, c);
|
|
886
762
|
}
|
|
887
|
-
const before = sortedCandles(new Map((
|
|
763
|
+
const before = sortedCandles(new Map((loadedShardCandles.get(state.key) || [])
|
|
888
764
|
.filter((c) => Array.isArray(c) && Number.isFinite(Number(c[0])))
|
|
889
765
|
.map((c) => [Number(c[0]), c])));
|
|
890
766
|
const after = sortedCandles(state.candles);
|
|
@@ -903,7 +779,6 @@ async function runCachedWindows(opts) {
|
|
|
903
779
|
const meta = { ...metaForWindow(synthWindow), fetchedAt, queriedRanges: union };
|
|
904
780
|
meta.timeRange = { gte: synthWindow.gte, lte: synthWindow.lte };
|
|
905
781
|
meta.shard = state.key;
|
|
906
|
-
delete meta.chunkIndex;
|
|
907
782
|
persistCacheChunk(state.file, meta, after);
|
|
908
783
|
state.queried = union;
|
|
909
784
|
state.pendingQueried = [];
|
|
@@ -911,5 +786,5 @@ async function runCachedWindows(opts) {
|
|
|
911
786
|
}
|
|
912
787
|
return merged;
|
|
913
788
|
}
|
|
914
|
-
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS,
|
|
789
|
+
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
|
|
915
790
|
//# sourceMappingURL=window_cache.js.map
|
|
@@ -23,5 +23,15 @@ declare function bucketStartMs(nowMs: number, intervalSeconds: number): number |
|
|
|
23
23
|
* evaluate the interval/clock.
|
|
24
24
|
*/
|
|
25
25
|
declare function latestClosedBucketStartMs(nowMs: number, intervalSeconds: number): number | null;
|
|
26
|
-
|
|
26
|
+
/**
|
|
27
|
+
* Parse a BitShares chain timestamp (e.g. `block_time`) to epoch ms.
|
|
28
|
+
*
|
|
29
|
+
* Chain timestamps are ISO-8601 without a trailing `Z`; `Date.parse` treats
|
|
30
|
+
* them as local time, so append `Z` unless present. Single home for the helper
|
|
31
|
+
* previously copied into market_adapter.ts, kibana_candles.ts,
|
|
32
|
+
* kibana_feed_source.ts, scripts/diagnose-pool-history.ts and
|
|
33
|
+
* scripts/chart_command.ts. Returns `NaN` for empty/unparseable input.
|
|
34
|
+
*/
|
|
35
|
+
declare function parseChainTimeToMs(timeStr: unknown): number;
|
|
36
|
+
export { toIntervalLabel, slugPart, bucketStartMs, latestClosedBucketStartMs, parseChainTimeToMs };
|
|
27
37
|
//# sourceMappingURL=interval_utils.d.ts.map
|
|
@@ -51,5 +51,20 @@ function latestClosedBucketStartMs(nowMs, intervalSeconds) {
|
|
|
51
51
|
return null;
|
|
52
52
|
return currentBucketStart - Number(intervalSeconds) * 1000;
|
|
53
53
|
}
|
|
54
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Parse a BitShares chain timestamp (e.g. `block_time`) to epoch ms.
|
|
56
|
+
*
|
|
57
|
+
* Chain timestamps are ISO-8601 without a trailing `Z`; `Date.parse` treats
|
|
58
|
+
* them as local time, so append `Z` unless present. Single home for the helper
|
|
59
|
+
* previously copied into market_adapter.ts, kibana_candles.ts,
|
|
60
|
+
* kibana_feed_source.ts, scripts/diagnose-pool-history.ts and
|
|
61
|
+
* scripts/chart_command.ts. Returns `NaN` for empty/unparseable input.
|
|
62
|
+
*/
|
|
63
|
+
function parseChainTimeToMs(timeStr) {
|
|
64
|
+
if (!timeStr)
|
|
65
|
+
return Number.NaN;
|
|
66
|
+
const s = String(timeStr);
|
|
67
|
+
return Date.parse(s.endsWith('Z') ? s : `${s}Z`);
|
|
68
|
+
}
|
|
69
|
+
export { toIntervalLabel, slugPart, bucketStartMs, latestClosedBucketStartMs, parseChainTimeToMs };
|
|
55
70
|
//# sourceMappingURL=interval_utils.js.map
|
|
@@ -52,6 +52,7 @@ interface CliArgOptions {
|
|
|
52
52
|
}
|
|
53
53
|
interface MarketChartOptions {
|
|
54
54
|
dataFile?: string;
|
|
55
|
+
dataBundle?: LpDataBundle;
|
|
55
56
|
logger?: {
|
|
56
57
|
log: (...args: unknown[]) => void;
|
|
57
58
|
};
|
|
@@ -61,6 +62,7 @@ interface MarketChartOptions {
|
|
|
61
62
|
}
|
|
62
63
|
interface ComparisonChartOptions {
|
|
63
64
|
dataFile?: string;
|
|
65
|
+
dataBundle?: LpDataBundle;
|
|
64
66
|
logger?: {
|
|
65
67
|
log: (...args: unknown[]) => void;
|
|
66
68
|
};
|
|
@@ -71,6 +73,7 @@ interface ComparisonChartOptions {
|
|
|
71
73
|
}
|
|
72
74
|
interface BundleChartOptions {
|
|
73
75
|
dataFile?: string;
|
|
76
|
+
dataBundle?: LpDataBundle;
|
|
74
77
|
logger?: {
|
|
75
78
|
log: (...args: unknown[]) => void;
|
|
76
79
|
};
|