dexbot 1.6.12 → 1.6.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -1
- package/analysis/README.md +9 -11
- package/analysis/ama_fitting/README.md +5 -3
- package/claw/README.md +1 -0
- package/claw/docs/AI_BOT_LIBRARY_API.md +2 -0
- package/claw/modules/claw_bridge.ts +0 -3
- package/claw/modules/claw_skill_md.ts +1 -0
- package/claw/package.json +3 -3
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/claw/tsconfig.json +2 -2
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +1 -14
- package/dist/analysis/ama_fitting/fetch_lp_candles.js +51 -41
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -24
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +36 -18
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js +18 -66
- package/dist/analysis/analyze_args.d.ts +12 -0
- package/dist/analysis/analyze_args.js +20 -0
- package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +1 -16
- package/dist/analysis/bot_fitting/backtest_ama_sweep.js +20 -101
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js +26 -92
- package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
- package/dist/analysis/bot_fitting/shared_utils.d.ts +71 -3
- package/dist/analysis/bot_fitting/shared_utils.js +124 -8
- package/dist/analysis/chart_ui.d.ts +30 -1
- package/dist/analysis/chart_ui.js +37 -1
- package/dist/analysis/fills_cache.d.ts +3 -23
- package/dist/analysis/fills_cache.js +3 -84
- package/dist/analysis/fills_source.d.ts +31 -1
- package/dist/analysis/fills_source.js +69 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -23
- package/dist/analysis/grid_correction_check.js +15 -58
- package/dist/analysis/math_utils.d.ts +85 -1
- package/dist/analysis/math_utils.js +116 -1
- package/dist/analysis/price_sources.d.ts +22 -1
- package/dist/analysis/price_sources.js +23 -0
- package/dist/analysis/resolve_source.js +31 -10
- package/dist/analysis/trade_profitability.d.ts +6 -0
- package/dist/analysis/trade_profitability.js +23 -97
- package/dist/analysis/tradingview/analyze_tradingview.d.ts +1 -17
- package/dist/analysis/tradingview/analyze_tradingview.js +1 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +2 -15
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +22 -80
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +11 -37
- package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
- package/dist/analysis/worker_pool.d.ts +38 -0
- package/dist/analysis/worker_pool.js +67 -0
- package/dist/dexbot.js +67 -48
- package/dist/market_adapter/ama_signal_runner.js +21 -54
- package/dist/market_adapter/candle_utils.d.ts +7 -1
- package/dist/market_adapter/candle_utils.js +11 -1
- package/dist/market_adapter/cli_args.d.ts +22 -0
- package/dist/market_adapter/cli_args.js +30 -0
- package/dist/market_adapter/core/config_normalizers.js +11 -6
- package/dist/market_adapter/core/kibana_candles.d.ts +3 -1
- package/dist/market_adapter/core/kibana_candles.js +3 -2
- package/dist/market_adapter/core/market_adapter_service.d.ts +85 -85
- package/dist/market_adapter/core/market_adapter_service.js +93 -158
- package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
- package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
- package/dist/market_adapter/inputs/fetch_book_data.js +0 -1
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +63 -107
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +8 -1
- package/dist/market_adapter/inputs/fetch_lp_data.js +25 -7
- package/dist/market_adapter/inputs/kibana_feed_source.js +4 -26
- package/dist/market_adapter/inputs/window_cache.d.ts +4 -72
- package/dist/market_adapter/inputs/window_cache.js +52 -177
- package/dist/market_adapter/interval_utils.d.ts +11 -1
- package/dist/market_adapter/interval_utils.js +16 -1
- package/dist/market_adapter/lp_chart_runner.d.ts +3 -0
- package/dist/market_adapter/lp_chart_runner.js +37 -19
- package/dist/market_adapter/market_adapter.js +27 -72
- package/dist/market_adapter/utils/data_discovery.d.ts +45 -1
- package/dist/market_adapter/utils/data_discovery.js +220 -27
- package/dist/market_adapter/utils/month_shards.d.ts +84 -0
- package/dist/market_adapter/utils/month_shards.js +237 -0
- package/dist/modules/account_bots.d.ts +1 -0
- package/dist/modules/account_bots.js +102 -121
- package/dist/modules/account_orders.d.ts +13 -0
- package/dist/modules/account_orders.js +38 -44
- package/dist/modules/bitshares-native/chain_client.d.ts +1 -1
- package/dist/modules/bitshares-native/chain_client.js +14 -13
- package/dist/modules/bitshares-native/resolvers.js +22 -25
- package/dist/modules/bitshares-native/serial/types.js +22 -50
- package/dist/modules/bitshares-native/signing_client.d.ts +1 -1
- package/dist/modules/bot_defaults.d.ts +2 -1
- package/dist/modules/bot_settings.d.ts +1 -1
- package/dist/modules/broadcast_failure.d.ts +2 -1
- package/dist/modules/chain_orders.d.ts +1 -1
- package/dist/modules/chain_orders.js +51 -96
- package/dist/modules/cli_start_onboarding.d.ts +2 -1
- package/dist/modules/constants.d.ts +7 -1
- package/dist/modules/constants.js +20 -1
- package/dist/modules/cr_planner.d.ts +1 -1
- package/dist/modules/cr_planner.js +1 -5
- package/dist/modules/credit_pricing.d.ts +2 -1
- package/dist/modules/credit_pricing.js +36 -40
- package/dist/modules/credit_runtime.d.ts +7 -0
- package/dist/modules/credit_runtime.js +45 -73
- package/dist/modules/daemon_node_health.d.ts +4 -3
- package/dist/modules/dexbot_class.d.ts +28 -17
- package/dist/modules/dexbot_class.js +105 -58
- package/dist/modules/dexbot_cow_runtime.d.ts +0 -7
- package/dist/modules/dexbot_cow_runtime.js +54 -28
- package/dist/modules/dexbot_credential_client.d.ts +0 -7
- package/dist/modules/dexbot_credential_client.js +1 -2
- package/dist/modules/dexbot_fill_runtime.d.ts +1 -2
- package/dist/modules/dexbot_fill_runtime.js +5 -6
- package/dist/modules/dexbot_maintenance_runtime.d.ts +20 -3
- package/dist/modules/dexbot_maintenance_runtime.js +84 -69
- package/dist/modules/dexbot_startup_runtime.d.ts +0 -1
- package/dist/modules/dexbot_startup_runtime.js +13 -14
- package/dist/modules/general_settings.d.ts +1 -1
- package/dist/modules/graceful_shutdown.js +17 -11
- package/dist/modules/key_store.js +0 -1
- package/dist/modules/lazy_shim.d.ts +11 -0
- package/dist/modules/lazy_shim.js +16 -0
- package/dist/modules/node_connect_policy.d.ts +3 -2
- package/dist/modules/node_failure_ledger.d.ts +6 -5
- package/dist/modules/order/format.d.ts +12 -3
- package/dist/modules/order/format.js +15 -3
- package/dist/modules/order/genesis_policy.d.ts +3 -3
- package/dist/modules/order/genesis_policy.js +5 -2
- package/dist/modules/order/grid.d.ts +4 -2
- package/dist/modules/order/grid.js +91 -35
- package/dist/modules/order/index.d.ts +1 -0
- package/dist/modules/order/logger.d.ts +1 -8
- package/dist/modules/order/logger.js +0 -9
- package/dist/modules/order/manager.d.ts +9 -13
- package/dist/modules/order/manager.js +33 -50
- package/dist/modules/order/utils/order.js +39 -35
- package/dist/modules/order/utils/system.d.ts +9 -29
- package/dist/modules/order/utils/system.js +17 -2
- package/dist/modules/order/utils/validate.js +2 -2
- package/dist/modules/order/utils/withPoolRef.d.ts +2 -1
- package/dist/modules/paths.js +32 -34
- package/dist/modules/runtime_settings.d.ts +2 -4
- package/dist/modules/settings_merge.d.ts +2 -2
- package/dist/modules/socket_json_client.d.ts +2 -1
- package/dist/modules/types.d.ts +42 -27
- package/dist/modules/utils/errors.js +1 -3
- package/dist/modules/version_notice.d.ts +11 -10
- package/dist/pm2.d.ts +0 -1
- package/dist/pm2.js +56 -35
- package/dist/scripts/analyze-git.js +2 -15
- package/dist/scripts/chart_command.js +2 -2
- package/dist/scripts/check_any.d.ts +4 -0
- package/dist/scripts/check_any.js +53 -21
- package/dist/scripts/diagnose-pool-history.js +1 -6
- package/dist/scripts/run-tests.js +0 -1
- package/dist/scripts/sync-version.js +0 -10
- package/docs/BITSHARES_ONBOARDING.md +8 -12
- package/docs/COPY_ON_WRITE_MASTER_PLAN.md +1 -1
- package/docs/DEXBOT_COMPARISON.md +6 -6
- package/docs/EVOLUTION.md +9 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/README.md +1 -1
- package/docs/WORKFLOW.md +1 -0
- package/docs/architecture.md +19 -10
- package/market_adapter/README.md +6 -4
- package/package.json +6 -9
- package/scripts/README.md +43 -38
- package/tests/README.md +25 -10
- package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
- package/dist/scripts/diagnose-kibana-candles.js +0 -82
- package/scripts/git-viewer.sh +0 -489
- package/scripts/postinstall.js +0 -16
|
@@ -21,7 +21,7 @@ const { getAmaWarmupBars } = require('../core/strategies/ama');
|
|
|
21
21
|
const { getErrorMessage } = require('../../modules/utils/errors');
|
|
22
22
|
const { isSameBotName } = require('../../modules/utils/sanitize_key');
|
|
23
23
|
const { candleFileForBot } = require('../../analysis/bot_key_utils');
|
|
24
|
-
const { DEFAULTS: MARKET_ADAPTER_DEFAULTS, resolveAmaForBot, resolveBotCfg, } = require('../market_adapter
|
|
24
|
+
const { DEFAULTS: MARKET_ADAPTER_DEFAULTS, resolveAmaForBot, resolveBotCfg, } = require('../market_adapter');
|
|
25
25
|
const DEFAULT_INTERVAL = '1h';
|
|
26
26
|
const DEFAULT_LIMIT = 1000;
|
|
27
27
|
const DEFAULT_BASE = 'XRP';
|
|
@@ -237,6 +237,61 @@ function extractMarketsFromList(list, mapper) {
|
|
|
237
237
|
const rows = asArray(list);
|
|
238
238
|
return rows.map(mapper).filter((row) => Boolean(row && row.id && row.base && row.quote));
|
|
239
239
|
}
|
|
240
|
+
// Interval → venue-specific unit tables. Most CEX APIs share one of a few
|
|
241
|
+
// shapes; keep one table per shape and compose where a venue overrides a
|
|
242
|
+
// single unit (Kraken day/week, MEXC's case-sensitive `1W`).
|
|
243
|
+
const INTERVAL_MINUTES = {
|
|
244
|
+
'1m': '1',
|
|
245
|
+
'5m': '5',
|
|
246
|
+
'15m': '15',
|
|
247
|
+
'30m': '30',
|
|
248
|
+
'1h': '60',
|
|
249
|
+
'4h': '240',
|
|
250
|
+
'6h': '360',
|
|
251
|
+
'12h': '720',
|
|
252
|
+
'1d': 'D',
|
|
253
|
+
'1w': 'W',
|
|
254
|
+
};
|
|
255
|
+
const INTERVAL_KRAKEN = {
|
|
256
|
+
...INTERVAL_MINUTES,
|
|
257
|
+
'1d': '1440',
|
|
258
|
+
'1w': '10080',
|
|
259
|
+
};
|
|
260
|
+
const INTERVAL_DURATION = {
|
|
261
|
+
'1m': '1min',
|
|
262
|
+
'5m': '5min',
|
|
263
|
+
'15m': '15min',
|
|
264
|
+
'30m': '30min',
|
|
265
|
+
'1h': '1hour',
|
|
266
|
+
'4h': '4hour',
|
|
267
|
+
'6h': '6hour',
|
|
268
|
+
'12h': '12hour',
|
|
269
|
+
'1d': '1day',
|
|
270
|
+
'1w': '1week',
|
|
271
|
+
};
|
|
272
|
+
const INTERVAL_HTX = {
|
|
273
|
+
...INTERVAL_DURATION,
|
|
274
|
+
'1h': '60min',
|
|
275
|
+
};
|
|
276
|
+
const INTERVAL_BITGET = {
|
|
277
|
+
'1m': '1m',
|
|
278
|
+
'5m': '5m',
|
|
279
|
+
'15m': '15m',
|
|
280
|
+
'30m': '30m',
|
|
281
|
+
'1h': '1h',
|
|
282
|
+
'4h': '4h',
|
|
283
|
+
'6h': '6h',
|
|
284
|
+
'12h': '12h',
|
|
285
|
+
'1d': '1d',
|
|
286
|
+
'1w': '1w',
|
|
287
|
+
};
|
|
288
|
+
const INTERVAL_MEXC = {
|
|
289
|
+
...INTERVAL_BITGET,
|
|
290
|
+
'1h': '60m',
|
|
291
|
+
// MEXC interval enums are case-sensitive
|
|
292
|
+
'1w': '1W',
|
|
293
|
+
};
|
|
294
|
+
const intervalFormatter = (map) => (interval) => map[lower(interval)] || interval;
|
|
240
295
|
const EXCHANGES = {
|
|
241
296
|
binance: {
|
|
242
297
|
name: 'Binance',
|
|
@@ -261,21 +316,7 @@ const EXCHANGES = {
|
|
|
261
316
|
},
|
|
262
317
|
bybit: {
|
|
263
318
|
name: 'Bybit',
|
|
264
|
-
formatInterval: (
|
|
265
|
-
const map = {
|
|
266
|
-
'1m': '1',
|
|
267
|
-
'5m': '5',
|
|
268
|
-
'15m': '15',
|
|
269
|
-
'30m': '30',
|
|
270
|
-
'1h': '60',
|
|
271
|
-
'4h': '240',
|
|
272
|
-
'6h': '360',
|
|
273
|
-
'12h': '720',
|
|
274
|
-
'1d': 'D',
|
|
275
|
-
'1w': 'W',
|
|
276
|
-
};
|
|
277
|
-
return map[lower(interval)] || interval;
|
|
278
|
-
},
|
|
319
|
+
formatInterval: intervalFormatter(INTERVAL_MINUTES),
|
|
279
320
|
marketsUrl: 'https://api.bybit.com/v5/market/instruments-info?category=spot&limit=1000',
|
|
280
321
|
candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
|
|
281
322
|
const url = new URL('https://api.bybit.com/v5/market/kline');
|
|
@@ -318,21 +359,7 @@ const EXCHANGES = {
|
|
|
318
359
|
},
|
|
319
360
|
bitget: {
|
|
320
361
|
name: 'Bitget',
|
|
321
|
-
formatInterval: (
|
|
322
|
-
const map = {
|
|
323
|
-
'1m': '1m',
|
|
324
|
-
'5m': '5m',
|
|
325
|
-
'15m': '15m',
|
|
326
|
-
'30m': '30m',
|
|
327
|
-
'1h': '1h',
|
|
328
|
-
'4h': '4h',
|
|
329
|
-
'6h': '6h',
|
|
330
|
-
'12h': '12h',
|
|
331
|
-
'1d': '1d',
|
|
332
|
-
'1w': '1w',
|
|
333
|
-
};
|
|
334
|
-
return map[lower(interval)] || interval;
|
|
335
|
-
},
|
|
362
|
+
formatInterval: intervalFormatter(INTERVAL_BITGET),
|
|
336
363
|
marketsUrl: 'https://api.bitget.com/api/v2/spot/public/symbols',
|
|
337
364
|
candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
|
|
338
365
|
const url = new URL('https://api.bitget.com/api/v2/spot/market/candles');
|
|
@@ -353,21 +380,7 @@ const EXCHANGES = {
|
|
|
353
380
|
},
|
|
354
381
|
kucoin: {
|
|
355
382
|
name: 'KuCoin',
|
|
356
|
-
formatInterval: (
|
|
357
|
-
const map = {
|
|
358
|
-
'1m': '1min',
|
|
359
|
-
'5m': '5min',
|
|
360
|
-
'15m': '15min',
|
|
361
|
-
'30m': '30min',
|
|
362
|
-
'1h': '1hour',
|
|
363
|
-
'4h': '4hour',
|
|
364
|
-
'6h': '6hour',
|
|
365
|
-
'12h': '12hour',
|
|
366
|
-
'1d': '1day',
|
|
367
|
-
'1w': '1week',
|
|
368
|
-
};
|
|
369
|
-
return map[lower(interval)] || interval;
|
|
370
|
-
},
|
|
383
|
+
formatInterval: intervalFormatter(INTERVAL_DURATION),
|
|
371
384
|
marketsUrl: 'https://api.kucoin.com/api/v2/symbols',
|
|
372
385
|
candlesUrl: ({ id, interval, sinceMs, untilMs }) => {
|
|
373
386
|
const url = new URL('https://api.kucoin.com/api/v1/market/candles');
|
|
@@ -387,21 +400,7 @@ const EXCHANGES = {
|
|
|
387
400
|
},
|
|
388
401
|
htx: {
|
|
389
402
|
name: 'HTX',
|
|
390
|
-
formatInterval: (
|
|
391
|
-
const map = {
|
|
392
|
-
'1m': '1min',
|
|
393
|
-
'5m': '5min',
|
|
394
|
-
'15m': '15min',
|
|
395
|
-
'30m': '30min',
|
|
396
|
-
'1h': '60min',
|
|
397
|
-
'4h': '4hour',
|
|
398
|
-
'6h': '6hour',
|
|
399
|
-
'12h': '12hour',
|
|
400
|
-
'1d': '1day',
|
|
401
|
-
'1w': '1week',
|
|
402
|
-
};
|
|
403
|
-
return map[lower(interval)] || interval;
|
|
404
|
-
},
|
|
403
|
+
formatInterval: intervalFormatter(INTERVAL_HTX),
|
|
405
404
|
marketsUrl: 'https://api.htx.com/v1/common/symbols',
|
|
406
405
|
candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
|
|
407
406
|
// Official Huobi/HTX spot historical kline endpoint is /market/history/kline
|
|
@@ -433,21 +432,7 @@ const EXCHANGES = {
|
|
|
433
432
|
},
|
|
434
433
|
kraken: {
|
|
435
434
|
name: 'Kraken',
|
|
436
|
-
formatInterval: (
|
|
437
|
-
const map = {
|
|
438
|
-
'1m': '1',
|
|
439
|
-
'5m': '5',
|
|
440
|
-
'15m': '15',
|
|
441
|
-
'30m': '30',
|
|
442
|
-
'1h': '60',
|
|
443
|
-
'4h': '240',
|
|
444
|
-
'6h': '360',
|
|
445
|
-
'12h': '720',
|
|
446
|
-
'1d': '1440',
|
|
447
|
-
'1w': '10080',
|
|
448
|
-
};
|
|
449
|
-
return map[lower(interval)] || interval;
|
|
450
|
-
},
|
|
435
|
+
formatInterval: intervalFormatter(INTERVAL_KRAKEN),
|
|
451
436
|
marketsUrl: 'https://api.kraken.com/0/public/AssetPairs',
|
|
452
437
|
candlesUrl: ({ id, interval, intervalSeconds, limit, sinceMs }) => {
|
|
453
438
|
const url = new URL('https://api.kraken.com/0/public/OHLC');
|
|
@@ -481,21 +466,7 @@ const EXCHANGES = {
|
|
|
481
466
|
name: 'OKX',
|
|
482
467
|
// /api/v5/market/candles caps limit at CEX_PAGE_LIMIT_CAPS.okx per request
|
|
483
468
|
maxLimit: MARKET_ADAPTER.CEX_PAGE_LIMIT_CAPS.okx,
|
|
484
|
-
formatInterval: (
|
|
485
|
-
const map = {
|
|
486
|
-
'1m': '1',
|
|
487
|
-
'5m': '5',
|
|
488
|
-
'15m': '15',
|
|
489
|
-
'30m': '30',
|
|
490
|
-
'1h': '60',
|
|
491
|
-
'4h': '240',
|
|
492
|
-
'6h': '360',
|
|
493
|
-
'12h': '720',
|
|
494
|
-
'1d': 'D',
|
|
495
|
-
'1w': 'W',
|
|
496
|
-
};
|
|
497
|
-
return map[lower(interval)] || interval;
|
|
498
|
-
},
|
|
469
|
+
formatInterval: intervalFormatter(INTERVAL_MINUTES),
|
|
499
470
|
marketsUrl: 'https://www.okx.com/api/v5/public/instruments?instType=SPOT',
|
|
500
471
|
candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
|
|
501
472
|
const url = new URL('https://www.okx.com/api/v5/market/candles');
|
|
@@ -518,22 +489,7 @@ const EXCHANGES = {
|
|
|
518
489
|
name: 'MEXC',
|
|
519
490
|
// /api/v3/klines caps limit at CEX_PAGE_LIMIT_CAPS.mexc per request
|
|
520
491
|
maxLimit: MARKET_ADAPTER.CEX_PAGE_LIMIT_CAPS.mexc,
|
|
521
|
-
formatInterval: (
|
|
522
|
-
const map = {
|
|
523
|
-
'1m': '1m',
|
|
524
|
-
'5m': '5m',
|
|
525
|
-
'15m': '15m',
|
|
526
|
-
'30m': '30m',
|
|
527
|
-
'1h': '60m',
|
|
528
|
-
'4h': '4h',
|
|
529
|
-
'6h': '6h',
|
|
530
|
-
'12h': '12h',
|
|
531
|
-
'1d': '1d',
|
|
532
|
-
// MEXC interval enums are case-sensitive
|
|
533
|
-
'1w': '1W',
|
|
534
|
-
};
|
|
535
|
-
return map[lower(interval)] || interval;
|
|
536
|
-
},
|
|
492
|
+
formatInterval: intervalFormatter(INTERVAL_MEXC),
|
|
537
493
|
marketsUrl: 'https://api.mexc.com/api/v3/exchangeInfo',
|
|
538
494
|
candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
|
|
539
495
|
const url = new URL('https://api.mexc.com/api/v3/klines');
|
|
@@ -38,11 +38,18 @@ declare function applyPrecisionOverrides(assetA: AssetRef, assetB: AssetRef, pre
|
|
|
38
38
|
assetA: AssetRef;
|
|
39
39
|
assetB: AssetRef;
|
|
40
40
|
};
|
|
41
|
+
declare function loadCachedFetchContext(bot: BotConfig, intervalSeconds: number, dir?: string): {
|
|
42
|
+
poolId: string;
|
|
43
|
+
assetA: AssetRef;
|
|
44
|
+
assetB: AssetRef;
|
|
45
|
+
source: string;
|
|
46
|
+
path: string;
|
|
47
|
+
} | null;
|
|
41
48
|
declare function fetchCandlesSequentially(fullPoolId: string, assetA: AssetRef, assetB: AssetRef, config: LpConfig, outPath: string): Promise<number[][]>;
|
|
42
49
|
declare function parseBotsConfig(raw: string, sourceLabel?: string): BotConfig[];
|
|
43
50
|
/**
|
|
44
51
|
* Pick a bot: by name if given, otherwise first active bot with startPrice: "pool".
|
|
45
52
|
*/
|
|
46
53
|
declare function selectBot(bots: BotConfig[], botName: string | null | undefined): BotConfig;
|
|
47
|
-
export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath };
|
|
54
|
+
export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath, loadCachedFetchContext };
|
|
48
55
|
//# sourceMappingURL=fetch_lp_data.d.ts.map
|
|
@@ -35,6 +35,8 @@ import { normalizePoolId, resolveAsset, findPoolByAssets } from '../utils/chain.
|
|
|
35
35
|
import { writeJsonAtomic } from '../utils/atomic_write.js';
|
|
36
36
|
import { buildFetchWindowsFromRange, formatWindowLine, runCachedWindows, } from './window_cache.js';
|
|
37
37
|
import { PATHS } from '../../modules/paths.js';
|
|
38
|
+
import { loadLpSeriesFromBasePath } from '../utils/data_discovery.js';
|
|
39
|
+
import { shardStemFromName } from '../utils/month_shards.js';
|
|
38
40
|
import * as bitsharesClient from '../../modules/bitshares_client.js';
|
|
39
41
|
import { getErrorMessage, resolveSeamMs } from '../../modules/utils/errors.js';
|
|
40
42
|
import { isSameBotName } from '../../modules/utils/sanitize_key.js';
|
|
@@ -185,13 +187,24 @@ function buildRequestKey(config, fullPoolId, assetA, assetB, timeRange, outPath)
|
|
|
185
187
|
chunkMonths,
|
|
186
188
|
};
|
|
187
189
|
}
|
|
188
|
-
function loadCachedFetchContext(bot, intervalSeconds) {
|
|
189
|
-
const dir = pairFolderPath(bot.assetA, bot.assetB);
|
|
190
|
+
function loadCachedFetchContext(bot, intervalSeconds, dir = pairFolderPath(bot.assetA, bot.assetB)) {
|
|
190
191
|
if (!storage.exists(dir))
|
|
191
192
|
return null;
|
|
192
193
|
const label = toIntervalLabel(intervalSeconds);
|
|
194
|
+
// Accept the whole-history export (`….json`) AND month shards
|
|
195
|
+
// (`….shard_YYYY-MM.json`). After the shard refactor `dexbot tv`/`dw` only
|
|
196
|
+
// write shards, so a shard-only pair had no cached context and fell back to
|
|
197
|
+
// a full blockchain connect. A shard's meta carries the same
|
|
198
|
+
// pool/asset/precision identity, and reading one shard is cheaper than
|
|
199
|
+
// parsing a multi-MB whole-history export.
|
|
200
|
+
const matchesInterval = (name) => {
|
|
201
|
+
if (name.endsWith(`${label}.json`))
|
|
202
|
+
return true;
|
|
203
|
+
const stem = shardStemFromName(name);
|
|
204
|
+
return stem ? stem.endsWith(`_${label}`) : false;
|
|
205
|
+
};
|
|
193
206
|
const dataFiles = storage.readdir(dir)
|
|
194
|
-
.filter(
|
|
207
|
+
.filter(matchesInterval)
|
|
195
208
|
.sort();
|
|
196
209
|
for (const file of dataFiles) {
|
|
197
210
|
try {
|
|
@@ -307,7 +320,6 @@ async function fetchCandlesSequentially(fullPoolId, assetA, assetB, config, outP
|
|
|
307
320
|
assetA: requestKey.assetA,
|
|
308
321
|
assetB: requestKey.assetB,
|
|
309
322
|
intervalSeconds: requestKey.intervalSeconds,
|
|
310
|
-
chunkIndex: window.index,
|
|
311
323
|
timeRange: { gte: window.gte, lte: window.lte },
|
|
312
324
|
format: '[timestamp_ms, open, high, low, close, volume_A]',
|
|
313
325
|
}),
|
|
@@ -545,6 +557,12 @@ async function run() {
|
|
|
545
557
|
keyBySymbols: `${assetA.symbol}|${assetB.symbol}`,
|
|
546
558
|
keyByIds: `${assetA.id}|${assetB.id}`,
|
|
547
559
|
};
|
|
560
|
+
// Derive the whole-history export from the assembled shard cache rather
|
|
561
|
+
// than from this run's fetched range: a later narrow run must not overwrite
|
|
562
|
+
// a full-history file with a subset. Falls back to the fetched candles when
|
|
563
|
+
// the shards were withheld (e.g. a partial fetch).
|
|
564
|
+
const assembled = loadLpSeriesFromBasePath(outPath);
|
|
565
|
+
const exportCandles = assembled && assembled.candles.length > 0 ? assembled.candles : candles;
|
|
548
566
|
const output = {
|
|
549
567
|
meta: {
|
|
550
568
|
fetchedAt: new Date().toISOString(),
|
|
@@ -555,7 +573,7 @@ async function run() {
|
|
|
555
573
|
pair,
|
|
556
574
|
intervalSeconds: config.intervalSeconds,
|
|
557
575
|
lookbackHours: config.lookbackHours ?? 0,
|
|
558
|
-
candleCount:
|
|
576
|
+
candleCount: exportCandles.length,
|
|
559
577
|
priceUnit: `${assetB.symbol} per ${assetA.symbol}`,
|
|
560
578
|
// Candle format: [timestamp_ms, open, high, low, close, volume_in_assetA]
|
|
561
579
|
// Raw Kibana LP exchange documents are ordered and converted to true OHLC.
|
|
@@ -564,7 +582,7 @@ async function run() {
|
|
|
564
582
|
// Actual received used (operation_result_object), not min_to_receive.
|
|
565
583
|
format: '[timestamp_ms, open, high, low, close, volume_A]',
|
|
566
584
|
},
|
|
567
|
-
candles,
|
|
585
|
+
candles: exportCandles,
|
|
568
586
|
};
|
|
569
587
|
writeJsonAtomic(outPath, output);
|
|
570
588
|
const kb = (storage.stat(outPath).size / 1024).toFixed(1);
|
|
@@ -578,5 +596,5 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
|
|
|
578
596
|
process.exit(1);
|
|
579
597
|
});
|
|
580
598
|
}
|
|
581
|
-
export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath };
|
|
599
|
+
export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath, loadCachedFetchContext };
|
|
582
600
|
//# sourceMappingURL=fetch_lp_data.js.map
|
|
@@ -30,7 +30,8 @@
|
|
|
30
30
|
* - Volume is the feed publish count per bucket, NOT traded asset volume.
|
|
31
31
|
*/
|
|
32
32
|
import { fillCandleGaps } from '../candle_utils.js';
|
|
33
|
-
import { resolveRequestedFillRange } from '../core/kibana_candles.js';
|
|
33
|
+
import { resolveRequestedFillRange, hitSortKey, hitSequence } from '../core/kibana_candles.js';
|
|
34
|
+
import { parseChainTimeToMs } from '../interval_utils.js';
|
|
34
35
|
import { kibanaSearch, DEFAULT_CONFIG as BASE_CONFIG } from '../core/kibana_client.js';
|
|
35
36
|
import { path } from '../../modules/path_api.js';
|
|
36
37
|
import { PATHS } from '../../modules/paths.js';
|
|
@@ -110,31 +111,9 @@ function parseFeedTimestamp(source) {
|
|
|
110
111
|
const raw = source?.block_data?.block_time;
|
|
111
112
|
if (raw == null)
|
|
112
113
|
return null;
|
|
113
|
-
const
|
|
114
|
-
const tsMs = Date.parse(text.endsWith('Z') ? text : `${text}Z`);
|
|
114
|
+
const tsMs = parseChainTimeToMs(String(raw));
|
|
115
115
|
return Number.isFinite(tsMs) ? tsMs : null;
|
|
116
116
|
}
|
|
117
|
-
function hitSortKey(hit) {
|
|
118
|
-
const h = hit;
|
|
119
|
-
const sort = Array.isArray(h?.sort) ? h.sort : [];
|
|
120
|
-
return sort.map((v) => String(v)).join('|') || String(h?._id || '');
|
|
121
|
-
}
|
|
122
|
-
function hitSequence(source) {
|
|
123
|
-
const src = source;
|
|
124
|
-
const candidates = [
|
|
125
|
-
src?.operation_id_num,
|
|
126
|
-
src?.account_history?.operation_id,
|
|
127
|
-
src?.account_history?.sequence,
|
|
128
|
-
];
|
|
129
|
-
for (const value of candidates) {
|
|
130
|
-
if (typeof value === 'number' && Number.isFinite(value))
|
|
131
|
-
return value;
|
|
132
|
-
const m = String(value || '').match(/(\d+)$/);
|
|
133
|
-
if (m)
|
|
134
|
-
return Number(m[1]);
|
|
135
|
-
}
|
|
136
|
-
return Number.NaN;
|
|
137
|
-
}
|
|
138
117
|
/**
|
|
139
118
|
* Kibana hit → { tsMs, price, sequence, kibanaSortKey } in backing-per-MPA,
|
|
140
119
|
* or null when the hit carries no usable settlement price for the pair.
|
|
@@ -151,7 +130,7 @@ function hitToFeedPrice(hit, { mpaAsset, backingAsset }) {
|
|
|
151
130
|
return {
|
|
152
131
|
tsMs,
|
|
153
132
|
price: price,
|
|
154
|
-
sequence: hitSequence(source),
|
|
133
|
+
sequence: hitSequence(source, 'account_history.operation_id'),
|
|
155
134
|
kibanaSortKey: hitSortKey(hit),
|
|
156
135
|
};
|
|
157
136
|
}
|
|
@@ -562,7 +541,6 @@ async function fetchFeedCandlesSequentially(feedCtx, assetA, assetB, opts = {})
|
|
|
562
541
|
assetA: requestKey.assetA,
|
|
563
542
|
assetB: requestKey.assetB,
|
|
564
543
|
intervalSeconds,
|
|
565
|
-
chunkIndex: window.index,
|
|
566
544
|
timeRange: { gte: window.gte, lte: window.lte },
|
|
567
545
|
format: '[timestamp_ms, open, high, low, close, feed_publish_count]',
|
|
568
546
|
}),
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges } from '../utils/month_shards.js';
|
|
2
|
+
import type { QueriedRange } from '../utils/month_shards.js';
|
|
1
3
|
type Candle = number[];
|
|
2
4
|
interface FileCover {
|
|
3
5
|
gte: number | null;
|
|
@@ -75,68 +77,12 @@ interface RunCachedWindowsOptions {
|
|
|
75
77
|
declare const TAIL_REFRESH_HOURS = 48;
|
|
76
78
|
declare const TAIL_SETTLE_LAG_MS: number;
|
|
77
79
|
declare const GAP_SETTLE_HORIZON_MS: number;
|
|
78
|
-
declare const GAP_MERGE_TOLERANCE_BUCKETS = 2;
|
|
79
|
-
declare const MAX_SUBFETCH_RANGES = 8;
|
|
80
|
-
declare const MAX_SUBFETCH_SPAN_RATIO = 0.5;
|
|
81
80
|
declare const MAX_COVERAGE_SPANS = 64;
|
|
82
|
-
declare function shardKeyForTimestamp(tsMs: unknown): string;
|
|
83
|
-
declare function shardBoundsForKey(key: unknown): {
|
|
84
|
-
start: number;
|
|
85
|
-
end: number;
|
|
86
|
-
};
|
|
87
|
-
declare function shardKeysForRange(gteMs: unknown, lteMs: unknown): string[];
|
|
88
|
-
declare function shardPathFor(outPath: string, shardKey: string): string;
|
|
89
81
|
declare function readCacheChunk(chunkFile: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean): CacheChunk | null;
|
|
90
82
|
declare function loadBucketCache(outPath: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean, range?: {
|
|
91
83
|
gte: number;
|
|
92
84
|
lte: number;
|
|
93
85
|
} | null): BucketCache;
|
|
94
|
-
declare function cachedCandlesInRange(localCache: BucketCache, gteMs: number, lteMs: number): Candle[];
|
|
95
|
-
/** Span of time actually queried, plus when the query ran (null = unknown). */
|
|
96
|
-
type QueriedRange = {
|
|
97
|
-
gte: number;
|
|
98
|
-
lte: number;
|
|
99
|
-
at: number | null;
|
|
100
|
-
};
|
|
101
|
-
/** Parse a verification timestamp: epoch ms or ISO. Anything else is unknown. */
|
|
102
|
-
declare function timestampMs(raw: unknown): number | null;
|
|
103
|
-
declare function verifiedAt(range: unknown): number | null;
|
|
104
|
-
/**
|
|
105
|
-
* Normalize recorded spans: sort, then merge overlapping / bucket-adjacent
|
|
106
|
-
* ones — but ONLY when they assert the same verification time. Merging spans
|
|
107
|
-
* with different `at` would have to pick one verdict for the whole union, and
|
|
108
|
-
* either pick loses (min → the merged span vouches for buckets the older
|
|
109
|
-
* query never saw; max → buckets verified long ago look freshly verified).
|
|
110
|
-
* Keeping them separate preserves exact per-bucket freshness, which is the
|
|
111
|
-
* whole point of `at`. Spans therefore accumulate ~1 per tail run, each a few
|
|
112
|
-
* dozen bytes — negligible next to the candles in the same file, and the
|
|
113
|
-
* same-`at` case (bulk fetches, and every pre-`at` shard) still collapses to
|
|
114
|
-
* one span as before.
|
|
115
|
-
*/
|
|
116
|
-
declare function unionQueriedRanges(ranges: {
|
|
117
|
-
gte: number;
|
|
118
|
-
lte: number;
|
|
119
|
-
at?: number | null;
|
|
120
|
-
}[], bucketMs: number): QueriedRange[];
|
|
121
|
-
declare function rangesCoveredBy(have: {
|
|
122
|
-
gte: number;
|
|
123
|
-
lte: number;
|
|
124
|
-
}[], want: {
|
|
125
|
-
gte: number;
|
|
126
|
-
lte: number;
|
|
127
|
-
}[]): boolean;
|
|
128
|
-
/**
|
|
129
|
-
* Change detection for shard writes: `have` already accounts for every span in
|
|
130
|
-
* `want`, in extent AND in verification time. Without the `at` half, a run
|
|
131
|
-
* whose only news is "these buckets were re-verified 1h later" would skip the
|
|
132
|
-
* write and the next run would fall back to the wide fixed refresh window.
|
|
133
|
-
* An unknown `at` on either side is never treated as fresh.
|
|
134
|
-
*/
|
|
135
|
-
declare function coverageSatisfied(have: QueriedRange[], want: QueriedRange[]): boolean;
|
|
136
|
-
/** Every recorded span across the loaded files, newest verification included. */
|
|
137
|
-
declare function allQueriedRanges(fileCover: {
|
|
138
|
-
queried?: QueriedRange[] | null;
|
|
139
|
-
}[]): QueriedRange[];
|
|
140
86
|
/**
|
|
141
87
|
* How settled a window is, given the recorded coverage.
|
|
142
88
|
*
|
|
@@ -158,20 +104,6 @@ declare function settleCoverage(opts: {
|
|
|
158
104
|
hasTimestamps: boolean;
|
|
159
105
|
firstUnsettled: number | null;
|
|
160
106
|
};
|
|
161
|
-
/**
|
|
162
|
-
* Bound the persisted span list. Because spans merge only when they assert
|
|
163
|
-
* the same `at`, a shard that is re-verified on every run gains one span per
|
|
164
|
-
* run and would grow without limit (and `settleCoverage` scans the list once
|
|
165
|
-
* per tail window). Dropping the OLDEST spans never invents emptiness, but it
|
|
166
|
-
* is not free: coverage is consulted by BOTH `pruneImmutableGaps` (to skip a
|
|
167
|
-
* query) and `settleCoverage` (to decide a bucket is fresh). A dropped span
|
|
168
|
-
* makes its buckets look unsettled, and the tail refresh would otherwise start
|
|
169
|
-
* at the earliest of them. That is why the refresh is capped at
|
|
170
|
-
* TAIL_REFRESH_HOURS: the worst a dropped span can cost is the legacy 48h
|
|
171
|
-
* overlap, never weeks. The input is already sorted by extent, so the tail of
|
|
172
|
-
* the list is the newest.
|
|
173
|
-
*/
|
|
174
|
-
declare function compactCoverage(spans: QueriedRange[], maxSpans: number): QueriedRange[];
|
|
175
107
|
declare function buildFetchWindowsFromRange(timeRange: {
|
|
176
108
|
gte?: unknown;
|
|
177
109
|
lte?: unknown;
|
|
@@ -229,6 +161,6 @@ declare function formatWindowLine(unit: string, index: number, total: number, gt
|
|
|
229
161
|
* touch nothing on disk.
|
|
230
162
|
*/
|
|
231
163
|
declare function runCachedWindows(opts: RunCachedWindowsOptions): Promise<Candle[]>;
|
|
232
|
-
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS,
|
|
233
|
-
export type { QueriedRange };
|
|
164
|
+
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
|
|
165
|
+
export type { QueriedRange } from '../utils/month_shards.js';
|
|
234
166
|
//# sourceMappingURL=window_cache.d.ts.map
|