dexbot 1.6.12 → 1.6.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/CHANGELOG.md +33 -1
  2. package/analysis/README.md +9 -11
  3. package/analysis/ama_fitting/README.md +5 -3
  4. package/claw/README.md +1 -0
  5. package/claw/docs/AI_BOT_LIBRARY_API.md +2 -0
  6. package/claw/modules/claw_bridge.ts +0 -3
  7. package/claw/modules/claw_skill_md.ts +1 -0
  8. package/claw/package.json +3 -3
  9. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  10. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  11. package/claw/tsconfig.json +2 -2
  12. package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +1 -14
  13. package/dist/analysis/ama_fitting/fetch_lp_candles.js +51 -41
  14. package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -24
  15. package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +36 -18
  16. package/dist/analysis/ama_fitting/optimizer_high_resolution.js +18 -66
  17. package/dist/analysis/analyze_args.d.ts +12 -0
  18. package/dist/analysis/analyze_args.js +20 -0
  19. package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +1 -16
  20. package/dist/analysis/bot_fitting/backtest_ama_sweep.js +20 -101
  21. package/dist/analysis/bot_fitting/backtest_bot_fitting.js +26 -92
  22. package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
  23. package/dist/analysis/bot_fitting/shared_utils.d.ts +71 -3
  24. package/dist/analysis/bot_fitting/shared_utils.js +124 -8
  25. package/dist/analysis/chart_ui.d.ts +30 -1
  26. package/dist/analysis/chart_ui.js +37 -1
  27. package/dist/analysis/fills_cache.d.ts +3 -23
  28. package/dist/analysis/fills_cache.js +3 -84
  29. package/dist/analysis/fills_source.d.ts +31 -1
  30. package/dist/analysis/fills_source.js +69 -1
  31. package/dist/analysis/grid_correction_check.d.ts +1 -23
  32. package/dist/analysis/grid_correction_check.js +15 -58
  33. package/dist/analysis/math_utils.d.ts +85 -1
  34. package/dist/analysis/math_utils.js +116 -1
  35. package/dist/analysis/price_sources.d.ts +22 -1
  36. package/dist/analysis/price_sources.js +23 -0
  37. package/dist/analysis/resolve_source.js +31 -10
  38. package/dist/analysis/trade_profitability.d.ts +6 -0
  39. package/dist/analysis/trade_profitability.js +23 -97
  40. package/dist/analysis/tradingview/analyze_tradingview.d.ts +1 -17
  41. package/dist/analysis/tradingview/analyze_tradingview.js +1 -1
  42. package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +2 -15
  43. package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
  44. package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +22 -80
  45. package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +11 -37
  46. package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
  47. package/dist/analysis/worker_pool.d.ts +38 -0
  48. package/dist/analysis/worker_pool.js +67 -0
  49. package/dist/dexbot.js +67 -48
  50. package/dist/market_adapter/ama_signal_runner.js +21 -54
  51. package/dist/market_adapter/candle_utils.d.ts +7 -1
  52. package/dist/market_adapter/candle_utils.js +11 -1
  53. package/dist/market_adapter/cli_args.d.ts +22 -0
  54. package/dist/market_adapter/cli_args.js +30 -0
  55. package/dist/market_adapter/core/config_normalizers.js +11 -6
  56. package/dist/market_adapter/core/kibana_candles.d.ts +3 -1
  57. package/dist/market_adapter/core/kibana_candles.js +3 -2
  58. package/dist/market_adapter/core/market_adapter_service.d.ts +85 -85
  59. package/dist/market_adapter/core/market_adapter_service.js +93 -158
  60. package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
  61. package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
  62. package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
  63. package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
  64. package/dist/market_adapter/inputs/fetch_book_data.js +0 -1
  65. package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +63 -107
  66. package/dist/market_adapter/inputs/fetch_lp_data.d.ts +8 -1
  67. package/dist/market_adapter/inputs/fetch_lp_data.js +25 -7
  68. package/dist/market_adapter/inputs/kibana_feed_source.js +4 -26
  69. package/dist/market_adapter/inputs/window_cache.d.ts +4 -72
  70. package/dist/market_adapter/inputs/window_cache.js +52 -177
  71. package/dist/market_adapter/interval_utils.d.ts +11 -1
  72. package/dist/market_adapter/interval_utils.js +16 -1
  73. package/dist/market_adapter/lp_chart_runner.d.ts +3 -0
  74. package/dist/market_adapter/lp_chart_runner.js +37 -19
  75. package/dist/market_adapter/market_adapter.js +27 -72
  76. package/dist/market_adapter/utils/data_discovery.d.ts +45 -1
  77. package/dist/market_adapter/utils/data_discovery.js +220 -27
  78. package/dist/market_adapter/utils/month_shards.d.ts +84 -0
  79. package/dist/market_adapter/utils/month_shards.js +237 -0
  80. package/dist/modules/account_bots.d.ts +1 -0
  81. package/dist/modules/account_bots.js +102 -121
  82. package/dist/modules/account_orders.d.ts +13 -0
  83. package/dist/modules/account_orders.js +38 -44
  84. package/dist/modules/bitshares-native/chain_client.d.ts +1 -1
  85. package/dist/modules/bitshares-native/chain_client.js +14 -13
  86. package/dist/modules/bitshares-native/resolvers.js +22 -25
  87. package/dist/modules/bitshares-native/serial/types.js +22 -50
  88. package/dist/modules/bitshares-native/signing_client.d.ts +1 -1
  89. package/dist/modules/bot_defaults.d.ts +2 -1
  90. package/dist/modules/bot_settings.d.ts +1 -1
  91. package/dist/modules/broadcast_failure.d.ts +2 -1
  92. package/dist/modules/chain_orders.d.ts +1 -1
  93. package/dist/modules/chain_orders.js +51 -96
  94. package/dist/modules/cli_start_onboarding.d.ts +2 -1
  95. package/dist/modules/constants.d.ts +7 -1
  96. package/dist/modules/constants.js +20 -1
  97. package/dist/modules/cr_planner.d.ts +1 -1
  98. package/dist/modules/cr_planner.js +1 -5
  99. package/dist/modules/credit_pricing.d.ts +2 -1
  100. package/dist/modules/credit_pricing.js +36 -40
  101. package/dist/modules/credit_runtime.d.ts +7 -0
  102. package/dist/modules/credit_runtime.js +45 -73
  103. package/dist/modules/daemon_node_health.d.ts +4 -3
  104. package/dist/modules/dexbot_class.d.ts +28 -17
  105. package/dist/modules/dexbot_class.js +105 -58
  106. package/dist/modules/dexbot_cow_runtime.d.ts +0 -7
  107. package/dist/modules/dexbot_cow_runtime.js +54 -28
  108. package/dist/modules/dexbot_credential_client.d.ts +0 -7
  109. package/dist/modules/dexbot_credential_client.js +1 -2
  110. package/dist/modules/dexbot_fill_runtime.d.ts +1 -2
  111. package/dist/modules/dexbot_fill_runtime.js +5 -6
  112. package/dist/modules/dexbot_maintenance_runtime.d.ts +20 -3
  113. package/dist/modules/dexbot_maintenance_runtime.js +84 -69
  114. package/dist/modules/dexbot_startup_runtime.d.ts +0 -1
  115. package/dist/modules/dexbot_startup_runtime.js +13 -14
  116. package/dist/modules/general_settings.d.ts +1 -1
  117. package/dist/modules/graceful_shutdown.js +17 -11
  118. package/dist/modules/key_store.js +0 -1
  119. package/dist/modules/lazy_shim.d.ts +11 -0
  120. package/dist/modules/lazy_shim.js +16 -0
  121. package/dist/modules/node_connect_policy.d.ts +3 -2
  122. package/dist/modules/node_failure_ledger.d.ts +6 -5
  123. package/dist/modules/order/format.d.ts +12 -3
  124. package/dist/modules/order/format.js +15 -3
  125. package/dist/modules/order/genesis_policy.d.ts +3 -3
  126. package/dist/modules/order/genesis_policy.js +5 -2
  127. package/dist/modules/order/grid.d.ts +4 -2
  128. package/dist/modules/order/grid.js +91 -35
  129. package/dist/modules/order/index.d.ts +1 -0
  130. package/dist/modules/order/logger.d.ts +1 -8
  131. package/dist/modules/order/logger.js +0 -9
  132. package/dist/modules/order/manager.d.ts +9 -13
  133. package/dist/modules/order/manager.js +33 -50
  134. package/dist/modules/order/utils/order.js +39 -35
  135. package/dist/modules/order/utils/system.d.ts +9 -29
  136. package/dist/modules/order/utils/system.js +17 -2
  137. package/dist/modules/order/utils/validate.js +2 -2
  138. package/dist/modules/order/utils/withPoolRef.d.ts +2 -1
  139. package/dist/modules/paths.js +32 -34
  140. package/dist/modules/runtime_settings.d.ts +2 -4
  141. package/dist/modules/settings_merge.d.ts +2 -2
  142. package/dist/modules/socket_json_client.d.ts +2 -1
  143. package/dist/modules/types.d.ts +42 -27
  144. package/dist/modules/utils/errors.js +1 -3
  145. package/dist/modules/version_notice.d.ts +11 -10
  146. package/dist/pm2.d.ts +0 -1
  147. package/dist/pm2.js +56 -35
  148. package/dist/scripts/analyze-git.js +2 -15
  149. package/dist/scripts/chart_command.js +2 -2
  150. package/dist/scripts/check_any.d.ts +4 -0
  151. package/dist/scripts/check_any.js +53 -21
  152. package/dist/scripts/diagnose-pool-history.js +1 -6
  153. package/dist/scripts/run-tests.js +0 -1
  154. package/dist/scripts/sync-version.js +0 -10
  155. package/docs/BITSHARES_ONBOARDING.md +8 -12
  156. package/docs/COPY_ON_WRITE_MASTER_PLAN.md +1 -1
  157. package/docs/DEXBOT_COMPARISON.md +6 -6
  158. package/docs/EVOLUTION.md +9 -8
  159. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  160. package/docs/README.md +1 -1
  161. package/docs/WORKFLOW.md +1 -0
  162. package/docs/architecture.md +19 -10
  163. package/market_adapter/README.md +6 -4
  164. package/package.json +6 -9
  165. package/scripts/README.md +43 -38
  166. package/tests/README.md +25 -10
  167. package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
  168. package/dist/scripts/diagnose-kibana-candles.js +0 -82
  169. package/scripts/git-viewer.sh +0 -489
  170. package/scripts/postinstall.js +0 -16
@@ -21,7 +21,7 @@ const { getAmaWarmupBars } = require('../core/strategies/ama');
21
21
  const { getErrorMessage } = require('../../modules/utils/errors');
22
22
  const { isSameBotName } = require('../../modules/utils/sanitize_key');
23
23
  const { candleFileForBot } = require('../../analysis/bot_key_utils');
24
- const { DEFAULTS: MARKET_ADAPTER_DEFAULTS, resolveAmaForBot, resolveBotCfg, } = require('../market_adapter/market_adapter');
24
+ const { DEFAULTS: MARKET_ADAPTER_DEFAULTS, resolveAmaForBot, resolveBotCfg, } = require('../market_adapter');
25
25
  const DEFAULT_INTERVAL = '1h';
26
26
  const DEFAULT_LIMIT = 1000;
27
27
  const DEFAULT_BASE = 'XRP';
@@ -237,6 +237,61 @@ function extractMarketsFromList(list, mapper) {
237
237
  const rows = asArray(list);
238
238
  return rows.map(mapper).filter((row) => Boolean(row && row.id && row.base && row.quote));
239
239
  }
240
+ // Interval → venue-specific unit tables. Most CEX APIs share one of a few
241
+ // shapes; keep one table per shape and compose where a venue overrides a
242
+ // single unit (Kraken day/week, MEXC's case-sensitive `1W`).
243
+ const INTERVAL_MINUTES = {
244
+ '1m': '1',
245
+ '5m': '5',
246
+ '15m': '15',
247
+ '30m': '30',
248
+ '1h': '60',
249
+ '4h': '240',
250
+ '6h': '360',
251
+ '12h': '720',
252
+ '1d': 'D',
253
+ '1w': 'W',
254
+ };
255
+ const INTERVAL_KRAKEN = {
256
+ ...INTERVAL_MINUTES,
257
+ '1d': '1440',
258
+ '1w': '10080',
259
+ };
260
+ const INTERVAL_DURATION = {
261
+ '1m': '1min',
262
+ '5m': '5min',
263
+ '15m': '15min',
264
+ '30m': '30min',
265
+ '1h': '1hour',
266
+ '4h': '4hour',
267
+ '6h': '6hour',
268
+ '12h': '12hour',
269
+ '1d': '1day',
270
+ '1w': '1week',
271
+ };
272
+ const INTERVAL_HTX = {
273
+ ...INTERVAL_DURATION,
274
+ '1h': '60min',
275
+ };
276
+ const INTERVAL_BITGET = {
277
+ '1m': '1m',
278
+ '5m': '5m',
279
+ '15m': '15m',
280
+ '30m': '30m',
281
+ '1h': '1h',
282
+ '4h': '4h',
283
+ '6h': '6h',
284
+ '12h': '12h',
285
+ '1d': '1d',
286
+ '1w': '1w',
287
+ };
288
+ const INTERVAL_MEXC = {
289
+ ...INTERVAL_BITGET,
290
+ '1h': '60m',
291
+ // MEXC interval enums are case-sensitive
292
+ '1w': '1W',
293
+ };
294
+ const intervalFormatter = (map) => (interval) => map[lower(interval)] || interval;
240
295
  const EXCHANGES = {
241
296
  binance: {
242
297
  name: 'Binance',
@@ -261,21 +316,7 @@ const EXCHANGES = {
261
316
  },
262
317
  bybit: {
263
318
  name: 'Bybit',
264
- formatInterval: (interval) => {
265
- const map = {
266
- '1m': '1',
267
- '5m': '5',
268
- '15m': '15',
269
- '30m': '30',
270
- '1h': '60',
271
- '4h': '240',
272
- '6h': '360',
273
- '12h': '720',
274
- '1d': 'D',
275
- '1w': 'W',
276
- };
277
- return map[lower(interval)] || interval;
278
- },
319
+ formatInterval: intervalFormatter(INTERVAL_MINUTES),
279
320
  marketsUrl: 'https://api.bybit.com/v5/market/instruments-info?category=spot&limit=1000',
280
321
  candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
281
322
  const url = new URL('https://api.bybit.com/v5/market/kline');
@@ -318,21 +359,7 @@ const EXCHANGES = {
318
359
  },
319
360
  bitget: {
320
361
  name: 'Bitget',
321
- formatInterval: (interval) => {
322
- const map = {
323
- '1m': '1m',
324
- '5m': '5m',
325
- '15m': '15m',
326
- '30m': '30m',
327
- '1h': '1h',
328
- '4h': '4h',
329
- '6h': '6h',
330
- '12h': '12h',
331
- '1d': '1d',
332
- '1w': '1w',
333
- };
334
- return map[lower(interval)] || interval;
335
- },
362
+ formatInterval: intervalFormatter(INTERVAL_BITGET),
336
363
  marketsUrl: 'https://api.bitget.com/api/v2/spot/public/symbols',
337
364
  candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
338
365
  const url = new URL('https://api.bitget.com/api/v2/spot/market/candles');
@@ -353,21 +380,7 @@ const EXCHANGES = {
353
380
  },
354
381
  kucoin: {
355
382
  name: 'KuCoin',
356
- formatInterval: (interval) => {
357
- const map = {
358
- '1m': '1min',
359
- '5m': '5min',
360
- '15m': '15min',
361
- '30m': '30min',
362
- '1h': '1hour',
363
- '4h': '4hour',
364
- '6h': '6hour',
365
- '12h': '12hour',
366
- '1d': '1day',
367
- '1w': '1week',
368
- };
369
- return map[lower(interval)] || interval;
370
- },
383
+ formatInterval: intervalFormatter(INTERVAL_DURATION),
371
384
  marketsUrl: 'https://api.kucoin.com/api/v2/symbols',
372
385
  candlesUrl: ({ id, interval, sinceMs, untilMs }) => {
373
386
  const url = new URL('https://api.kucoin.com/api/v1/market/candles');
@@ -387,21 +400,7 @@ const EXCHANGES = {
387
400
  },
388
401
  htx: {
389
402
  name: 'HTX',
390
- formatInterval: (interval) => {
391
- const map = {
392
- '1m': '1min',
393
- '5m': '5min',
394
- '15m': '15min',
395
- '30m': '30min',
396
- '1h': '60min',
397
- '4h': '4hour',
398
- '6h': '6hour',
399
- '12h': '12hour',
400
- '1d': '1day',
401
- '1w': '1week',
402
- };
403
- return map[lower(interval)] || interval;
404
- },
403
+ formatInterval: intervalFormatter(INTERVAL_HTX),
405
404
  marketsUrl: 'https://api.htx.com/v1/common/symbols',
406
405
  candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
407
406
  // Official Huobi/HTX spot historical kline endpoint is /market/history/kline
@@ -433,21 +432,7 @@ const EXCHANGES = {
433
432
  },
434
433
  kraken: {
435
434
  name: 'Kraken',
436
- formatInterval: (interval) => {
437
- const map = {
438
- '1m': '1',
439
- '5m': '5',
440
- '15m': '15',
441
- '30m': '30',
442
- '1h': '60',
443
- '4h': '240',
444
- '6h': '360',
445
- '12h': '720',
446
- '1d': '1440',
447
- '1w': '10080',
448
- };
449
- return map[lower(interval)] || interval;
450
- },
435
+ formatInterval: intervalFormatter(INTERVAL_KRAKEN),
451
436
  marketsUrl: 'https://api.kraken.com/0/public/AssetPairs',
452
437
  candlesUrl: ({ id, interval, intervalSeconds, limit, sinceMs }) => {
453
438
  const url = new URL('https://api.kraken.com/0/public/OHLC');
@@ -481,21 +466,7 @@ const EXCHANGES = {
481
466
  name: 'OKX',
482
467
  // /api/v5/market/candles caps limit at CEX_PAGE_LIMIT_CAPS.okx per request
483
468
  maxLimit: MARKET_ADAPTER.CEX_PAGE_LIMIT_CAPS.okx,
484
- formatInterval: (interval) => {
485
- const map = {
486
- '1m': '1',
487
- '5m': '5',
488
- '15m': '15',
489
- '30m': '30',
490
- '1h': '60',
491
- '4h': '240',
492
- '6h': '360',
493
- '12h': '720',
494
- '1d': 'D',
495
- '1w': 'W',
496
- };
497
- return map[lower(interval)] || interval;
498
- },
469
+ formatInterval: intervalFormatter(INTERVAL_MINUTES),
499
470
  marketsUrl: 'https://www.okx.com/api/v5/public/instruments?instType=SPOT',
500
471
  candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
501
472
  const url = new URL('https://www.okx.com/api/v5/market/candles');
@@ -518,22 +489,7 @@ const EXCHANGES = {
518
489
  name: 'MEXC',
519
490
  // /api/v3/klines caps limit at CEX_PAGE_LIMIT_CAPS.mexc per request
520
491
  maxLimit: MARKET_ADAPTER.CEX_PAGE_LIMIT_CAPS.mexc,
521
- formatInterval: (interval) => {
522
- const map = {
523
- '1m': '1m',
524
- '5m': '5m',
525
- '15m': '15m',
526
- '30m': '30m',
527
- '1h': '60m',
528
- '4h': '4h',
529
- '6h': '6h',
530
- '12h': '12h',
531
- '1d': '1d',
532
- // MEXC interval enums are case-sensitive
533
- '1w': '1W',
534
- };
535
- return map[lower(interval)] || interval;
536
- },
492
+ formatInterval: intervalFormatter(INTERVAL_MEXC),
537
493
  marketsUrl: 'https://api.mexc.com/api/v3/exchangeInfo',
538
494
  candlesUrl: ({ id, interval, limit, sinceMs, untilMs }) => {
539
495
  const url = new URL('https://api.mexc.com/api/v3/klines');
@@ -38,11 +38,18 @@ declare function applyPrecisionOverrides(assetA: AssetRef, assetB: AssetRef, pre
38
38
  assetA: AssetRef;
39
39
  assetB: AssetRef;
40
40
  };
41
+ declare function loadCachedFetchContext(bot: BotConfig, intervalSeconds: number, dir?: string): {
42
+ poolId: string;
43
+ assetA: AssetRef;
44
+ assetB: AssetRef;
45
+ source: string;
46
+ path: string;
47
+ } | null;
41
48
  declare function fetchCandlesSequentially(fullPoolId: string, assetA: AssetRef, assetB: AssetRef, config: LpConfig, outPath: string): Promise<number[][]>;
42
49
  declare function parseBotsConfig(raw: string, sourceLabel?: string): BotConfig[];
43
50
  /**
44
51
  * Pick a bot: by name if given, otherwise first active bot with startPrice: "pool".
45
52
  */
46
53
  declare function selectBot(bots: BotConfig[], botName: string | null | undefined): BotConfig;
47
- export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath };
54
+ export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath, loadCachedFetchContext };
48
55
  //# sourceMappingURL=fetch_lp_data.d.ts.map
@@ -35,6 +35,8 @@ import { normalizePoolId, resolveAsset, findPoolByAssets } from '../utils/chain.
35
35
  import { writeJsonAtomic } from '../utils/atomic_write.js';
36
36
  import { buildFetchWindowsFromRange, formatWindowLine, runCachedWindows, } from './window_cache.js';
37
37
  import { PATHS } from '../../modules/paths.js';
38
+ import { loadLpSeriesFromBasePath } from '../utils/data_discovery.js';
39
+ import { shardStemFromName } from '../utils/month_shards.js';
38
40
  import * as bitsharesClient from '../../modules/bitshares_client.js';
39
41
  import { getErrorMessage, resolveSeamMs } from '../../modules/utils/errors.js';
40
42
  import { isSameBotName } from '../../modules/utils/sanitize_key.js';
@@ -185,13 +187,24 @@ function buildRequestKey(config, fullPoolId, assetA, assetB, timeRange, outPath)
185
187
  chunkMonths,
186
188
  };
187
189
  }
188
- function loadCachedFetchContext(bot, intervalSeconds) {
189
- const dir = pairFolderPath(bot.assetA, bot.assetB);
190
+ function loadCachedFetchContext(bot, intervalSeconds, dir = pairFolderPath(bot.assetA, bot.assetB)) {
190
191
  if (!storage.exists(dir))
191
192
  return null;
192
193
  const label = toIntervalLabel(intervalSeconds);
194
+ // Accept the whole-history export (`….json`) AND month shards
195
+ // (`….shard_YYYY-MM.json`). After the shard refactor `dexbot tv`/`dw` only
196
+ // write shards, so a shard-only pair had no cached context and fell back to
197
+ // a full blockchain connect. A shard's meta carries the same
198
+ // pool/asset/precision identity, and reading one shard is cheaper than
199
+ // parsing a multi-MB whole-history export.
200
+ const matchesInterval = (name) => {
201
+ if (name.endsWith(`${label}.json`))
202
+ return true;
203
+ const stem = shardStemFromName(name);
204
+ return stem ? stem.endsWith(`_${label}`) : false;
205
+ };
193
206
  const dataFiles = storage.readdir(dir)
194
- .filter((name) => name.endsWith(`${label}.json`))
207
+ .filter(matchesInterval)
195
208
  .sort();
196
209
  for (const file of dataFiles) {
197
210
  try {
@@ -307,7 +320,6 @@ async function fetchCandlesSequentially(fullPoolId, assetA, assetB, config, outP
307
320
  assetA: requestKey.assetA,
308
321
  assetB: requestKey.assetB,
309
322
  intervalSeconds: requestKey.intervalSeconds,
310
- chunkIndex: window.index,
311
323
  timeRange: { gte: window.gte, lte: window.lte },
312
324
  format: '[timestamp_ms, open, high, low, close, volume_A]',
313
325
  }),
@@ -545,6 +557,12 @@ async function run() {
545
557
  keyBySymbols: `${assetA.symbol}|${assetB.symbol}`,
546
558
  keyByIds: `${assetA.id}|${assetB.id}`,
547
559
  };
560
+ // Derive the whole-history export from the assembled shard cache rather
561
+ // than from this run's fetched range: a later narrow run must not overwrite
562
+ // a full-history file with a subset. Falls back to the fetched candles when
563
+ // the shards were withheld (e.g. a partial fetch).
564
+ const assembled = loadLpSeriesFromBasePath(outPath);
565
+ const exportCandles = assembled && assembled.candles.length > 0 ? assembled.candles : candles;
548
566
  const output = {
549
567
  meta: {
550
568
  fetchedAt: new Date().toISOString(),
@@ -555,7 +573,7 @@ async function run() {
555
573
  pair,
556
574
  intervalSeconds: config.intervalSeconds,
557
575
  lookbackHours: config.lookbackHours ?? 0,
558
- candleCount: candles.length,
576
+ candleCount: exportCandles.length,
559
577
  priceUnit: `${assetB.symbol} per ${assetA.symbol}`,
560
578
  // Candle format: [timestamp_ms, open, high, low, close, volume_in_assetA]
561
579
  // Raw Kibana LP exchange documents are ordered and converted to true OHLC.
@@ -564,7 +582,7 @@ async function run() {
564
582
  // Actual received used (operation_result_object), not min_to_receive.
565
583
  format: '[timestamp_ms, open, high, low, close, volume_A]',
566
584
  },
567
- candles,
585
+ candles: exportCandles,
568
586
  };
569
587
  writeJsonAtomic(outPath, output);
570
588
  const kb = (storage.stat(outPath).size / 1024).toFixed(1);
@@ -578,5 +596,5 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
578
596
  process.exit(1);
579
597
  });
580
598
  }
581
- export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath };
599
+ export { applyPrecisionOverrides, parseBotsConfig, selectBot, fetchCandlesSequentially, outputPath, loadCachedFetchContext };
582
600
  //# sourceMappingURL=fetch_lp_data.js.map
@@ -30,7 +30,8 @@
30
30
  * - Volume is the feed publish count per bucket, NOT traded asset volume.
31
31
  */
32
32
  import { fillCandleGaps } from '../candle_utils.js';
33
- import { resolveRequestedFillRange } from '../core/kibana_candles.js';
33
+ import { resolveRequestedFillRange, hitSortKey, hitSequence } from '../core/kibana_candles.js';
34
+ import { parseChainTimeToMs } from '../interval_utils.js';
34
35
  import { kibanaSearch, DEFAULT_CONFIG as BASE_CONFIG } from '../core/kibana_client.js';
35
36
  import { path } from '../../modules/path_api.js';
36
37
  import { PATHS } from '../../modules/paths.js';
@@ -110,31 +111,9 @@ function parseFeedTimestamp(source) {
110
111
  const raw = source?.block_data?.block_time;
111
112
  if (raw == null)
112
113
  return null;
113
- const text = String(raw);
114
- const tsMs = Date.parse(text.endsWith('Z') ? text : `${text}Z`);
114
+ const tsMs = parseChainTimeToMs(String(raw));
115
115
  return Number.isFinite(tsMs) ? tsMs : null;
116
116
  }
117
- function hitSortKey(hit) {
118
- const h = hit;
119
- const sort = Array.isArray(h?.sort) ? h.sort : [];
120
- return sort.map((v) => String(v)).join('|') || String(h?._id || '');
121
- }
122
- function hitSequence(source) {
123
- const src = source;
124
- const candidates = [
125
- src?.operation_id_num,
126
- src?.account_history?.operation_id,
127
- src?.account_history?.sequence,
128
- ];
129
- for (const value of candidates) {
130
- if (typeof value === 'number' && Number.isFinite(value))
131
- return value;
132
- const m = String(value || '').match(/(\d+)$/);
133
- if (m)
134
- return Number(m[1]);
135
- }
136
- return Number.NaN;
137
- }
138
117
  /**
139
118
  * Kibana hit → { tsMs, price, sequence, kibanaSortKey } in backing-per-MPA,
140
119
  * or null when the hit carries no usable settlement price for the pair.
@@ -151,7 +130,7 @@ function hitToFeedPrice(hit, { mpaAsset, backingAsset }) {
151
130
  return {
152
131
  tsMs,
153
132
  price: price,
154
- sequence: hitSequence(source),
133
+ sequence: hitSequence(source, 'account_history.operation_id'),
155
134
  kibanaSortKey: hitSortKey(hit),
156
135
  };
157
136
  }
@@ -562,7 +541,6 @@ async function fetchFeedCandlesSequentially(feedCtx, assetA, assetB, opts = {})
562
541
  assetA: requestKey.assetA,
563
542
  assetB: requestKey.assetB,
564
543
  intervalSeconds,
565
- chunkIndex: window.index,
566
544
  timeRange: { gte: window.gte, lte: window.lte },
567
545
  format: '[timestamp_ms, open, high, low, close, feed_publish_count]',
568
546
  }),
@@ -1,3 +1,5 @@
1
+ import { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges } from '../utils/month_shards.js';
2
+ import type { QueriedRange } from '../utils/month_shards.js';
1
3
  type Candle = number[];
2
4
  interface FileCover {
3
5
  gte: number | null;
@@ -75,68 +77,12 @@ interface RunCachedWindowsOptions {
75
77
  declare const TAIL_REFRESH_HOURS = 48;
76
78
  declare const TAIL_SETTLE_LAG_MS: number;
77
79
  declare const GAP_SETTLE_HORIZON_MS: number;
78
- declare const GAP_MERGE_TOLERANCE_BUCKETS = 2;
79
- declare const MAX_SUBFETCH_RANGES = 8;
80
- declare const MAX_SUBFETCH_SPAN_RATIO = 0.5;
81
80
  declare const MAX_COVERAGE_SPANS = 64;
82
- declare function shardKeyForTimestamp(tsMs: unknown): string;
83
- declare function shardBoundsForKey(key: unknown): {
84
- start: number;
85
- end: number;
86
- };
87
- declare function shardKeysForRange(gteMs: unknown, lteMs: unknown): string[];
88
- declare function shardPathFor(outPath: string, shardKey: string): string;
89
81
  declare function readCacheChunk(chunkFile: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean): CacheChunk | null;
90
82
  declare function loadBucketCache(outPath: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean, range?: {
91
83
  gte: number;
92
84
  lte: number;
93
85
  } | null): BucketCache;
94
- declare function cachedCandlesInRange(localCache: BucketCache, gteMs: number, lteMs: number): Candle[];
95
- /** Span of time actually queried, plus when the query ran (null = unknown). */
96
- type QueriedRange = {
97
- gte: number;
98
- lte: number;
99
- at: number | null;
100
- };
101
- /** Parse a verification timestamp: epoch ms or ISO. Anything else is unknown. */
102
- declare function timestampMs(raw: unknown): number | null;
103
- declare function verifiedAt(range: unknown): number | null;
104
- /**
105
- * Normalize recorded spans: sort, then merge overlapping / bucket-adjacent
106
- * ones — but ONLY when they assert the same verification time. Merging spans
107
- * with different `at` would have to pick one verdict for the whole union, and
108
- * either pick loses (min → the merged span vouches for buckets the older
109
- * query never saw; max → buckets verified long ago look freshly verified).
110
- * Keeping them separate preserves exact per-bucket freshness, which is the
111
- * whole point of `at`. Spans therefore accumulate ~1 per tail run, each a few
112
- * dozen bytes — negligible next to the candles in the same file, and the
113
- * same-`at` case (bulk fetches, and every pre-`at` shard) still collapses to
114
- * one span as before.
115
- */
116
- declare function unionQueriedRanges(ranges: {
117
- gte: number;
118
- lte: number;
119
- at?: number | null;
120
- }[], bucketMs: number): QueriedRange[];
121
- declare function rangesCoveredBy(have: {
122
- gte: number;
123
- lte: number;
124
- }[], want: {
125
- gte: number;
126
- lte: number;
127
- }[]): boolean;
128
- /**
129
- * Change detection for shard writes: `have` already accounts for every span in
130
- * `want`, in extent AND in verification time. Without the `at` half, a run
131
- * whose only news is "these buckets were re-verified 1h later" would skip the
132
- * write and the next run would fall back to the wide fixed refresh window.
133
- * An unknown `at` on either side is never treated as fresh.
134
- */
135
- declare function coverageSatisfied(have: QueriedRange[], want: QueriedRange[]): boolean;
136
- /** Every recorded span across the loaded files, newest verification included. */
137
- declare function allQueriedRanges(fileCover: {
138
- queried?: QueriedRange[] | null;
139
- }[]): QueriedRange[];
140
86
  /**
141
87
  * How settled a window is, given the recorded coverage.
142
88
  *
@@ -158,20 +104,6 @@ declare function settleCoverage(opts: {
158
104
  hasTimestamps: boolean;
159
105
  firstUnsettled: number | null;
160
106
  };
161
- /**
162
- * Bound the persisted span list. Because spans merge only when they assert
163
- * the same `at`, a shard that is re-verified on every run gains one span per
164
- * run and would grow without limit (and `settleCoverage` scans the list once
165
- * per tail window). Dropping the OLDEST spans never invents emptiness, but it
166
- * is not free: coverage is consulted by BOTH `pruneImmutableGaps` (to skip a
167
- * query) and `settleCoverage` (to decide a bucket is fresh). A dropped span
168
- * makes its buckets look unsettled, and the tail refresh would otherwise start
169
- * at the earliest of them. That is why the refresh is capped at
170
- * TAIL_REFRESH_HOURS: the worst a dropped span can cost is the legacy 48h
171
- * overlap, never weeks. The input is already sorted by extent, so the tail of
172
- * the list is the newest.
173
- */
174
- declare function compactCoverage(spans: QueriedRange[], maxSpans: number): QueriedRange[];
175
107
  declare function buildFetchWindowsFromRange(timeRange: {
176
108
  gte?: unknown;
177
109
  lte?: unknown;
@@ -229,6 +161,6 @@ declare function formatWindowLine(unit: string, index: number, total: number, gt
229
161
  * touch nothing on disk.
230
162
  */
231
163
  declare function runCachedWindows(opts: RunCachedWindowsOptions): Promise<Candle[]>;
232
- export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, GAP_MERGE_TOLERANCE_BUCKETS, MAX_SUBFETCH_RANGES, MAX_SUBFETCH_SPAN_RATIO, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, cachedCandlesInRange, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
233
- export type { QueriedRange };
164
+ export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
165
+ export type { QueriedRange } from '../utils/month_shards.js';
234
166
  //# sourceMappingURL=window_cache.d.ts.map