dexbot 1.6.12 → 1.6.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +65 -1
  2. package/README.md +2 -2
  3. package/analysis/README.md +9 -11
  4. package/analysis/ama_fitting/README.md +5 -3
  5. package/claw/README.md +1 -0
  6. package/claw/docs/AI_BOT_LIBRARY_API.md +3 -1
  7. package/claw/docs/DEXBOT2_TUNING_CHEAT_SHEET.md +4 -4
  8. package/claw/modules/claw_bridge.ts +0 -3
  9. package/claw/modules/claw_catalog.ts +8 -4
  10. package/claw/modules/claw_skill_md.ts +1 -0
  11. package/claw/modules/dexbot_profiles.ts +59 -15
  12. package/claw/modules/types.ts +2 -1
  13. package/claw/package.json +3 -3
  14. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  15. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  16. package/claw/skills/margin-trading/SKILL.md +1 -1
  17. package/claw/skills/margin-trading/references/position-management.md +1 -1
  18. package/claw/tsconfig.json +2 -2
  19. package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +1 -14
  20. package/dist/analysis/ama_fitting/fetch_lp_candles.js +51 -41
  21. package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -24
  22. package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +36 -18
  23. package/dist/analysis/ama_fitting/optimizer_high_resolution.js +18 -66
  24. package/dist/analysis/analyze_args.d.ts +12 -0
  25. package/dist/analysis/analyze_args.js +20 -0
  26. package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +1 -16
  27. package/dist/analysis/bot_fitting/backtest_ama_sweep.js +20 -101
  28. package/dist/analysis/bot_fitting/backtest_bot_fitting.js +26 -92
  29. package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
  30. package/dist/analysis/bot_fitting/shared_utils.d.ts +71 -3
  31. package/dist/analysis/bot_fitting/shared_utils.js +124 -8
  32. package/dist/analysis/chart_ui.d.ts +30 -1
  33. package/dist/analysis/chart_ui.js +37 -1
  34. package/dist/analysis/fills_cache.d.ts +3 -23
  35. package/dist/analysis/fills_cache.js +3 -84
  36. package/dist/analysis/fills_source.d.ts +31 -1
  37. package/dist/analysis/fills_source.js +69 -1
  38. package/dist/analysis/grid_correction_check.d.ts +1 -23
  39. package/dist/analysis/grid_correction_check.js +15 -58
  40. package/dist/analysis/math_utils.d.ts +85 -1
  41. package/dist/analysis/math_utils.js +116 -1
  42. package/dist/analysis/price_sources.d.ts +22 -1
  43. package/dist/analysis/price_sources.js +23 -0
  44. package/dist/analysis/resolve_source.js +31 -10
  45. package/dist/analysis/trade_profitability.d.ts +6 -0
  46. package/dist/analysis/trade_profitability.js +23 -97
  47. package/dist/analysis/tradingview/analyze_tradingview.d.ts +1 -17
  48. package/dist/analysis/tradingview/analyze_tradingview.js +1 -1
  49. package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +27 -40
  50. package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
  51. package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +22 -80
  52. package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +11 -37
  53. package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
  54. package/dist/analysis/worker_pool.d.ts +38 -0
  55. package/dist/analysis/worker_pool.js +67 -0
  56. package/dist/dexbot.js +67 -48
  57. package/dist/market_adapter/ama_signal_runner.js +21 -54
  58. package/dist/market_adapter/candle_utils.d.ts +7 -1
  59. package/dist/market_adapter/candle_utils.js +11 -1
  60. package/dist/market_adapter/cli_args.d.ts +22 -0
  61. package/dist/market_adapter/cli_args.js +30 -0
  62. package/dist/market_adapter/core/config_normalizers.js +11 -6
  63. package/dist/market_adapter/core/kibana_candles.d.ts +3 -1
  64. package/dist/market_adapter/core/kibana_candles.js +3 -2
  65. package/dist/market_adapter/core/market_adapter_service.d.ts +85 -85
  66. package/dist/market_adapter/core/market_adapter_service.js +93 -158
  67. package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
  68. package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
  69. package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
  70. package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
  71. package/dist/market_adapter/inputs/fetch_book_data.js +0 -1
  72. package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +63 -107
  73. package/dist/market_adapter/inputs/fetch_lp_data.d.ts +8 -1
  74. package/dist/market_adapter/inputs/fetch_lp_data.js +25 -7
  75. package/dist/market_adapter/inputs/kibana_feed_source.js +4 -26
  76. package/dist/market_adapter/inputs/window_cache.d.ts +4 -72
  77. package/dist/market_adapter/inputs/window_cache.js +52 -177
  78. package/dist/market_adapter/interval_utils.d.ts +11 -1
  79. package/dist/market_adapter/interval_utils.js +16 -1
  80. package/dist/market_adapter/lp_chart_runner.d.ts +3 -0
  81. package/dist/market_adapter/lp_chart_runner.js +37 -19
  82. package/dist/market_adapter/market_adapter.js +27 -72
  83. package/dist/market_adapter/utils/data_discovery.d.ts +45 -1
  84. package/dist/market_adapter/utils/data_discovery.js +220 -27
  85. package/dist/market_adapter/utils/month_shards.d.ts +84 -0
  86. package/dist/market_adapter/utils/month_shards.js +237 -0
  87. package/dist/modules/account_bots.d.ts +8 -6
  88. package/dist/modules/account_bots.js +112 -131
  89. package/dist/modules/account_orders.d.ts +49 -4
  90. package/dist/modules/account_orders.js +114 -49
  91. package/dist/modules/bitshares-native/chain_client.d.ts +1 -1
  92. package/dist/modules/bitshares-native/chain_client.js +14 -13
  93. package/dist/modules/bitshares-native/crypto/ecc_core.d.ts +1 -2
  94. package/dist/modules/bitshares-native/crypto/ecc_core.js +1 -1
  95. package/dist/modules/bitshares-native/resolvers.js +22 -25
  96. package/dist/modules/bitshares-native/serial/chain_constants.d.ts +1 -2
  97. package/dist/modules/bitshares-native/serial/chain_constants.js +1 -2
  98. package/dist/modules/bitshares-native/serial/types.js +22 -50
  99. package/dist/modules/bitshares-native/signing_client.d.ts +1 -1
  100. package/dist/modules/bot_defaults.d.ts +18 -3
  101. package/dist/modules/bot_defaults.js +34 -8
  102. package/dist/modules/bot_settings.d.ts +5 -1
  103. package/dist/modules/bot_settings.js +55 -21
  104. package/dist/modules/broadcast_failure.d.ts +2 -1
  105. package/dist/modules/chain_orders.d.ts +1 -1
  106. package/dist/modules/chain_orders.js +51 -96
  107. package/dist/modules/cli_start_onboarding.d.ts +2 -1
  108. package/dist/modules/config.js +6 -1
  109. package/dist/modules/constants.d.ts +14 -6
  110. package/dist/modules/constants.js +32 -6
  111. package/dist/modules/cr_planner.d.ts +1 -1
  112. package/dist/modules/cr_planner.js +1 -5
  113. package/dist/modules/credit_pricing.d.ts +2 -1
  114. package/dist/modules/credit_pricing.js +36 -40
  115. package/dist/modules/credit_runtime.d.ts +7 -0
  116. package/dist/modules/credit_runtime.js +45 -73
  117. package/dist/modules/daemon_node_health.d.ts +4 -3
  118. package/dist/modules/dexbot_class.d.ts +33 -17
  119. package/dist/modules/dexbot_class.js +113 -61
  120. package/dist/modules/dexbot_cow_runtime.d.ts +62 -181
  121. package/dist/modules/dexbot_cow_runtime.js +380 -69
  122. package/dist/modules/dexbot_credential_client.d.ts +1 -23
  123. package/dist/modules/dexbot_credential_client.js +2 -3
  124. package/dist/modules/dexbot_fill_runtime.d.ts +2 -3
  125. package/dist/modules/dexbot_fill_runtime.js +17 -7
  126. package/dist/modules/dexbot_maintenance_runtime.d.ts +36 -81
  127. package/dist/modules/dexbot_maintenance_runtime.js +137 -133
  128. package/dist/modules/dexbot_startup_runtime.d.ts +0 -1
  129. package/dist/modules/dexbot_startup_runtime.js +25 -14
  130. package/dist/modules/dexbot_state_recovery.js +16 -0
  131. package/dist/modules/general_settings.d.ts +1 -1
  132. package/dist/modules/graceful_shutdown.js +17 -11
  133. package/dist/modules/key_store.d.ts +0 -5
  134. package/dist/modules/key_store.js +1 -2
  135. package/dist/modules/lazy_shim.d.ts +11 -0
  136. package/dist/modules/lazy_shim.js +16 -0
  137. package/dist/modules/node_connect_policy.d.ts +3 -2
  138. package/dist/modules/node_failure_ledger.d.ts +6 -5
  139. package/dist/modules/order/accounting.js +11 -11
  140. package/dist/modules/order/export.js +2 -2
  141. package/dist/modules/order/format.d.ts +26 -3
  142. package/dist/modules/order/format.js +33 -3
  143. package/dist/modules/order/genesis_policy.d.ts +4 -4
  144. package/dist/modules/order/genesis_policy.js +6 -3
  145. package/dist/modules/order/grid.d.ts +26 -12
  146. package/dist/modules/order/grid.js +468 -80
  147. package/dist/modules/order/grid_reconcile.d.ts +1 -1
  148. package/dist/modules/order/grid_reconcile.js +5 -5
  149. package/dist/modules/order/grid_reconcile_internal.js +48 -40
  150. package/dist/modules/order/index.d.ts +35 -8
  151. package/dist/modules/order/logger.d.ts +1 -8
  152. package/dist/modules/order/logger.js +2 -11
  153. package/dist/modules/order/manager.d.ts +18 -15
  154. package/dist/modules/order/manager.js +91 -74
  155. package/dist/modules/order/strategy.d.ts +2 -2
  156. package/dist/modules/order/strategy.js +51 -21
  157. package/dist/modules/order/sync_engine.js +4 -0
  158. package/dist/modules/order/utils/math.d.ts +3 -3
  159. package/dist/modules/order/utils/math.js +38 -27
  160. package/dist/modules/order/utils/order.d.ts +58 -58
  161. package/dist/modules/order/utils/order.js +158 -137
  162. package/dist/modules/order/utils/system.d.ts +202 -35
  163. package/dist/modules/order/utils/system.js +409 -31
  164. package/dist/modules/order/utils/validate.js +28 -6
  165. package/dist/modules/order/utils/withPoolRef.d.ts +2 -1
  166. package/dist/modules/paths.js +32 -34
  167. package/dist/modules/runtime_settings.d.ts +3 -5
  168. package/dist/modules/runtime_settings.js +3 -3
  169. package/dist/modules/settings_merge.d.ts +2 -2
  170. package/dist/modules/socket_json_client.d.ts +2 -1
  171. package/dist/modules/types.d.ts +89 -30
  172. package/dist/modules/utils/errors.js +1 -3
  173. package/dist/modules/version_notice.d.ts +11 -10
  174. package/dist/pm2.d.ts +0 -1
  175. package/dist/pm2.js +56 -35
  176. package/dist/scripts/analyze-git.js +2 -15
  177. package/dist/scripts/analyze-orders.d.ts +1 -1
  178. package/dist/scripts/analyze-orders.js +4 -4
  179. package/dist/scripts/chart_command.js +2 -2
  180. package/dist/scripts/check_any.d.ts +4 -0
  181. package/dist/scripts/check_any.js +53 -21
  182. package/dist/scripts/diagnose-pool-history.js +1 -6
  183. package/dist/scripts/divergence-calc.js +6 -6
  184. package/dist/scripts/run-tests.js +0 -1
  185. package/dist/scripts/sync-version.js +0 -10
  186. package/dist/scripts/validate_bots.d.ts +1 -1
  187. package/dist/scripts/validate_bots.js +3 -3
  188. package/dist/unlock.js +7 -2
  189. package/docs/BITSHARES_ONBOARDING.md +8 -12
  190. package/docs/COPY_ON_WRITE_MASTER_PLAN.md +2 -3
  191. package/docs/COW_INVARIANTS.md +6 -6
  192. package/docs/DEXBOT_COMPARISON.md +7 -7
  193. package/docs/EVOLUTION.md +10 -8
  194. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +8 -6
  195. package/docs/GRID_PRICE_INVARIANT.md +3 -3
  196. package/docs/GRID_RECALCULATION.md +4 -1
  197. package/docs/GRID_RECONCILE.md +4 -4
  198. package/docs/LIFECYCLE.md +11 -11
  199. package/docs/LOGGING.md +14 -0
  200. package/docs/MPA_CREDIT_USAGE.md +1 -1
  201. package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +12 -4
  202. package/docs/README.md +3 -3
  203. package/docs/WORKFLOW.md +1 -0
  204. package/docs/architecture.md +25 -16
  205. package/docs/developer_guide.md +7 -2
  206. package/market_adapter/README.md +6 -4
  207. package/package.json +6 -9
  208. package/scripts/README.md +43 -38
  209. package/tests/README.md +26 -11
  210. package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
  211. package/dist/scripts/diagnose-kibana-candles.js +0 -82
  212. package/scripts/git-viewer.sh +0 -489
  213. package/scripts/postinstall.js +0 -16
@@ -7,7 +7,7 @@ import { calculateAMA } from './core/strategies/ama.js';
7
7
  import { MARKET_ADAPTER } from '../modules/constants.js';
8
8
  import { generateHTML } from './lp_chart_core.js';
9
9
  import { loadStrategiesForLpChart } from './lp_chart_strategy_loader.js';
10
- import { findLatestLpData } from './utils/data_discovery.js';
10
+ import { findLatestLpData, loadLatestLpSeries, loadLpSeriesFromPath } from './utils/data_discovery.js';
11
11
  import { PATHS } from '../modules/paths.js';
12
12
  /**
13
13
  * LP CHART RUNNER — Shared LP chart orchestration
@@ -30,7 +30,7 @@ import { PATHS } from '../modules/paths.js';
30
30
  * - fetch/export of LP data (`market_adapter/inputs/fetch_lp_data.ts`)
31
31
  */
32
32
  const storage = getStorage();
33
- const { ensureDir, readJSON } = storage;
33
+ const { ensureDir } = storage;
34
34
  const ANALYSIS_CHARTS_DIR = PATHS.ANALYSIS.CHARTS_DIR;
35
35
  const AMA_PROFILES_FILE = PATHS.PROFILES.MARKET_PROFILES_JSON;
36
36
  const DEFAULT_COMPARISON_COLORS = ['#26a69a', '#fb8c00', '#5c9ee6', '#ef5350'];
@@ -100,7 +100,11 @@ function normalizeLpCandle(candle, index) {
100
100
  throw new Error(`Invalid candle row at index ${index}: unsupported format`);
101
101
  }
102
102
  function resolveLpDataFile(dataFile) {
103
- const resolved = dataFile ? path.resolve(dataFile) : findLatestLpData();
103
+ // Explicit path wins. Otherwise resolve through the assembling loader so a
104
+ // shard-only cache yields a representative source path instead of null.
105
+ const resolved = dataFile
106
+ ? path.resolve(dataFile)
107
+ : (loadLatestLpSeries()?.path ?? findLatestLpData());
104
108
  if (!resolved) {
105
109
  throw new Error('No LP data file found. Use --data <path> or run the fetch step first.');
106
110
  }
@@ -109,16 +113,9 @@ function resolveLpDataFile(dataFile) {
109
113
  }
110
114
  return resolved;
111
115
  }
112
- function loadLpDataFile(dataFile) {
113
- const resolved = resolveLpDataFile(dataFile);
114
- const raw = readJSON(resolved);
115
- const meta = (raw.meta ?? null);
116
- const candles = Array.isArray(raw?.candles) ? raw.candles : (Array.isArray(raw) ? raw : []);
117
- if (!Array.isArray(candles) || candles.length === 0) {
118
- throw new Error('No candles in data file.');
119
- }
116
+ function bundleFrom(dataFile, meta, candles) {
120
117
  return {
121
- dataFile: resolved,
118
+ dataFile,
122
119
  meta,
123
120
  candleArrays: candles.map((c, index) => normalizeLpCandle(c, index)),
124
121
  candleObjects: candles.map((c, index) => {
@@ -134,6 +131,24 @@ function loadLpDataFile(dataFile) {
134
131
  }),
135
132
  };
136
133
  }
134
+ function loadLpDataFile(dataFile) {
135
+ // Explicit path: read the file as before. Auto-discovery: assemble the
136
+ // complete series (merging month shards when they are the newest source),
137
+ // so a shard-only cache charts the whole history, not one month.
138
+ if (!dataFile) {
139
+ const series = loadLatestLpSeries();
140
+ if (!series || series.candles.length === 0) {
141
+ throw new Error('No LP data found. Use --data <path> or run the fetch step first.');
142
+ }
143
+ return bundleFrom(series.path, series.meta, series.candles);
144
+ }
145
+ const resolved = resolveLpDataFile(dataFile);
146
+ const series = loadLpSeriesFromPath(resolved);
147
+ if (!series || series.candles.length === 0) {
148
+ throw new Error('No candles in data file.');
149
+ }
150
+ return bundleFrom(series.path, series.meta, series.candles);
151
+ }
137
152
  function openInBrowser(filePath) {
138
153
  const url = `file://${filePath}`;
139
154
  const cmd = process.platform === 'darwin'
@@ -203,7 +218,7 @@ function writeChartHtml({ meta, candleArrays, amaResults, outFile }) {
203
218
  }
204
219
  function generateMarketLpChart(options = {}) {
205
220
  const logger = options.logger ?? console;
206
- const { dataFile, meta, candleArrays } = loadLpDataFile(options.dataFile);
221
+ const { dataFile, meta, candleArrays } = options.dataBundle ?? loadLpDataFile(options.dataFile);
207
222
  logger.log(`Reading: ${path.relative(process.cwd(), dataFile)}`);
208
223
  const poolLabel = meta?.pool ? `pool ${meta.pool}` : `${meta?.assetA?.symbol || '?'}\/${meta?.assetB?.symbol || '?'}`;
209
224
  logger.log(` ${candleArrays.length} candles · ${poolLabel}`);
@@ -241,7 +256,7 @@ function generateMarketLpChart(options = {}) {
241
256
  }
242
257
  function generateComparisonLpChart(options = {}) {
243
258
  const logger = options.logger ?? console;
244
- const { dataFile, meta, candleArrays, candleObjects } = loadLpDataFile(options.dataFile);
259
+ const { dataFile, meta, candleArrays, candleObjects } = options.dataBundle ?? loadLpDataFile(options.dataFile);
245
260
  const defaultStrategies = options.defaultStrategies ?? DEFAULT_COMPARISON_STRATEGIES;
246
261
  const strategies = loadStrategiesForLpChart({
247
262
  dataFile,
@@ -284,23 +299,26 @@ function generateComparisonLpChart(options = {}) {
284
299
  }
285
300
  function generateLpChartBundle(options = {}) {
286
301
  const logger = options.logger ?? console;
287
- const dataFile = resolveLpDataFile(options.dataFile);
288
- logger.log(`Generating LP charts from ${path.relative(process.cwd(), dataFile)}`);
302
+ // Resolve/assemble ONCE, then hand the same loaded bundle to both charts —
303
+ // auto-discovery over shards would otherwise re-read and re-merge the whole
304
+ // family for each sub-chart.
305
+ const bundle = options.dataBundle ?? loadLpDataFile(options.dataFile);
306
+ logger.log(`Generating LP charts from ${path.relative(process.cwd(), bundle.dataFile)}`);
289
307
  const marketChart = generateMarketLpChart({
290
- dataFile,
308
+ dataBundle: bundle,
291
309
  noOpen: options.noOpen,
292
310
  logger,
293
311
  profilesFile: options.profilesFile,
294
312
  });
295
313
  const comparisonChart = generateComparisonLpChart({
296
- dataFile,
314
+ dataBundle: bundle,
297
315
  noOpen: options.openComparison === true ? !!options.noOpen : true,
298
316
  logger,
299
317
  defaultStrategies: options.defaultStrategies ?? DEFAULT_COMPARISON_STRATEGIES,
300
318
  profilesFile: options.comparisonProfilesFile ?? null,
301
319
  });
302
320
  return {
303
- dataFile,
321
+ dataFile: bundle.dataFile,
304
322
  marketChart,
305
323
  comparisonChart,
306
324
  };
@@ -12,7 +12,7 @@ import { normalizeBotEntry } from '../modules/bot_settings.js';
12
12
  import * as kibanaSource from './inputs/kibana_source.js';
13
13
  import * as kibanaMarketSource from './core/kibana_market_candles.js';
14
14
  import { tradesToCandles, detectMissingCandleTimestamps, fillCandleGaps, detectStaleTail, pruneStaleTail, mergeCandles } from './candle_utils.js';
15
- import { toIntervalLabel, bucketStartMs, latestClosedBucketStartMs } from './interval_utils.js';
15
+ import { toIntervalLabel, bucketStartMs, latestClosedBucketStartMs, parseChainTimeToMs } from './interval_utils.js';
16
16
  import { loadMarketProfiles } from '../analysis/tradingview/tradingview_uplot_chart_generator.js';
17
17
  import { candleFileForBot as candleFilePathForLabel } from '../analysis/bot_key_utils.js';
18
18
  import { writeJsonAtomic } from './utils/atomic_write.js';
@@ -27,6 +27,7 @@ import { resetMarketAdapterWhitelistCache, isBotWhitelisted, isBotDynamicWeightW
27
27
  import { normalizeAssetSymbol, normalizeMarketSource, resolveBotContext, isExactPair, isSamePair, isExactPairIds, isSamePairIds, getBitsharesClient, setBitsharesClientForTests, } from './utils/chain.js';
28
28
  import { normalizeNativeMarketHistoryCandles, } from './utils/native_history.js';
29
29
  import { formatLogPercent, buildWeightSummary, buildDynamicWeightInputsLog, buildDynamicWeightTuningLog, buildAsymmetricBoundsLog, buildStartupDefaultsLog, } from './log_format.js';
30
+ import { walkCliArgs } from './cli_args.js';
30
31
  /**
31
32
  * PRICE ADAPTER — standalone or auto-launched by dexbot runtime.
32
33
  *
@@ -616,73 +617,33 @@ function parseArgs() {
616
617
  const provided = {
617
618
  deltaThresholdPercent: false,
618
619
  };
619
- for (let i = 0; i < args.length; i++) {
620
- const a = args[i];
621
- const v = args[i + 1];
622
- switch (a) {
623
- case '--once':
624
- cfg.once = true;
625
- break;
626
- case '--pollSeconds':
627
- cfg.pollSeconds = Number(v);
628
- i++;
629
- break;
630
- case '--deltaPercent':
620
+ walkCliArgs(args, {
621
+ onHelp: () => {
622
+ printHelp();
623
+ process.exit(0);
624
+ },
625
+ flag: {
626
+ '--once': () => { cfg.once = true; },
627
+ '--metricsJson': () => { cfg.metricsJson = true; },
628
+ '--quiet': () => { cfg.quiet = true; },
629
+ '--dryRun': () => { cfg.dryRun = true; },
630
+ '--whitelist-all': () => { cfg.whitelistAll = true; },
631
+ },
632
+ value: {
633
+ '--pollSeconds': (v) => { cfg.pollSeconds = Number(v); },
634
+ '--deltaPercent': (v) => {
631
635
  cfg.deltaThresholdPercent = Number(v);
632
636
  provided.deltaThresholdPercent = true;
633
- i++;
634
- break;
635
- case '--gridResetFactor':
636
- throw new Error('--gridResetFactor is no longer supported; use --deltaPercent <percent>');
637
- case '--bootstrapHours':
638
- cfg.bootstrapLookbackHours = Number(v);
639
- i++;
640
- break;
641
- case '--nativeBackfillHours':
642
- cfg.nativeBackfillHours = Number(v);
643
- i++;
644
- break;
645
- case '--maxStaleHours':
646
- cfg.maxStaleHours = Number(v);
647
- i++;
648
- break;
649
- case '--sourceRetries':
650
- cfg.sourceRetries = Number(v);
651
- i++;
652
- break;
653
- case '--retryDelayMs':
654
- cfg.retryDelayMs = Number(v);
655
- i++;
656
- break;
657
- case '--metricsJson':
658
- cfg.metricsJson = true;
659
- break;
660
- case '--quiet':
661
- cfg.quiet = true;
662
- break;
663
- case '--dryRun':
664
- cfg.dryRun = true;
665
- break;
666
- case '--whitelist-all':
667
- cfg.whitelistAll = true;
668
- break;
669
- case '--maxPages':
670
- cfg.maxPages = Number(v);
671
- i++;
672
- break;
673
- case '--pageLimit':
674
- cfg.pageLimit = Number(v);
675
- i++;
676
- break;
677
- case '--help':
678
- case '-h':
679
- printHelp();
680
- process.exit(0);
681
- break;
682
- default:
683
- throw new Error(`Unknown argument: ${a}`);
684
- }
685
- }
637
+ },
638
+ '--bootstrapHours': (v) => { cfg.bootstrapLookbackHours = Number(v); },
639
+ '--nativeBackfillHours': (v) => { cfg.nativeBackfillHours = Number(v); },
640
+ '--maxStaleHours': (v) => { cfg.maxStaleHours = Number(v); },
641
+ '--sourceRetries': (v) => { cfg.sourceRetries = Number(v); },
642
+ '--retryDelayMs': (v) => { cfg.retryDelayMs = Number(v); },
643
+ '--maxPages': (v) => { cfg.maxPages = Number(v); },
644
+ '--pageLimit': (v) => { cfg.pageLimit = Number(v); },
645
+ },
646
+ });
686
647
  const merged = applyRuntimeDefaultsFromGeneralSettings(cfg, provided);
687
648
  return validateConfig(merged);
688
649
  }
@@ -808,12 +769,6 @@ function loadJson(filePath, defaultValue) {
808
769
  function saveJson(filePath, data) {
809
770
  writeJsonAtomic(filePath, data);
810
771
  }
811
- function parseChainTimeToMs(timeStr) {
812
- if (!timeStr)
813
- return Number.NaN;
814
- const s = String(timeStr);
815
- return Date.parse(s.endsWith('Z') ? s : `${s}Z`);
816
- }
817
772
  function candleFileForBot(botKey, intervalSeconds = RUNTIME_DEFAULTS.intervalSeconds) {
818
773
  const label = intervalSeconds === RUNTIME_DEFAULTS.intervalSeconds
819
774
  ? RUNTIME_DEFAULTS.intervalLabel
@@ -1,6 +1,50 @@
1
1
  interface FindLatestLpDataOptions {
2
2
  dataDir?: string;
3
3
  }
4
+ /**
5
+ * Result of auto-discovering the newest LP series. `candles` is always the
6
+ * complete series, whether it came from one whole-history export
7
+ * (`assembled: false`) or was merged across month shards (`assembled: true`).
8
+ */
9
+ interface LpSeriesResult {
10
+ path: string;
11
+ meta: Record<string, unknown> | null;
12
+ candles: unknown[];
13
+ assembled: boolean;
14
+ }
15
+ /**
16
+ * Whole-history-export discovery only: returns the newest `lp_pool_*.json` that
17
+ * is a complete series, or null. Month shards and legacy `.chunk_*` files are
18
+ * excluded — a single shard is a slice, and (because the active shard is
19
+ * rewritten every run) by mtime it would otherwise always win. Use
20
+ * `loadLatestLpSeries` when a shard-only cache should still yield a series.
21
+ */
4
22
  declare function findLatestLpData(options?: FindLatestLpDataOptions): string | null;
5
- export { findLatestLpData };
23
+ /**
24
+ * Auto-discover and load the newest LP series.
25
+ *
26
+ * Month shards are the cache and a whole-history export is a derived snapshot
27
+ * of the same shard family (both writers, `fetch_lp_data` and
28
+ * `fetch_lp_candles`, regenerate the export from the assembled shards after
29
+ * fetching). The two therefore agree in extent, and the ordinary mtime race
30
+ * decides — an export produced by an external tool can still be newer and win.
31
+ * A shard-only directory is assembled; an export-only directory is read as-is.
32
+ */
33
+ declare function loadLatestLpSeries(options?: FindLatestLpDataOptions): LpSeriesResult | null;
34
+ /**
35
+ * Assemble the whole series for a base path directly from its month shards,
36
+ * regardless of any sibling whole-history export. Writers use this to derive
37
+ * the export from the cache, so the monolithic file can never become a
38
+ * narrower or staler snapshot than the shards beside it.
39
+ */
40
+ declare function loadLpSeriesFromBasePath(basePath: string): LpSeriesResult | null;
41
+ /**
42
+ * Load one explicit LP path. A month shard is transparently upgraded to its
43
+ * whole family: pointing `--data` at `.shard_2026-10.json` charts the full
44
+ * cached history, never a silent one-month slice. A whole-history export (or
45
+ * a lone shard whose siblings are missing) is read as-is.
46
+ */
47
+ declare function loadLpSeriesFromPath(filePath: string): LpSeriesResult | null;
48
+ export { findLatestLpData, loadLatestLpSeries, loadLpSeriesFromPath, loadLpSeriesFromBasePath };
49
+ export type { LpSeriesResult, FindLatestLpDataOptions };
6
50
  //# sourceMappingURL=data_discovery.d.ts.map
@@ -2,47 +2,240 @@
2
2
  import { path } from '../../modules/path_api.js';
3
3
  import { getStorage } from '../../modules/storage/index.js';
4
4
  import { PATHS } from '../../modules/paths.js';
5
+ import { mergeCandles, higherVolumeWins } from '../candle_utils.js';
6
+ import { shardKeyFromName, shardStemFromName } from './month_shards.js';
5
7
  const storage = getStorage();
6
- function findLatestLpData(options = {}) {
7
- const dataDir = options.dataDir ? path.resolve(options.dataDir) : PATHS.MARKET_ADAPTER.LP_DATA_DIR;
8
+ function isLpJsonName(name) {
9
+ return name.startsWith('lp_pool_') && name.endsWith('.json');
10
+ }
11
+ /**
12
+ * All shard files for one base path (`<base>.shard_YYYY-MM.json`), read from
13
+ * disk. Empty when the base is not a shard-addressed series.
14
+ */
15
+ function readShardFamily(basePath) {
16
+ const resolved = path.resolve(basePath);
17
+ const parsed = path.parse(resolved);
18
+ if (!storage.exists(parsed.dir))
19
+ return [];
8
20
  const out = [];
9
- if (!storage.exists(dataDir))
21
+ for (const name of storage.readdir(parsed.dir)) {
22
+ if (!shardKeyFromName(name, parsed.name, parsed.ext))
23
+ continue;
24
+ const full = path.join(parsed.dir, name);
25
+ try {
26
+ const info = storage.stat(full);
27
+ if (info.isFile())
28
+ out.push({ path: full, mtime: info.mtimeMs });
29
+ }
30
+ catch (_) { /* unreadable sibling: skip it, keep the rest */ }
31
+ }
32
+ return out.sort((a, b) => (a.path < b.path ? -1 : 1));
33
+ }
34
+ /**
35
+ * Merge an ordered shard family into one series. `sourcePath` is the file the
36
+ * caller should report (the newest shard, or the explicitly requested one).
37
+ */
38
+ function assembleLpShards(files, sourcePath) {
39
+ let merged = [];
40
+ let meta = null;
41
+ for (const file of files) {
42
+ const raw = storage.readJSON(file.path);
43
+ if (!meta)
44
+ meta = metaOf(raw);
45
+ merged = mergeCandles(merged, rawCandlesOf(raw), { onCollision: higherVolumeWins });
46
+ }
47
+ if (merged.length === 0)
10
48
  return null;
11
- const stack = [dataDir];
12
- while (stack.length > 0) {
13
- const dir = stack.pop();
14
- let names = [];
49
+ const firstTs = Number(merged[0][0]);
50
+ const lastTs = Number(merged[merged.length - 1][0]);
51
+ // A shard's meta describes one month; publish the assembled extent instead
52
+ // and drop per-shard bookkeeping that would misrepresent a whole series.
53
+ const mergedMeta = {
54
+ ...(meta || {}),
55
+ candleCount: merged.length,
56
+ firstTs: new Date(firstTs).toISOString(),
57
+ lastTs: new Date(lastTs).toISOString(),
58
+ timeRange: { gte: new Date(firstTs).toISOString(), lte: new Date(lastTs).toISOString() },
59
+ assembledFrom: 'month-shards',
60
+ };
61
+ delete mergedMeta.shard;
62
+ delete mergedMeta.queriedRanges;
63
+ return { path: sourcePath, meta: mergedMeta, candles: merged, assembled: true };
64
+ }
65
+ /**
66
+ * Scan the LP data dir for whole-history exports and month-shard families.
67
+ *
68
+ * The on-disk layout is fixed at two levels (`<dataDir>/<pair>/<file>`), so the
69
+ * scan is deliberately NOT recursive: it stats each direct child once and only
70
+ * descends one level into pair folders. Legacy run-relative `.chunk_*` files
71
+ * are ignored — nothing reads them and they are not a series.
72
+ */
73
+ function scanLpDataDir(dataDir) {
74
+ const whole = [];
75
+ const shardFamilies = new Map();
76
+ if (!storage.exists(dataDir))
77
+ return { whole, shardFamilies };
78
+ const consider = (full, stat) => {
79
+ const name = path.basename(full);
80
+ if (!isLpJsonName(name) || name.includes('.chunk_'))
81
+ return;
82
+ let info = stat;
83
+ if (!info) {
84
+ try {
85
+ info = storage.stat(full);
86
+ }
87
+ catch (_) {
88
+ return;
89
+ }
90
+ }
91
+ if (!info.isFile())
92
+ return;
93
+ const stem = shardStemFromName(name, path.extname(name));
94
+ if (!stem) {
95
+ whole.push({ path: full, mtime: info.mtimeMs });
96
+ return;
97
+ }
98
+ const key = path.join(path.dirname(full), stem);
99
+ let family = shardFamilies.get(key);
100
+ if (!family) {
101
+ family = { stem, files: [], maxMtime: 0 };
102
+ shardFamilies.set(key, family);
103
+ }
104
+ family.files.push({ path: full, mtime: info.mtimeMs });
105
+ if (info.mtimeMs > family.maxMtime)
106
+ family.maxMtime = info.mtimeMs;
107
+ };
108
+ for (const name of storage.readdir(dataDir)) {
109
+ const full = path.join(dataDir, name);
110
+ let stat = null;
15
111
  try {
16
- names = storage.readdir(dir);
112
+ stat = storage.stat(full);
17
113
  }
18
114
  catch (_) {
19
- // Directory vanished or is unreadable — skip it instead of
20
- // aborting the whole scan.
21
115
  continue;
22
116
  }
23
- for (const name of names) {
24
- const full = path.join(dir, name);
25
- let stat = null;
117
+ if (stat.isDirectory()) {
118
+ let names = [];
26
119
  try {
27
- stat = storage.stat(full);
120
+ names = storage.readdir(full);
28
121
  }
29
122
  catch (_) {
30
- // Dangling symlink or race-deleted entry — skip the entry.
31
- continue;
32
- }
33
- if (stat.isDirectory()) {
34
- stack.push(full);
35
123
  continue;
36
124
  }
37
- if (!stat.isFile() || !name.endsWith('.json'))
38
- continue;
39
- if (!name.startsWith('lp_pool_'))
40
- continue;
41
- out.push({ path: full, mtime: stat.mtimeMs });
125
+ for (const inner of names)
126
+ consider(path.join(full, inner));
127
+ }
128
+ else if (stat.isFile()) {
129
+ consider(full, stat);
42
130
  }
43
131
  }
44
- out.sort((a, b) => b.mtime - a.mtime);
45
- return out.length > 0 ? out[0].path : null;
132
+ return { whole, shardFamilies };
133
+ }
134
+ function newestOf(files) {
135
+ let best = null;
136
+ for (const f of files)
137
+ if (!best || f.mtime > best.mtime)
138
+ best = f;
139
+ return best;
140
+ }
141
+ function newestFamily(families) {
142
+ let best = null;
143
+ for (const fam of families.values())
144
+ if (!best || fam.maxMtime > best.maxMtime)
145
+ best = fam;
146
+ return best;
147
+ }
148
+ function rawCandlesOf(raw) {
149
+ if (Array.isArray(raw))
150
+ return raw;
151
+ const r = raw;
152
+ if (r && Array.isArray(r.candles))
153
+ return r.candles;
154
+ if (r && Array.isArray(r.data))
155
+ return r.data;
156
+ return [];
157
+ }
158
+ function metaOf(raw) {
159
+ if (Array.isArray(raw))
160
+ return null;
161
+ const meta = raw?.meta;
162
+ return meta && typeof meta === 'object' ? meta : null;
163
+ }
164
+ /**
165
+ * Whole-history-export discovery only: returns the newest `lp_pool_*.json` that
166
+ * is a complete series, or null. Month shards and legacy `.chunk_*` files are
167
+ * excluded — a single shard is a slice, and (because the active shard is
168
+ * rewritten every run) by mtime it would otherwise always win. Use
169
+ * `loadLatestLpSeries` when a shard-only cache should still yield a series.
170
+ */
171
+ function findLatestLpData(options = {}) {
172
+ const dataDir = options.dataDir ? path.resolve(options.dataDir) : PATHS.MARKET_ADAPTER.LP_DATA_DIR;
173
+ const { whole } = scanLpDataDir(dataDir);
174
+ return newestOf(whole)?.path ?? null;
175
+ }
176
+ /**
177
+ * Auto-discover and load the newest LP series.
178
+ *
179
+ * Month shards are the cache and a whole-history export is a derived snapshot
180
+ * of the same shard family (both writers, `fetch_lp_data` and
181
+ * `fetch_lp_candles`, regenerate the export from the assembled shards after
182
+ * fetching). The two therefore agree in extent, and the ordinary mtime race
183
+ * decides — an export produced by an external tool can still be newer and win.
184
+ * A shard-only directory is assembled; an export-only directory is read as-is.
185
+ */
186
+ function loadLatestLpSeries(options = {}) {
187
+ const dataDir = options.dataDir ? path.resolve(options.dataDir) : PATHS.MARKET_ADAPTER.LP_DATA_DIR;
188
+ const { whole, shardFamilies } = scanLpDataDir(dataDir);
189
+ const newestWhole = newestOf(whole);
190
+ const family = newestFamily(shardFamilies);
191
+ // Whole-history export wins ties/races; only a strictly newer shard family
192
+ // forces the merge path.
193
+ if (newestWhole && (!family || newestWhole.mtime >= family.maxMtime)) {
194
+ const raw = storage.readJSON(newestWhole.path);
195
+ const candles = rawCandlesOf(raw);
196
+ if (candles.length === 0)
197
+ return null;
198
+ return { path: newestWhole.path, meta: metaOf(raw), candles, assembled: false };
199
+ }
200
+ if (!family)
201
+ return null;
202
+ const ordered = [...family.files].sort((a, b) => (a.path < b.path ? -1 : 1));
203
+ return assembleLpShards(ordered, ordered[ordered.length - 1].path);
204
+ }
205
+ /**
206
+ * Assemble the whole series for a base path directly from its month shards,
207
+ * regardless of any sibling whole-history export. Writers use this to derive
208
+ * the export from the cache, so the monolithic file can never become a
209
+ * narrower or staler snapshot than the shards beside it.
210
+ */
211
+ function loadLpSeriesFromBasePath(basePath) {
212
+ const resolved = path.resolve(basePath);
213
+ const family = readShardFamily(resolved);
214
+ if (family.length === 0)
215
+ return null;
216
+ return assembleLpShards(family, family[family.length - 1].path);
217
+ }
218
+ /**
219
+ * Load one explicit LP path. A month shard is transparently upgraded to its
220
+ * whole family: pointing `--data` at `.shard_2026-10.json` charts the full
221
+ * cached history, never a silent one-month slice. A whole-history export (or
222
+ * a lone shard whose siblings are missing) is read as-is.
223
+ */
224
+ function loadLpSeriesFromPath(filePath) {
225
+ const resolved = path.resolve(filePath);
226
+ const parsed = path.parse(resolved);
227
+ const stem = shardStemFromName(parsed.base, parsed.ext);
228
+ if (stem) {
229
+ const family = readShardFamily(path.join(parsed.dir, `${stem}${parsed.ext}`));
230
+ const assembled = assembleLpShards(family, resolved);
231
+ if (assembled)
232
+ return assembled;
233
+ }
234
+ const raw = storage.readJSON(resolved);
235
+ const candles = rawCandlesOf(raw);
236
+ if (candles.length === 0)
237
+ return null;
238
+ return { path: resolved, meta: metaOf(raw), candles, assembled: false };
46
239
  }
47
- export { findLatestLpData };
240
+ export { findLatestLpData, loadLatestLpSeries, loadLpSeriesFromPath, loadLpSeriesFromBasePath };
48
241
  //# sourceMappingURL=data_discovery.js.map
@@ -0,0 +1,84 @@
1
+ /** A span that was actually queried, with the time the query ran. */
2
+ type QueriedRange = {
3
+ gte: number;
4
+ lte: number;
5
+ at: number | null;
6
+ };
7
+ /** A plain inclusive ms range. */
8
+ type Span = {
9
+ gte: number;
10
+ lte: number;
11
+ };
12
+ declare function shardKeyForTimestamp(tsMs: unknown): string;
13
+ declare function shardBoundsForKey(key: unknown): {
14
+ start: number;
15
+ end: number;
16
+ };
17
+ declare function shardKeysForRange(gteMs: unknown, lteMs: unknown): string[];
18
+ declare function shardPathFor(basePath: string, shardKey: string): string;
19
+ /**
20
+ * If `name` is a shard file for `baseName`, return its `YYYY-MM` key;
21
+ * otherwise null. Guards the `sibling`/family scans against stray files.
22
+ */
23
+ declare function shardKeyFromName(name: string, baseName: string, ext?: string): string | null;
24
+ /**
25
+ * The base stem of a shard filename, or null when `name` is not a shard.
26
+ * `lp_pool_133_1h.shard_2026-10.json` → `lp_pool_133_1h`. Callers that must
27
+ * group or upgrade a family use this instead of re-deriving the marker regex.
28
+ */
29
+ declare function shardStemFromName(name: string, ext?: string): string | null;
30
+ /** Parse a verification timestamp: epoch ms or ISO. Anything else is unknown. */
31
+ declare function timestampMs(raw: unknown): number | null;
32
+ declare function verifiedAt(range: unknown): number | null;
33
+ /**
34
+ * Normalize recorded spans: sort, then merge overlapping / bucket-adjacent
35
+ * ones — but ONLY when they assert the same verification time. Merging spans
36
+ * with different `at` would have to pick one verdict for the whole union, and
37
+ * either pick loses (min → the merged span vouches for buckets the older
38
+ * query never saw; max → buckets verified long ago look freshly verified).
39
+ * Keeping them separate preserves exact per-bucket freshness. The same-`at`
40
+ * case (bulk fetches, and every pre-`at` shard) collapses to one span.
41
+ *
42
+ * `bucketMs` widens adjacency: `0` merges only overlaps, `1` merges adjacent
43
+ * integer-ms spans (fills), a bucket size merges spans a bucket apart (candles).
44
+ */
45
+ declare function unionQueriedRanges(ranges: {
46
+ gte: number;
47
+ lte: number;
48
+ at?: number | null;
49
+ }[], bucketMs?: number): QueriedRange[];
50
+ /**
51
+ * Bound the persisted span list. Because spans merge only when they assert the
52
+ * same `at`, a shard that is re-verified on every run gains one span per run
53
+ * and would otherwise grow without limit. Dropping the OLDEST spans never
54
+ * invents emptiness. The input is already sorted by extent, so the tail is
55
+ * newest.
56
+ */
57
+ declare function compactCoverage(spans: QueriedRange[], maxSpans: number): QueriedRange[];
58
+ declare function rangesCoveredBy(have: {
59
+ gte: number;
60
+ lte: number;
61
+ }[], want: {
62
+ gte: number;
63
+ lte: number;
64
+ }[]): boolean;
65
+ /**
66
+ * Change detection for shard writes: `have` already accounts for every span in
67
+ * `want`, in extent AND in verification time. Without the `at` half, a run
68
+ * whose only news is "these buckets were re-verified 1h later" would skip the
69
+ * write and the next run would fall back to the wide fixed refresh window.
70
+ * An unknown `at` on either side is never treated as fresh.
71
+ */
72
+ declare function coverageSatisfied(have: QueriedRange[], want: QueriedRange[]): boolean;
73
+ /** Every recorded span across the loaded files, newest verification included. */
74
+ declare function allQueriedRanges(fileCover: {
75
+ queried?: QueriedRange[] | null;
76
+ }[]): QueriedRange[];
77
+ declare function clipRangeTo(q: QueriedRange, gteMs: number, lteMs: number): QueriedRange | null;
78
+ /** Sort and merge overlapping / adjacent (integer-ms) spans. */
79
+ declare function mergeSpans(spans: Span[]): Span[];
80
+ /** `span` minus the union of `covered` (both as inclusive ms ranges). */
81
+ declare function subtractRanges(span: Span, covered: Span[]): Span[];
82
+ export { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, shardKeyFromName, shardStemFromName, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges, clipRangeTo, mergeSpans, subtractRanges, };
83
+ export type { QueriedRange, Span };
84
+ //# sourceMappingURL=month_shards.d.ts.map