dexbot 1.6.12 → 1.6.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/CHANGELOG.md +33 -1
  2. package/analysis/README.md +9 -11
  3. package/analysis/ama_fitting/README.md +5 -3
  4. package/claw/README.md +1 -0
  5. package/claw/docs/AI_BOT_LIBRARY_API.md +2 -0
  6. package/claw/modules/claw_bridge.ts +0 -3
  7. package/claw/modules/claw_skill_md.ts +1 -0
  8. package/claw/package.json +3 -3
  9. package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
  10. package/claw/runtimes/openclaw-plugin/package.json +1 -1
  11. package/claw/tsconfig.json +2 -2
  12. package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +1 -14
  13. package/dist/analysis/ama_fitting/fetch_lp_candles.js +51 -41
  14. package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -24
  15. package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +36 -18
  16. package/dist/analysis/ama_fitting/optimizer_high_resolution.js +18 -66
  17. package/dist/analysis/analyze_args.d.ts +12 -0
  18. package/dist/analysis/analyze_args.js +20 -0
  19. package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +1 -16
  20. package/dist/analysis/bot_fitting/backtest_ama_sweep.js +20 -101
  21. package/dist/analysis/bot_fitting/backtest_bot_fitting.js +26 -92
  22. package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
  23. package/dist/analysis/bot_fitting/shared_utils.d.ts +71 -3
  24. package/dist/analysis/bot_fitting/shared_utils.js +124 -8
  25. package/dist/analysis/chart_ui.d.ts +30 -1
  26. package/dist/analysis/chart_ui.js +37 -1
  27. package/dist/analysis/fills_cache.d.ts +3 -23
  28. package/dist/analysis/fills_cache.js +3 -84
  29. package/dist/analysis/fills_source.d.ts +31 -1
  30. package/dist/analysis/fills_source.js +69 -1
  31. package/dist/analysis/grid_correction_check.d.ts +1 -23
  32. package/dist/analysis/grid_correction_check.js +15 -58
  33. package/dist/analysis/math_utils.d.ts +85 -1
  34. package/dist/analysis/math_utils.js +116 -1
  35. package/dist/analysis/price_sources.d.ts +22 -1
  36. package/dist/analysis/price_sources.js +23 -0
  37. package/dist/analysis/resolve_source.js +31 -10
  38. package/dist/analysis/trade_profitability.d.ts +6 -0
  39. package/dist/analysis/trade_profitability.js +23 -97
  40. package/dist/analysis/tradingview/analyze_tradingview.d.ts +1 -17
  41. package/dist/analysis/tradingview/analyze_tradingview.js +1 -1
  42. package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +2 -15
  43. package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
  44. package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +22 -80
  45. package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +11 -37
  46. package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
  47. package/dist/analysis/worker_pool.d.ts +38 -0
  48. package/dist/analysis/worker_pool.js +67 -0
  49. package/dist/dexbot.js +67 -48
  50. package/dist/market_adapter/ama_signal_runner.js +21 -54
  51. package/dist/market_adapter/candle_utils.d.ts +7 -1
  52. package/dist/market_adapter/candle_utils.js +11 -1
  53. package/dist/market_adapter/cli_args.d.ts +22 -0
  54. package/dist/market_adapter/cli_args.js +30 -0
  55. package/dist/market_adapter/core/config_normalizers.js +11 -6
  56. package/dist/market_adapter/core/kibana_candles.d.ts +3 -1
  57. package/dist/market_adapter/core/kibana_candles.js +3 -2
  58. package/dist/market_adapter/core/market_adapter_service.d.ts +85 -85
  59. package/dist/market_adapter/core/market_adapter_service.js +93 -158
  60. package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
  61. package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
  62. package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
  63. package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
  64. package/dist/market_adapter/inputs/fetch_book_data.js +0 -1
  65. package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +63 -107
  66. package/dist/market_adapter/inputs/fetch_lp_data.d.ts +8 -1
  67. package/dist/market_adapter/inputs/fetch_lp_data.js +25 -7
  68. package/dist/market_adapter/inputs/kibana_feed_source.js +4 -26
  69. package/dist/market_adapter/inputs/window_cache.d.ts +4 -72
  70. package/dist/market_adapter/inputs/window_cache.js +52 -177
  71. package/dist/market_adapter/interval_utils.d.ts +11 -1
  72. package/dist/market_adapter/interval_utils.js +16 -1
  73. package/dist/market_adapter/lp_chart_runner.d.ts +3 -0
  74. package/dist/market_adapter/lp_chart_runner.js +37 -19
  75. package/dist/market_adapter/market_adapter.js +27 -72
  76. package/dist/market_adapter/utils/data_discovery.d.ts +45 -1
  77. package/dist/market_adapter/utils/data_discovery.js +220 -27
  78. package/dist/market_adapter/utils/month_shards.d.ts +84 -0
  79. package/dist/market_adapter/utils/month_shards.js +237 -0
  80. package/dist/modules/account_bots.d.ts +1 -0
  81. package/dist/modules/account_bots.js +102 -121
  82. package/dist/modules/account_orders.d.ts +13 -0
  83. package/dist/modules/account_orders.js +38 -44
  84. package/dist/modules/bitshares-native/chain_client.d.ts +1 -1
  85. package/dist/modules/bitshares-native/chain_client.js +14 -13
  86. package/dist/modules/bitshares-native/resolvers.js +22 -25
  87. package/dist/modules/bitshares-native/serial/types.js +22 -50
  88. package/dist/modules/bitshares-native/signing_client.d.ts +1 -1
  89. package/dist/modules/bot_defaults.d.ts +2 -1
  90. package/dist/modules/bot_settings.d.ts +1 -1
  91. package/dist/modules/broadcast_failure.d.ts +2 -1
  92. package/dist/modules/chain_orders.d.ts +1 -1
  93. package/dist/modules/chain_orders.js +51 -96
  94. package/dist/modules/cli_start_onboarding.d.ts +2 -1
  95. package/dist/modules/constants.d.ts +7 -1
  96. package/dist/modules/constants.js +20 -1
  97. package/dist/modules/cr_planner.d.ts +1 -1
  98. package/dist/modules/cr_planner.js +1 -5
  99. package/dist/modules/credit_pricing.d.ts +2 -1
  100. package/dist/modules/credit_pricing.js +36 -40
  101. package/dist/modules/credit_runtime.d.ts +7 -0
  102. package/dist/modules/credit_runtime.js +45 -73
  103. package/dist/modules/daemon_node_health.d.ts +4 -3
  104. package/dist/modules/dexbot_class.d.ts +28 -17
  105. package/dist/modules/dexbot_class.js +105 -58
  106. package/dist/modules/dexbot_cow_runtime.d.ts +0 -7
  107. package/dist/modules/dexbot_cow_runtime.js +54 -28
  108. package/dist/modules/dexbot_credential_client.d.ts +0 -7
  109. package/dist/modules/dexbot_credential_client.js +1 -2
  110. package/dist/modules/dexbot_fill_runtime.d.ts +1 -2
  111. package/dist/modules/dexbot_fill_runtime.js +5 -6
  112. package/dist/modules/dexbot_maintenance_runtime.d.ts +20 -3
  113. package/dist/modules/dexbot_maintenance_runtime.js +84 -69
  114. package/dist/modules/dexbot_startup_runtime.d.ts +0 -1
  115. package/dist/modules/dexbot_startup_runtime.js +13 -14
  116. package/dist/modules/general_settings.d.ts +1 -1
  117. package/dist/modules/graceful_shutdown.js +17 -11
  118. package/dist/modules/key_store.js +0 -1
  119. package/dist/modules/lazy_shim.d.ts +11 -0
  120. package/dist/modules/lazy_shim.js +16 -0
  121. package/dist/modules/node_connect_policy.d.ts +3 -2
  122. package/dist/modules/node_failure_ledger.d.ts +6 -5
  123. package/dist/modules/order/format.d.ts +12 -3
  124. package/dist/modules/order/format.js +15 -3
  125. package/dist/modules/order/genesis_policy.d.ts +3 -3
  126. package/dist/modules/order/genesis_policy.js +5 -2
  127. package/dist/modules/order/grid.d.ts +4 -2
  128. package/dist/modules/order/grid.js +91 -35
  129. package/dist/modules/order/index.d.ts +1 -0
  130. package/dist/modules/order/logger.d.ts +1 -8
  131. package/dist/modules/order/logger.js +0 -9
  132. package/dist/modules/order/manager.d.ts +9 -13
  133. package/dist/modules/order/manager.js +33 -50
  134. package/dist/modules/order/utils/order.js +39 -35
  135. package/dist/modules/order/utils/system.d.ts +9 -29
  136. package/dist/modules/order/utils/system.js +17 -2
  137. package/dist/modules/order/utils/validate.js +2 -2
  138. package/dist/modules/order/utils/withPoolRef.d.ts +2 -1
  139. package/dist/modules/paths.js +32 -34
  140. package/dist/modules/runtime_settings.d.ts +2 -4
  141. package/dist/modules/settings_merge.d.ts +2 -2
  142. package/dist/modules/socket_json_client.d.ts +2 -1
  143. package/dist/modules/types.d.ts +42 -27
  144. package/dist/modules/utils/errors.js +1 -3
  145. package/dist/modules/version_notice.d.ts +11 -10
  146. package/dist/pm2.d.ts +0 -1
  147. package/dist/pm2.js +56 -35
  148. package/dist/scripts/analyze-git.js +2 -15
  149. package/dist/scripts/chart_command.js +2 -2
  150. package/dist/scripts/check_any.d.ts +4 -0
  151. package/dist/scripts/check_any.js +53 -21
  152. package/dist/scripts/diagnose-pool-history.js +1 -6
  153. package/dist/scripts/run-tests.js +0 -1
  154. package/dist/scripts/sync-version.js +0 -10
  155. package/docs/BITSHARES_ONBOARDING.md +8 -12
  156. package/docs/COPY_ON_WRITE_MASTER_PLAN.md +1 -1
  157. package/docs/DEXBOT_COMPARISON.md +6 -6
  158. package/docs/EVOLUTION.md +9 -8
  159. package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
  160. package/docs/README.md +1 -1
  161. package/docs/WORKFLOW.md +1 -0
  162. package/docs/architecture.md +19 -10
  163. package/market_adapter/README.md +6 -4
  164. package/package.json +6 -9
  165. package/scripts/README.md +43 -38
  166. package/tests/README.md +25 -10
  167. package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
  168. package/dist/scripts/diagnose-kibana-candles.js +0 -82
  169. package/scripts/git-viewer.sh +0 -489
  170. package/scripts/postinstall.js +0 -16
@@ -27,8 +27,7 @@
27
27
  * - `isMatch(meta, requestKey)` — same-pool/feed/interval/assets check,
28
28
  * - `fetchRange(gteIso, lteIso)` — query one (sub-)range, gap-filled grid,
29
29
  * - `metaForWindow(window)` — identity meta fields (source/feed/pool/...);
30
- * the runner overrides timeRange with the shard bounds and drops
31
- * chunkIndex, which is meaningless for stable shards.
30
+ * the runner overrides timeRange with the shard bounds.
32
31
  *
33
32
  * Node-only (disk I/O via storage). Browser-safe code must not import this.
34
33
  */
@@ -36,7 +35,8 @@ import { path } from '../../modules/path_api.js';
36
35
  import { getStorage } from '../../modules/storage/index.js';
37
36
  import { writeJsonAtomic } from '../utils/atomic_write.js';
38
37
  import { sleepMs, getErrorName, getErrorMessage } from '../../modules/utils/errors.js';
39
- import { mergeCandles } from '../candle_utils.js';
38
+ import { mergeCandles, higherVolumeWins } from '../candle_utils.js';
39
+ import { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, shardKeyFromName, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges, clipRangeTo, } from '../utils/month_shards.js';
40
40
  const storage = getStorage();
41
41
  const { readJSON } = storage;
42
42
  // Trailing overlap for the newest window. It is the refresh when the cache
@@ -87,49 +87,18 @@ const MAX_SUBFETCH_SPAN_RATIO = 0.5;
87
87
  // current month accumulates.
88
88
  const MAX_COVERAGE_SPANS = 64;
89
89
  // ─── Month-shard naming ───────────────────────────────────────────────────────
90
- // Shard key is the UTC calendar month; bounds are half-open [start, end) so
91
- // every bucket timestamp maps to exactly one shard (a bucket exactly at a
92
- // month boundary belongs to the new month).
93
- function shardKeyForTimestamp(tsMs) {
94
- const d = new Date(Number(tsMs));
95
- return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
96
- }
97
- function shardBoundsForKey(key) {
98
- const parts = String(key).split('-').map(Number);
99
- const start = Date.UTC(parts[0], parts[1] - 1, 1);
100
- const end = parts[1] === 12 ? Date.UTC(parts[0] + 1, 0, 1) : Date.UTC(parts[0], parts[1], 1);
101
- return { start, end };
102
- }
103
- function shardKeysForRange(gteMs, lteMs) {
104
- const keys = [];
105
- let cursor = new Date(Date.UTC(new Date(Number(gteMs)).getUTCFullYear(), new Date(Number(gteMs)).getUTCMonth(), 1));
106
- const last = new Date(Number(lteMs));
107
- while (cursor <= last) {
108
- keys.push(shardKeyForTimestamp(cursor.getTime()));
109
- cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth() + 1, 1));
110
- }
111
- return keys;
112
- }
113
- function shardRangeOverlaps(shardKey, gteMs, lteMs) {
114
- const { start, end } = shardBoundsForKey(shardKey);
115
- return start <= Number(lteMs) && end > Number(gteMs);
116
- }
117
- function shardPathFor(outPath, shardKey) {
118
- const parsed = path.parse(outPath);
119
- return path.join(parsed.dir, `${parsed.name}.shard_${shardKey}${parsed.ext}`);
120
- }
90
+ // Shard key, bounds, path and coverage helpers live in
91
+ // `market_adapter/utils/month_shards.ts` so the candle and fills caches share
92
+ // one contract and cannot disagree on a month boundary.
121
93
  function siblingCacheFiles(outPath) {
122
94
  const resolved = path.resolve(outPath);
123
95
  const parsed = path.parse(resolved);
124
96
  if (!storage.exists(parsed.dir))
125
97
  return [];
126
- const shardPrefix = `${parsed.name}.shard_`;
127
98
  const out = [];
128
99
  for (const name of storage.readdir(parsed.dir)) {
129
- if (!name.endsWith(parsed.ext) || !name.startsWith(shardPrefix))
130
- continue;
131
- const key = name.slice(shardPrefix.length, name.length - parsed.ext.length);
132
- if (!/^\d{4}-\d{2}$/.test(key))
100
+ const key = shardKeyFromName(name, parsed.name, parsed.ext);
101
+ if (!key)
133
102
  continue;
134
103
  out.push({ file: path.join(parsed.dir, name), shardKey: key });
135
104
  }
@@ -188,11 +157,16 @@ function loadBucketCache(outPath, requestKey, isMatch, range) {
188
157
  const shards = [];
189
158
  let files = 0;
190
159
  const scoped = range && Number.isFinite(range.gte) && Number.isFinite(range.lte);
191
- for (const entry of siblingCacheFiles(outPath)) {
192
- // Shards outside the requested range are never even opened —
193
- // a narrow run reads only the months it needs.
194
- if (scoped && !shardRangeOverlaps(entry.shardKey, range.gte, range.lte))
195
- continue;
160
+ // Scoped load: enumerate only the calendar-month shards that overlap the
161
+ // run and open exactly those — no directory listing, and out-of-range
162
+ // months are never even stat'd. Unscoped callers (no finite range) fall
163
+ // back to the full listing.
164
+ const entries = scoped
165
+ ? shardKeysForRange(range.gte, range.lte)
166
+ .map((shardKey) => ({ file: shardPathFor(outPath, shardKey), shardKey }))
167
+ .filter((entry) => storage.exists(entry.file))
168
+ : siblingCacheFiles(outPath);
169
+ for (const entry of entries) {
196
170
  const chunk = readCacheChunk(entry.file, requestKey, isMatch);
197
171
  if (!chunk)
198
172
  continue;
@@ -221,109 +195,12 @@ function cachedCandlesInRange(localCache, gteMs, lteMs) {
221
195
  out.sort((a, b) => Number(a[0]) - Number(b[0]));
222
196
  return out;
223
197
  }
224
- /** Parse a verification timestamp: epoch ms or ISO. Anything else is unknown. */
225
- function timestampMs(raw) {
226
- // `null`/`undefined`/`''` mean "not recorded". They must NOT fall through
227
- // to Number(): Number(null) === 0 is finite, which would read an unknown
228
- // verification time as "verified at the epoch" — i.e. as maximally stale,
229
- // silently disabling the tail refresh on every pre-`at` shard.
230
- if (raw == null || raw === '')
231
- return null;
232
- if (typeof raw === 'string') {
233
- const parsed = Date.parse(raw);
234
- return Number.isFinite(parsed) ? parsed : null;
235
- }
236
- const at = Number(raw);
237
- return Number.isFinite(at) ? at : null;
238
- }
239
- function verifiedAt(range) {
240
- return timestampMs(range?.at);
241
- }
242
- /**
243
- * Normalize recorded spans: sort, then merge overlapping / bucket-adjacent
244
- * ones — but ONLY when they assert the same verification time. Merging spans
245
- * with different `at` would have to pick one verdict for the whole union, and
246
- * either pick loses (min → the merged span vouches for buckets the older
247
- * query never saw; max → buckets verified long ago look freshly verified).
248
- * Keeping them separate preserves exact per-bucket freshness, which is the
249
- * whole point of `at`. Spans therefore accumulate ~1 per tail run, each a few
250
- * dozen bytes — negligible next to the candles in the same file, and the
251
- * same-`at` case (bulk fetches, and every pre-`at` shard) still collapses to
252
- * one span as before.
253
- */
254
- function unionQueriedRanges(ranges, bucketMs) {
255
- const clean = (ranges || [])
256
- .filter((q) => q && Number.isFinite(Number(q.gte)) && Number.isFinite(Number(q.lte)) && Number(q.lte) >= Number(q.gte))
257
- .map((q) => ({ gte: Number(q.gte), lte: Number(q.lte), at: verifiedAt(q) }))
258
- .sort((a, b) => a.gte - b.gte || a.lte - b.lte);
259
- const merged = [];
260
- const gap = Number.isFinite(Number(bucketMs)) && Number(bucketMs) > 0 ? Number(bucketMs) : 0;
261
- for (const q of clean) {
262
- const top = merged[merged.length - 1];
263
- if (top && q.gte <= top.lte + gap && top.at === q.at) {
264
- if (q.lte > top.lte)
265
- top.lte = q.lte;
266
- }
267
- else {
268
- merged.push({ gte: q.gte, lte: q.lte, at: q.at });
269
- }
270
- }
271
- return merged;
272
- }
273
- function rangesCoveredBy(have, want) {
274
- for (const w of want || []) {
275
- let covered = false;
276
- for (const h of have || []) {
277
- if (h.gte <= w.gte && h.lte >= w.lte) {
278
- covered = true;
279
- break;
280
- }
281
- }
282
- if (!covered)
283
- return false;
284
- }
285
- return true;
286
- }
287
- /**
288
- * Change detection for shard writes: `have` already accounts for every span in
289
- * `want`, in extent AND in verification time. Without the `at` half, a run
290
- * whose only news is "these buckets were re-verified 1h later" would skip the
291
- * write and the next run would fall back to the wide fixed refresh window.
292
- * An unknown `at` on either side is never treated as fresh.
293
- */
294
- function coverageSatisfied(have, want) {
295
- if (!rangesCoveredBy(have, want))
296
- return false;
297
- for (const w of want || []) {
298
- const wAt = verifiedAt(w);
299
- if (wAt == null)
300
- continue;
301
- let fresh = false;
302
- for (const h of have || []) {
303
- if (h.gte > w.gte || h.lte < w.lte)
304
- continue;
305
- const hAt = verifiedAt(h);
306
- if (hAt != null && hAt >= wAt) {
307
- fresh = true;
308
- break;
309
- }
310
- }
311
- if (!fresh)
312
- return false;
313
- }
314
- return true;
315
- }
316
- /** Every recorded span across the loaded files, newest verification included. */
317
- function allQueriedRanges(fileCover) {
318
- const out = [];
319
- for (const f of fileCover || []) {
320
- for (const q of f?.queried || []) {
321
- if (q && Number.isFinite(Number(q.gte)) && Number.isFinite(Number(q.lte)))
322
- out.push(q);
323
- }
324
- }
325
- return out;
326
- }
198
+ // ─── Queried-range set ops ────────────────────────────────────────────────────
199
+ // Coverage provenance: normalize recorded spans (sort, merge overlapping or
200
+ // bucket-adjacent) so growth checks and absorption decisions are exact.
201
+ // A span also carries the time its covering query RAN (`at`), which is what
202
+ // makes the tail refresh incremental: extent says "we asked", `at` says "and
203
+ // the answer is still current".
327
204
  /**
328
205
  * How settled a window is, given the recorded coverage.
329
206
  *
@@ -359,29 +236,6 @@ function settleCoverage(opts) {
359
236
  }
360
237
  return { hasTimestamps: true, firstUnsettled: null };
361
238
  }
362
- /**
363
- * Bound the persisted span list. Because spans merge only when they assert
364
- * the same `at`, a shard that is re-verified on every run gains one span per
365
- * run and would grow without limit (and `settleCoverage` scans the list once
366
- * per tail window). Dropping the OLDEST spans never invents emptiness, but it
367
- * is not free: coverage is consulted by BOTH `pruneImmutableGaps` (to skip a
368
- * query) and `settleCoverage` (to decide a bucket is fresh). A dropped span
369
- * makes its buckets look unsettled, and the tail refresh would otherwise start
370
- * at the earliest of them. That is why the refresh is capped at
371
- * TAIL_REFRESH_HOURS: the worst a dropped span can cost is the legacy 48h
372
- * overlap, never weeks. The input is already sorted by extent, so the tail of
373
- * the list is the newest.
374
- */
375
- function compactCoverage(spans, maxSpans) {
376
- const list = spans || [];
377
- const max = Number.isFinite(Number(maxSpans)) && Number(maxSpans) >= 1 ? Math.floor(Number(maxSpans)) : list.length;
378
- return list.length <= max ? list : list.slice(list.length - max);
379
- }
380
- function clipRangeTo(q, gteMs, lteMs) {
381
- const gte = Math.max(q.gte, gteMs);
382
- const lte = Math.min(q.lte, lteMs);
383
- return lte >= gte ? { gte, lte, at: verifiedAt(q) } : null;
384
- }
385
239
  function addUtcMonths(date, months) {
386
240
  const result = new Date(date.getTime());
387
241
  const day = result.getUTCDate();
@@ -391,6 +245,18 @@ function addUtcMonths(date, months) {
391
245
  result.setUTCDate(Math.min(day, lastDay));
392
246
  return result;
393
247
  }
248
+ /** First instant of the UTC month after `date`. */
249
+ function nextUtcMonthStart(date) {
250
+ return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 1));
251
+ }
252
+ /** True when `date` sits exactly on a UTC month start (midnight, day 1). */
253
+ function isUtcMonthStart(date) {
254
+ return date.getUTCDate() === 1
255
+ && date.getUTCHours() === 0
256
+ && date.getUTCMinutes() === 0
257
+ && date.getUTCSeconds() === 0
258
+ && date.getUTCMilliseconds() === 0;
259
+ }
394
260
  function normalizeDateInput(raw, label) {
395
261
  const ms = Date.parse(String(raw || ''));
396
262
  if (!Number.isFinite(ms)) {
@@ -404,10 +270,20 @@ function buildFetchWindowsFromRange(timeRange, chunkMonths) {
404
270
  if (start >= end) {
405
271
  throw new Error(`Invalid fetch range: ${start.toISOString()} must be earlier than ${end.toISOString()}`);
406
272
  }
273
+ const step = Number.isFinite(Number(chunkMonths)) && Number(chunkMonths) >= 1
274
+ ? Math.floor(Number(chunkMonths))
275
+ : 1;
276
+ // Interior window boundaries snap to UTC month starts — the same grid the
277
+ // storage shards use — so a window maps onto whole calendar months instead
278
+ // of straddling two of them. The requested start/end stay exact: only the
279
+ // first window can be short (a mid-month start), and the last is clipped
280
+ // to `end`.
407
281
  const windows = [];
408
282
  let cursor = start;
409
283
  while (cursor < end) {
410
- const next = addUtcMonths(cursor, chunkMonths);
284
+ const next = isUtcMonthStart(cursor)
285
+ ? addUtcMonths(cursor, step)
286
+ : nextUtcMonthStart(cursor);
411
287
  const windowEnd = next < end ? next : end;
412
288
  windows.push({
413
289
  gte: cursor.toISOString(),
@@ -674,9 +550,6 @@ async function fetchRangeLogged(fetchRange, opts) {
674
550
  console.log(formatWindowLine(opts.unit, opts.index, opts.total, opts.gte, opts.lte, `-> ${candles.length} candles (${((Date.now() - startMs) / 1000).toFixed(1)}s)${attemptNote}${partialNote}${note}`));
675
551
  return { candles, complete };
676
552
  }
677
- function higherVolumeWins(existing, incoming) {
678
- return incoming[5] > existing[5] ? incoming : existing;
679
- }
680
553
  function sortedCandles(byTs) {
681
554
  return [...byTs.values()].sort((a, b) => Number(a[0]) - Number(b[0]));
682
555
  }
@@ -872,6 +745,9 @@ async function runCachedWindows(opts) {
872
745
  // else on disk is already current.
873
746
  if (overall) {
874
747
  const fetchedAt = new Date().toISOString();
748
+ // O(1) lookup of each shard's loaded candles for the write-on-change
749
+ // comparison below (instead of a linear scan per shard state).
750
+ const loadedShardCandles = new Map(localCache.shards.map((s) => [s.shardKey, s.candles]));
875
751
  for (const state of shardStates.values()) {
876
752
  const { start, end } = shardBoundsForKey(state.key);
877
753
  // Fold every loaded bucket into its home shard. Presence on disk
@@ -884,7 +760,7 @@ async function runCachedWindows(opts) {
884
760
  if (!prev || Number(c[5] || 0) > Number(prev[5] || 0))
885
761
  state.candles.set(ts, c);
886
762
  }
887
- const before = sortedCandles(new Map((localCache.shards.find((s) => s.shardKey === state.key)?.candles || [])
763
+ const before = sortedCandles(new Map((loadedShardCandles.get(state.key) || [])
888
764
  .filter((c) => Array.isArray(c) && Number.isFinite(Number(c[0])))
889
765
  .map((c) => [Number(c[0]), c])));
890
766
  const after = sortedCandles(state.candles);
@@ -903,7 +779,6 @@ async function runCachedWindows(opts) {
903
779
  const meta = { ...metaForWindow(synthWindow), fetchedAt, queriedRanges: union };
904
780
  meta.timeRange = { gte: synthWindow.gte, lte: synthWindow.lte };
905
781
  meta.shard = state.key;
906
- delete meta.chunkIndex;
907
782
  persistCacheChunk(state.file, meta, after);
908
783
  state.queried = union;
909
784
  state.pendingQueried = [];
@@ -911,5 +786,5 @@ async function runCachedWindows(opts) {
911
786
  }
912
787
  return merged;
913
788
  }
914
- export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, GAP_MERGE_TOLERANCE_BUCKETS, MAX_SUBFETCH_RANGES, MAX_SUBFETCH_SPAN_RATIO, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, cachedCandlesInRange, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
789
+ export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
915
790
  //# sourceMappingURL=window_cache.js.map
@@ -23,5 +23,15 @@ declare function bucketStartMs(nowMs: number, intervalSeconds: number): number |
23
23
  * evaluate the interval/clock.
24
24
  */
25
25
  declare function latestClosedBucketStartMs(nowMs: number, intervalSeconds: number): number | null;
26
- export { toIntervalLabel, slugPart, bucketStartMs, latestClosedBucketStartMs };
26
+ /**
27
+ * Parse a BitShares chain timestamp (e.g. `block_time`) to epoch ms.
28
+ *
29
+ * Chain timestamps are ISO-8601 without a trailing `Z`; `Date.parse` treats
30
+ * them as local time, so append `Z` unless present. Single home for the helper
31
+ * previously copied into market_adapter.ts, kibana_candles.ts,
32
+ * kibana_feed_source.ts, scripts/diagnose-pool-history.ts and
33
+ * scripts/chart_command.ts. Returns `NaN` for empty/unparseable input.
34
+ */
35
+ declare function parseChainTimeToMs(timeStr: unknown): number;
36
+ export { toIntervalLabel, slugPart, bucketStartMs, latestClosedBucketStartMs, parseChainTimeToMs };
27
37
  //# sourceMappingURL=interval_utils.d.ts.map
@@ -51,5 +51,20 @@ function latestClosedBucketStartMs(nowMs, intervalSeconds) {
51
51
  return null;
52
52
  return currentBucketStart - Number(intervalSeconds) * 1000;
53
53
  }
54
- export { toIntervalLabel, slugPart, bucketStartMs, latestClosedBucketStartMs };
54
+ /**
55
+ * Parse a BitShares chain timestamp (e.g. `block_time`) to epoch ms.
56
+ *
57
+ * Chain timestamps are ISO-8601 without a trailing `Z`; `Date.parse` treats
58
+ * them as local time, so append `Z` unless present. Single home for the helper
59
+ * previously copied into market_adapter.ts, kibana_candles.ts,
60
+ * kibana_feed_source.ts, scripts/diagnose-pool-history.ts and
61
+ * scripts/chart_command.ts. Returns `NaN` for empty/unparseable input.
62
+ */
63
+ function parseChainTimeToMs(timeStr) {
64
+ if (!timeStr)
65
+ return Number.NaN;
66
+ const s = String(timeStr);
67
+ return Date.parse(s.endsWith('Z') ? s : `${s}Z`);
68
+ }
69
+ export { toIntervalLabel, slugPart, bucketStartMs, latestClosedBucketStartMs, parseChainTimeToMs };
55
70
  //# sourceMappingURL=interval_utils.js.map
@@ -52,6 +52,7 @@ interface CliArgOptions {
52
52
  }
53
53
  interface MarketChartOptions {
54
54
  dataFile?: string;
55
+ dataBundle?: LpDataBundle;
55
56
  logger?: {
56
57
  log: (...args: unknown[]) => void;
57
58
  };
@@ -61,6 +62,7 @@ interface MarketChartOptions {
61
62
  }
62
63
  interface ComparisonChartOptions {
63
64
  dataFile?: string;
65
+ dataBundle?: LpDataBundle;
64
66
  logger?: {
65
67
  log: (...args: unknown[]) => void;
66
68
  };
@@ -71,6 +73,7 @@ interface ComparisonChartOptions {
71
73
  }
72
74
  interface BundleChartOptions {
73
75
  dataFile?: string;
76
+ dataBundle?: LpDataBundle;
74
77
  logger?: {
75
78
  log: (...args: unknown[]) => void;
76
79
  };
@@ -7,7 +7,7 @@ import { calculateAMA } from './core/strategies/ama.js';
7
7
  import { MARKET_ADAPTER } from '../modules/constants.js';
8
8
  import { generateHTML } from './lp_chart_core.js';
9
9
  import { loadStrategiesForLpChart } from './lp_chart_strategy_loader.js';
10
- import { findLatestLpData } from './utils/data_discovery.js';
10
+ import { findLatestLpData, loadLatestLpSeries, loadLpSeriesFromPath } from './utils/data_discovery.js';
11
11
  import { PATHS } from '../modules/paths.js';
12
12
  /**
13
13
  * LP CHART RUNNER — Shared LP chart orchestration
@@ -30,7 +30,7 @@ import { PATHS } from '../modules/paths.js';
30
30
  * - fetch/export of LP data (`market_adapter/inputs/fetch_lp_data.ts`)
31
31
  */
32
32
  const storage = getStorage();
33
- const { ensureDir, readJSON } = storage;
33
+ const { ensureDir } = storage;
34
34
  const ANALYSIS_CHARTS_DIR = PATHS.ANALYSIS.CHARTS_DIR;
35
35
  const AMA_PROFILES_FILE = PATHS.PROFILES.MARKET_PROFILES_JSON;
36
36
  const DEFAULT_COMPARISON_COLORS = ['#26a69a', '#fb8c00', '#5c9ee6', '#ef5350'];
@@ -100,7 +100,11 @@ function normalizeLpCandle(candle, index) {
100
100
  throw new Error(`Invalid candle row at index ${index}: unsupported format`);
101
101
  }
102
102
  function resolveLpDataFile(dataFile) {
103
- const resolved = dataFile ? path.resolve(dataFile) : findLatestLpData();
103
+ // Explicit path wins. Otherwise resolve through the assembling loader so a
104
+ // shard-only cache yields a representative source path instead of null.
105
+ const resolved = dataFile
106
+ ? path.resolve(dataFile)
107
+ : (loadLatestLpSeries()?.path ?? findLatestLpData());
104
108
  if (!resolved) {
105
109
  throw new Error('No LP data file found. Use --data <path> or run the fetch step first.');
106
110
  }
@@ -109,16 +113,9 @@ function resolveLpDataFile(dataFile) {
109
113
  }
110
114
  return resolved;
111
115
  }
112
- function loadLpDataFile(dataFile) {
113
- const resolved = resolveLpDataFile(dataFile);
114
- const raw = readJSON(resolved);
115
- const meta = (raw.meta ?? null);
116
- const candles = Array.isArray(raw?.candles) ? raw.candles : (Array.isArray(raw) ? raw : []);
117
- if (!Array.isArray(candles) || candles.length === 0) {
118
- throw new Error('No candles in data file.');
119
- }
116
+ function bundleFrom(dataFile, meta, candles) {
120
117
  return {
121
- dataFile: resolved,
118
+ dataFile,
122
119
  meta,
123
120
  candleArrays: candles.map((c, index) => normalizeLpCandle(c, index)),
124
121
  candleObjects: candles.map((c, index) => {
@@ -134,6 +131,24 @@ function loadLpDataFile(dataFile) {
134
131
  }),
135
132
  };
136
133
  }
134
+ function loadLpDataFile(dataFile) {
135
+ // Explicit path: read the file as before. Auto-discovery: assemble the
136
+ // complete series (merging month shards when they are the newest source),
137
+ // so a shard-only cache charts the whole history, not one month.
138
+ if (!dataFile) {
139
+ const series = loadLatestLpSeries();
140
+ if (!series || series.candles.length === 0) {
141
+ throw new Error('No LP data found. Use --data <path> or run the fetch step first.');
142
+ }
143
+ return bundleFrom(series.path, series.meta, series.candles);
144
+ }
145
+ const resolved = resolveLpDataFile(dataFile);
146
+ const series = loadLpSeriesFromPath(resolved);
147
+ if (!series || series.candles.length === 0) {
148
+ throw new Error('No candles in data file.');
149
+ }
150
+ return bundleFrom(series.path, series.meta, series.candles);
151
+ }
137
152
  function openInBrowser(filePath) {
138
153
  const url = `file://${filePath}`;
139
154
  const cmd = process.platform === 'darwin'
@@ -203,7 +218,7 @@ function writeChartHtml({ meta, candleArrays, amaResults, outFile }) {
203
218
  }
204
219
  function generateMarketLpChart(options = {}) {
205
220
  const logger = options.logger ?? console;
206
- const { dataFile, meta, candleArrays } = loadLpDataFile(options.dataFile);
221
+ const { dataFile, meta, candleArrays } = options.dataBundle ?? loadLpDataFile(options.dataFile);
207
222
  logger.log(`Reading: ${path.relative(process.cwd(), dataFile)}`);
208
223
  const poolLabel = meta?.pool ? `pool ${meta.pool}` : `${meta?.assetA?.symbol || '?'}\/${meta?.assetB?.symbol || '?'}`;
209
224
  logger.log(` ${candleArrays.length} candles · ${poolLabel}`);
@@ -241,7 +256,7 @@ function generateMarketLpChart(options = {}) {
241
256
  }
242
257
  function generateComparisonLpChart(options = {}) {
243
258
  const logger = options.logger ?? console;
244
- const { dataFile, meta, candleArrays, candleObjects } = loadLpDataFile(options.dataFile);
259
+ const { dataFile, meta, candleArrays, candleObjects } = options.dataBundle ?? loadLpDataFile(options.dataFile);
245
260
  const defaultStrategies = options.defaultStrategies ?? DEFAULT_COMPARISON_STRATEGIES;
246
261
  const strategies = loadStrategiesForLpChart({
247
262
  dataFile,
@@ -284,23 +299,26 @@ function generateComparisonLpChart(options = {}) {
284
299
  }
285
300
  function generateLpChartBundle(options = {}) {
286
301
  const logger = options.logger ?? console;
287
- const dataFile = resolveLpDataFile(options.dataFile);
288
- logger.log(`Generating LP charts from ${path.relative(process.cwd(), dataFile)}`);
302
+ // Resolve/assemble ONCE, then hand the same loaded bundle to both charts —
303
+ // auto-discovery over shards would otherwise re-read and re-merge the whole
304
+ // family for each sub-chart.
305
+ const bundle = options.dataBundle ?? loadLpDataFile(options.dataFile);
306
+ logger.log(`Generating LP charts from ${path.relative(process.cwd(), bundle.dataFile)}`);
289
307
  const marketChart = generateMarketLpChart({
290
- dataFile,
308
+ dataBundle: bundle,
291
309
  noOpen: options.noOpen,
292
310
  logger,
293
311
  profilesFile: options.profilesFile,
294
312
  });
295
313
  const comparisonChart = generateComparisonLpChart({
296
- dataFile,
314
+ dataBundle: bundle,
297
315
  noOpen: options.openComparison === true ? !!options.noOpen : true,
298
316
  logger,
299
317
  defaultStrategies: options.defaultStrategies ?? DEFAULT_COMPARISON_STRATEGIES,
300
318
  profilesFile: options.comparisonProfilesFile ?? null,
301
319
  });
302
320
  return {
303
- dataFile,
321
+ dataFile: bundle.dataFile,
304
322
  marketChart,
305
323
  comparisonChart,
306
324
  };
@@ -12,7 +12,7 @@ import { normalizeBotEntry } from '../modules/bot_settings.js';
12
12
  import * as kibanaSource from './inputs/kibana_source.js';
13
13
  import * as kibanaMarketSource from './core/kibana_market_candles.js';
14
14
  import { tradesToCandles, detectMissingCandleTimestamps, fillCandleGaps, detectStaleTail, pruneStaleTail, mergeCandles } from './candle_utils.js';
15
- import { toIntervalLabel, bucketStartMs, latestClosedBucketStartMs } from './interval_utils.js';
15
+ import { toIntervalLabel, bucketStartMs, latestClosedBucketStartMs, parseChainTimeToMs } from './interval_utils.js';
16
16
  import { loadMarketProfiles } from '../analysis/tradingview/tradingview_uplot_chart_generator.js';
17
17
  import { candleFileForBot as candleFilePathForLabel } from '../analysis/bot_key_utils.js';
18
18
  import { writeJsonAtomic } from './utils/atomic_write.js';
@@ -27,6 +27,7 @@ import { resetMarketAdapterWhitelistCache, isBotWhitelisted, isBotDynamicWeightW
27
27
  import { normalizeAssetSymbol, normalizeMarketSource, resolveBotContext, isExactPair, isSamePair, isExactPairIds, isSamePairIds, getBitsharesClient, setBitsharesClientForTests, } from './utils/chain.js';
28
28
  import { normalizeNativeMarketHistoryCandles, } from './utils/native_history.js';
29
29
  import { formatLogPercent, buildWeightSummary, buildDynamicWeightInputsLog, buildDynamicWeightTuningLog, buildAsymmetricBoundsLog, buildStartupDefaultsLog, } from './log_format.js';
30
+ import { walkCliArgs } from './cli_args.js';
30
31
  /**
31
32
  * PRICE ADAPTER — standalone or auto-launched by dexbot runtime.
32
33
  *
@@ -616,73 +617,33 @@ function parseArgs() {
616
617
  const provided = {
617
618
  deltaThresholdPercent: false,
618
619
  };
619
- for (let i = 0; i < args.length; i++) {
620
- const a = args[i];
621
- const v = args[i + 1];
622
- switch (a) {
623
- case '--once':
624
- cfg.once = true;
625
- break;
626
- case '--pollSeconds':
627
- cfg.pollSeconds = Number(v);
628
- i++;
629
- break;
630
- case '--deltaPercent':
620
+ walkCliArgs(args, {
621
+ onHelp: () => {
622
+ printHelp();
623
+ process.exit(0);
624
+ },
625
+ flag: {
626
+ '--once': () => { cfg.once = true; },
627
+ '--metricsJson': () => { cfg.metricsJson = true; },
628
+ '--quiet': () => { cfg.quiet = true; },
629
+ '--dryRun': () => { cfg.dryRun = true; },
630
+ '--whitelist-all': () => { cfg.whitelistAll = true; },
631
+ },
632
+ value: {
633
+ '--pollSeconds': (v) => { cfg.pollSeconds = Number(v); },
634
+ '--deltaPercent': (v) => {
631
635
  cfg.deltaThresholdPercent = Number(v);
632
636
  provided.deltaThresholdPercent = true;
633
- i++;
634
- break;
635
- case '--gridResetFactor':
636
- throw new Error('--gridResetFactor is no longer supported; use --deltaPercent <percent>');
637
- case '--bootstrapHours':
638
- cfg.bootstrapLookbackHours = Number(v);
639
- i++;
640
- break;
641
- case '--nativeBackfillHours':
642
- cfg.nativeBackfillHours = Number(v);
643
- i++;
644
- break;
645
- case '--maxStaleHours':
646
- cfg.maxStaleHours = Number(v);
647
- i++;
648
- break;
649
- case '--sourceRetries':
650
- cfg.sourceRetries = Number(v);
651
- i++;
652
- break;
653
- case '--retryDelayMs':
654
- cfg.retryDelayMs = Number(v);
655
- i++;
656
- break;
657
- case '--metricsJson':
658
- cfg.metricsJson = true;
659
- break;
660
- case '--quiet':
661
- cfg.quiet = true;
662
- break;
663
- case '--dryRun':
664
- cfg.dryRun = true;
665
- break;
666
- case '--whitelist-all':
667
- cfg.whitelistAll = true;
668
- break;
669
- case '--maxPages':
670
- cfg.maxPages = Number(v);
671
- i++;
672
- break;
673
- case '--pageLimit':
674
- cfg.pageLimit = Number(v);
675
- i++;
676
- break;
677
- case '--help':
678
- case '-h':
679
- printHelp();
680
- process.exit(0);
681
- break;
682
- default:
683
- throw new Error(`Unknown argument: ${a}`);
684
- }
685
- }
637
+ },
638
+ '--bootstrapHours': (v) => { cfg.bootstrapLookbackHours = Number(v); },
639
+ '--nativeBackfillHours': (v) => { cfg.nativeBackfillHours = Number(v); },
640
+ '--maxStaleHours': (v) => { cfg.maxStaleHours = Number(v); },
641
+ '--sourceRetries': (v) => { cfg.sourceRetries = Number(v); },
642
+ '--retryDelayMs': (v) => { cfg.retryDelayMs = Number(v); },
643
+ '--maxPages': (v) => { cfg.maxPages = Number(v); },
644
+ '--pageLimit': (v) => { cfg.pageLimit = Number(v); },
645
+ },
646
+ });
686
647
  const merged = applyRuntimeDefaultsFromGeneralSettings(cfg, provided);
687
648
  return validateConfig(merged);
688
649
  }
@@ -808,12 +769,6 @@ function loadJson(filePath, defaultValue) {
808
769
  function saveJson(filePath, data) {
809
770
  writeJsonAtomic(filePath, data);
810
771
  }
811
- function parseChainTimeToMs(timeStr) {
812
- if (!timeStr)
813
- return Number.NaN;
814
- const s = String(timeStr);
815
- return Date.parse(s.endsWith('Z') ? s : `${s}Z`);
816
- }
817
772
  function candleFileForBot(botKey, intervalSeconds = RUNTIME_DEFAULTS.intervalSeconds) {
818
773
  const label = intervalSeconds === RUNTIME_DEFAULTS.intervalSeconds
819
774
  ? RUNTIME_DEFAULTS.intervalLabel