dexbot 1.6.11 → 1.6.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -4
- package/README.md +2 -1
- package/analysis/README.md +11 -11
- package/analysis/ama_fitting/README.md +5 -3
- package/analysis/tradingview/README.md +1 -1
- package/analysis/trend_detection/DYNAMIC_WEIGHT_RESEARCH.md +3 -3
- package/claw/README.md +1 -0
- package/claw/docs/AI_BOT_LIBRARY_API.md +2 -0
- package/claw/docs/DEXBOT2_TUNING_CHEAT_SHEET.md +4 -3
- package/claw/modules/chain_broadcast.ts +7 -1
- package/claw/modules/claw_bridge.ts +0 -3
- package/claw/modules/claw_launcher.ts +1 -1
- package/claw/modules/claw_skill_md.ts +1 -0
- package/claw/modules/launcher_mode_detector.ts +9 -2
- package/claw/package.json +3 -3
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/claw/tsconfig.json +2 -2
- package/dist/analysis/account_resolver.d.ts +1 -1
- package/dist/analysis/account_resolver.js +2 -1
- package/dist/analysis/ama_fitting/analyze_ama_price_changes.js +3 -4
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +3 -21
- package/dist/analysis/ama_fitting/calibrate_convergence_er.js +0 -1
- package/dist/analysis/ama_fitting/fetch_lp_candles.js +55 -45
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -8
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +45 -24
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js +34 -78
- package/dist/analysis/analyze_args.d.ts +43 -0
- package/dist/analysis/analyze_args.js +61 -0
- package/dist/analysis/analyze_dynamic_weight.js +7 -17
- package/dist/analysis/analyze_kalman.js +7 -17
- package/dist/analysis/analyze_regime.js +7 -17
- package/dist/analysis/analyze_regime_windows.js +8 -18
- package/dist/analysis/analyze_risk_profile.js +8 -18
- package/dist/analysis/analyze_trade_heatmap.js +7 -17
- package/dist/analysis/analyze_volatility.js +7 -17
- package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +221 -28
- package/dist/analysis/bot_fitting/backtest_ama_sweep.js +23 -103
- package/dist/analysis/bot_fitting/backtest_bot_fitting.d.ts +34 -4
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js +32 -150
- package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
- package/dist/analysis/bot_fitting/shared_utils.d.ts +77 -9
- package/dist/analysis/bot_fitting/shared_utils.js +125 -9
- package/dist/analysis/bot_key_utils.d.ts +18 -10
- package/dist/analysis/bot_key_utils.js +7 -4
- package/dist/analysis/bot_usage/discover_bot_accounts.js +44 -47
- package/dist/analysis/bot_usage/kibana_bot_queries.d.ts +1 -21
- package/dist/analysis/chain_pool.d.ts +3 -1
- package/dist/analysis/chart_ui.d.ts +35 -3
- package/dist/analysis/chart_ui.js +43 -4
- package/dist/analysis/chart_utils.d.ts +5 -5
- package/dist/analysis/chart_utils.js +9 -0
- package/dist/analysis/fills_cache.d.ts +4 -24
- package/dist/analysis/fills_cache.js +3 -84
- package/dist/analysis/fills_source.d.ts +33 -3
- package/dist/analysis/fills_source.js +83 -13
- package/dist/analysis/grid_correction_check.d.ts +2 -24
- package/dist/analysis/grid_correction_check.js +27 -69
- package/dist/analysis/math_utils.d.ts +91 -6
- package/dist/analysis/math_utils.js +119 -7
- package/dist/analysis/pnl_report.js +29 -29
- package/dist/analysis/price_sources.d.ts +30 -9
- package/dist/analysis/price_sources.js +28 -4
- package/dist/analysis/resolve_bot_accounts.js +2 -2
- package/dist/analysis/resolve_source.d.ts +1 -1
- package/dist/analysis/resolve_source.js +31 -10
- package/dist/analysis/trade_profitability.d.ts +29 -2
- package/dist/analysis/trade_profitability.js +73 -147
- package/dist/analysis/tradingview/analyze_tradingview.d.ts +3 -8
- package/dist/analysis/tradingview/analyze_tradingview.js +7 -6
- package/dist/analysis/tradingview/grid_reset_config.d.ts +41 -32
- package/dist/analysis/tradingview/grid_reset_config.js +23 -11
- package/dist/analysis/tradingview/grid_reset_sim.d.ts +75 -9
- package/dist/analysis/tradingview/grid_reset_sim.js +4 -3
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.d.ts +69 -4
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +20 -34
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +26 -84
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.d.ts +63 -1
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +16 -41
- package/dist/analysis/trend_detection/huber_scale_variants.d.ts +9 -3
- package/dist/analysis/trend_detection/huber_scale_variants.js +4 -4
- package/dist/analysis/trend_detection/kalman_chart_generator.d.ts +30 -1
- package/dist/analysis/trend_detection/kalman_chart_generator.js +3 -3
- package/dist/analysis/trend_detection/regime_chart_generator.d.ts +31 -1
- package/dist/analysis/trend_detection/regime_chart_generator.js +2 -2
- package/dist/analysis/trend_detection/volatility_chart_generator.d.ts +33 -1
- package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
- package/dist/analysis/worker_pool.d.ts +38 -0
- package/dist/analysis/worker_pool.js +67 -0
- package/dist/credential-daemon.js +34 -20
- package/dist/dexbot.js +78 -59
- package/dist/market_adapter/ama_signal_runner.js +21 -54
- package/dist/market_adapter/candle_utils.d.ts +29 -19
- package/dist/market_adapter/candle_utils.js +35 -21
- package/dist/market_adapter/cli_args.d.ts +22 -0
- package/dist/market_adapter/cli_args.js +30 -0
- package/dist/market_adapter/core/asymmetric_bounds.d.ts +13 -4
- package/dist/market_adapter/core/config_normalizers.d.ts +3 -3
- package/dist/market_adapter/core/config_normalizers.js +11 -6
- package/dist/market_adapter/core/kibana_candles.d.ts +56 -19
- package/dist/market_adapter/core/kibana_candles.js +12 -9
- package/dist/market_adapter/core/kibana_client.d.ts +9 -1
- package/dist/market_adapter/core/kibana_client.js +8 -7
- package/dist/market_adapter/core/kibana_market_candles.d.ts +3 -2
- package/dist/market_adapter/core/market_adapter_service.d.ts +320 -153
- package/dist/market_adapter/core/market_adapter_service.js +228 -269
- package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
- package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
- package/dist/market_adapter/core/strategies/ama.d.ts +10 -2
- package/dist/market_adapter/core/strategies/ama.js +2 -4
- package/dist/market_adapter/core/strategies/ama_slope_model.d.ts +14 -1
- package/dist/market_adapter/core/strategies/ama_slope_model.js +12 -34
- package/dist/market_adapter/core/strategies/atr/calculator.d.ts +7 -2
- package/dist/market_adapter/core/strategies/atr/calculator.js +2 -34
- package/dist/market_adapter/core/strategies/collateral_manager.d.ts +7 -1
- package/dist/market_adapter/core/strategies/collateral_manager.js +0 -22
- package/dist/market_adapter/core/strategies/dynamic_weight_series.d.ts +34 -6
- package/dist/market_adapter/core/strategies/dynamic_weight_series.js +7 -7
- package/dist/market_adapter/core/strategies/regime_gate.d.ts +24 -15
- package/dist/market_adapter/core/strategies/regime_gate.js +4 -31
- package/dist/market_adapter/core/strategies/regime_interp.d.ts +7 -1
- package/dist/market_adapter/core/strategies/regime_interp.js +14 -22
- package/dist/market_adapter/core/strategies/volatility_shift.d.ts +11 -1
- package/dist/market_adapter/core/strategies/volatility_shift.js +0 -13
- package/dist/market_adapter/inputs/cex_candle_parsing.d.ts +29 -0
- package/dist/market_adapter/inputs/cex_candle_parsing.js +52 -0
- package/dist/market_adapter/inputs/fetch_book_data.d.ts +20 -4
- package/dist/market_adapter/inputs/fetch_book_data.js +3 -2
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +160 -332
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +50 -8
- package/dist/market_adapter/inputs/fetch_lp_data.js +47 -22
- package/dist/market_adapter/inputs/kibana_feed_source.d.ts +74 -15
- package/dist/market_adapter/inputs/kibana_feed_source.js +26 -41
- package/dist/market_adapter/inputs/kibana_source.d.ts +25 -3
- package/dist/market_adapter/inputs/kibana_source.js +2 -2
- package/dist/market_adapter/inputs/window_cache.d.ts +93 -156
- package/dist/market_adapter/inputs/window_cache.js +59 -184
- package/dist/market_adapter/interval_utils.d.ts +15 -5
- package/dist/market_adapter/interval_utils.js +16 -1
- package/dist/market_adapter/log_format.d.ts +6 -6
- package/dist/market_adapter/log_format.js +28 -21
- package/dist/market_adapter/lp_chart_core.d.ts +22 -1
- package/dist/market_adapter/lp_chart_core.js +26 -192
- package/dist/market_adapter/lp_chart_runner.d.ts +10 -7
- package/dist/market_adapter/lp_chart_runner.js +37 -19
- package/dist/market_adapter/lp_chart_strategy_loader.d.ts +41 -6
- package/dist/market_adapter/lp_chart_strategy_loader.js +8 -7
- package/dist/market_adapter/market_adapter.d.ts +94 -32
- package/dist/market_adapter/market_adapter.js +94 -120
- package/dist/market_adapter/utils/adapter_client.d.ts +3 -3
- package/dist/market_adapter/utils/adapter_client.js +2 -2
- package/dist/market_adapter/utils/atomic_write.d.ts +8 -1
- package/dist/market_adapter/utils/chain.d.ts +50 -41
- package/dist/market_adapter/utils/chain.js +40 -46
- package/dist/market_adapter/utils/data_discovery.d.ts +49 -2
- package/dist/market_adapter/utils/data_discovery.js +220 -27
- package/dist/market_adapter/utils/dynamic_grid_snapshot.d.ts +12 -3
- package/dist/market_adapter/utils/file_json_cache.d.ts +2 -2
- package/dist/market_adapter/utils/file_lock.d.ts +28 -14
- package/dist/market_adapter/utils/file_lock.js +36 -32
- package/dist/market_adapter/utils/month_shards.d.ts +84 -0
- package/dist/market_adapter/utils/month_shards.js +237 -0
- package/dist/market_adapter/utils/native_history.d.ts +9 -2
- package/dist/market_adapter/utils/native_history.js +35 -28
- package/dist/modules/account_bots.d.ts +578 -21
- package/dist/modules/account_bots.js +272 -267
- package/dist/modules/account_orders.d.ts +79 -30
- package/dist/modules/account_orders.js +60 -64
- package/dist/modules/authority_resolver.d.ts +12 -2
- package/dist/modules/authority_resolver.js +15 -8
- package/dist/modules/bitshares-native/chain_client.d.ts +39 -4
- package/dist/modules/bitshares-native/chain_client.js +59 -59
- package/dist/modules/bitshares-native/crypto/ecc.browser.js +11 -171
- package/dist/modules/bitshares-native/crypto/ecc.js +13 -176
- package/dist/modules/bitshares-native/crypto/ecc_core.d.ts +33 -0
- package/dist/modules/bitshares-native/crypto/ecc_core.js +213 -0
- package/dist/modules/bitshares-native/crypto/ecc_selector.d.ts +2 -1
- package/dist/modules/bitshares-native/crypto/ecc_selector.js +1 -13
- package/dist/modules/bitshares-native/index.d.ts +1 -1
- package/dist/modules/bitshares-native/lru_cache.d.ts +7 -7
- package/dist/modules/bitshares-native/resolvers.d.ts +18 -6
- package/dist/modules/bitshares-native/resolvers.js +22 -25
- package/dist/modules/bitshares-native/serial/lazy_buffer.d.ts +4 -0
- package/dist/modules/bitshares-native/serial/lazy_buffer.js +21 -0
- package/dist/modules/bitshares-native/serial/operations.d.ts +1 -1
- package/dist/modules/bitshares-native/serial/serializer.d.ts +16 -12
- package/dist/modules/bitshares-native/serial/serializer.js +7 -13
- package/dist/modules/bitshares-native/serial/types.d.ts +25 -16
- package/dist/modules/bitshares-native/serial/types.js +73 -108
- package/dist/modules/bitshares-native/signing_client.d.ts +32 -2
- package/dist/modules/bitshares-native/signing_client.js +19 -14
- package/dist/modules/bitshares-native/subscriptions.d.ts +81 -1
- package/dist/modules/bitshares-native/subscriptions.js +12 -12
- package/dist/modules/bitshares-native/transport.d.ts +5 -4
- package/dist/modules/bitshares-native/transport.js +5 -4
- package/dist/modules/bitshares-native/tx/builder.d.ts +40 -5
- package/dist/modules/bitshares-native/tx/builder.js +4 -3
- package/dist/modules/bitshares-native/tx/tx_cache.d.ts +4 -4
- package/dist/modules/bitshares_client.d.ts +45 -13
- package/dist/modules/bitshares_client.js +28 -19
- package/dist/modules/bot_defaults.d.ts +8 -16
- package/dist/modules/bot_defaults.js +1 -5
- package/dist/modules/bot_settings.d.ts +44 -10
- package/dist/modules/bot_settings.js +11 -8
- package/dist/modules/bots_file_lock.d.ts +6 -6
- package/dist/modules/bots_file_lock.js +16 -12
- package/dist/modules/broadcast_failure.d.ts +4 -3
- package/dist/modules/broadcast_failure.js +5 -4
- package/dist/modules/chain_keys.d.ts +42 -40
- package/dist/modules/chain_keys.js +53 -45
- package/dist/modules/chain_orders.d.ts +94 -60
- package/dist/modules/chain_orders.js +81 -118
- package/dist/modules/cli_start_onboarding.d.ts +2 -1
- package/dist/modules/constants.d.ts +534 -32
- package/dist/modules/constants.js +35 -48
- package/dist/modules/cr_planner.d.ts +41 -3
- package/dist/modules/cr_planner.js +1 -5
- package/dist/modules/credential_policy.d.ts +68 -8
- package/dist/modules/credential_policy.js +37 -31
- package/dist/modules/credential_session_cache.d.ts +2 -2
- package/dist/modules/credit_pricing.d.ts +43 -6
- package/dist/modules/credit_pricing.js +70 -60
- package/dist/modules/credit_runtime.d.ts +283 -118
- package/dist/modules/credit_runtime.js +500 -231
- package/dist/modules/crypto/browser_provider.js +6 -4
- package/dist/modules/crypto/node_provider.js +1 -1
- package/dist/modules/crypto/pure_scrypt.js +35 -4
- package/dist/modules/crypto/pure_secp256k1.js +72 -26
- package/dist/modules/crypto/sync.d.ts +9 -9
- package/dist/modules/crypto/sync.js +16 -9
- package/dist/modules/daemon_node_health.d.ts +4 -3
- package/dist/modules/dexbot_class.d.ts +213 -146
- package/dist/modules/dexbot_class.js +183 -86
- package/dist/modules/dexbot_cow_runtime.d.ts +160 -87
- package/dist/modules/dexbot_cow_runtime.js +302 -234
- package/dist/modules/dexbot_credential_client.d.ts +19 -18
- package/dist/modules/dexbot_credential_client.js +1 -2
- package/dist/modules/dexbot_fill_runtime.d.ts +63 -49
- package/dist/modules/dexbot_fill_runtime.js +53 -47
- package/dist/modules/dexbot_maintenance_runtime.d.ts +127 -71
- package/dist/modules/dexbot_maintenance_runtime.js +187 -147
- package/dist/modules/dexbot_startup_runtime.d.ts +11 -11
- package/dist/modules/dexbot_startup_runtime.js +56 -57
- package/dist/modules/dexbot_state_recovery.d.ts +13 -12
- package/dist/modules/dexbot_state_recovery.js +18 -13
- package/dist/modules/fund_registry.d.ts +4 -2
- package/dist/modules/fund_registry.js +10 -10
- package/dist/modules/general_settings.d.ts +30 -4
- package/dist/modules/graceful_shutdown.d.ts +3 -3
- package/dist/modules/graceful_shutdown.js +19 -13
- package/dist/modules/grid_price_source.d.ts +1 -1
- package/dist/modules/key_store.d.ts +14 -8
- package/dist/modules/key_store.js +7 -7
- package/dist/modules/launcher/adapter_requirement.d.ts +11 -5
- package/dist/modules/launcher/adapter_requirement.js +5 -5
- package/dist/modules/launcher/bot_supervisor.d.ts +106 -24
- package/dist/modules/launcher/bot_supervisor.js +12 -11
- package/dist/modules/launcher/credential_bootstrap.d.ts +1 -1
- package/dist/modules/launcher/credential_bootstrap.js +1 -1
- package/dist/modules/launcher/credential_daemon.d.ts +4 -4
- package/dist/modules/launcher/credential_daemon.js +2 -2
- package/dist/modules/launcher/credential_secret.d.ts +1 -1
- package/dist/modules/launcher/foreign_cred_daemon.d.ts +2 -2
- package/dist/modules/launcher/market_adapter_runtime.d.ts +61 -6
- package/dist/modules/launcher/market_adapter_watchdog.d.ts +10 -2
- package/dist/modules/launcher/market_adapter_watchdog.js +4 -0
- package/dist/modules/launcher/monolithic_runtime.d.ts +18 -16
- package/dist/modules/launcher/status_reporting.d.ts +22 -14
- package/dist/modules/launcher/supervisor_control.d.ts +8 -1
- package/dist/modules/launcher/supervisor_control.js +4 -2
- package/dist/modules/lazy_shim.d.ts +11 -0
- package/dist/modules/lazy_shim.js +16 -0
- package/dist/modules/market_adapter_whitelist.d.ts +9 -3
- package/dist/modules/market_adapter_whitelist.js +15 -3
- package/dist/modules/node_connect_policy.d.ts +3 -2
- package/dist/modules/node_failure_ledger.d.ts +6 -5
- package/dist/modules/node_manager.d.ts +41 -11
- package/dist/modules/node_manager.js +5 -4
- package/dist/modules/order/accounting.d.ts +53 -24
- package/dist/modules/order/accounting.js +16 -15
- package/dist/modules/order/async_lock.d.ts +5 -2
- package/dist/modules/order/async_lock.js +1 -1
- package/dist/modules/order/export.d.ts +7 -6
- package/dist/modules/order/export.js +1 -7
- package/dist/modules/order/format.d.ts +16 -7
- package/dist/modules/order/format.js +15 -3
- package/dist/modules/order/genesis_policy.d.ts +24 -21
- package/dist/modules/order/genesis_policy.js +17 -10
- package/dist/modules/order/grid.d.ts +144 -36
- package/dist/modules/order/grid.js +226 -145
- package/dist/modules/order/grid_reconcile.d.ts +21 -20
- package/dist/modules/order/grid_reconcile.js +13 -12
- package/dist/modules/order/grid_reconcile_internal.d.ts +85 -72
- package/dist/modules/order/grid_reconcile_internal.js +28 -19
- package/dist/modules/order/index.d.ts +264 -1
- package/dist/modules/order/logger.d.ts +46 -19
- package/dist/modules/order/logger.js +38 -49
- package/dist/modules/order/logger_state.d.ts +4 -4
- package/dist/modules/order/manager.d.ts +232 -191
- package/dist/modules/order/manager.js +110 -130
- package/dist/modules/order/processed_fill_store.d.ts +18 -3
- package/dist/modules/order/processed_fill_store.js +16 -27
- package/dist/modules/order/strategy.d.ts +21 -13
- package/dist/modules/order/strategy.js +5 -6
- package/dist/modules/order/sync_engine.d.ts +91 -33
- package/dist/modules/order/sync_engine.js +84 -53
- package/dist/modules/order/utils/math.d.ts +157 -102
- package/dist/modules/order/utils/math.js +77 -38
- package/dist/modules/order/utils/order.d.ts +310 -110
- package/dist/modules/order/utils/order.js +153 -126
- package/dist/modules/order/utils/slot.d.ts +1 -1
- package/dist/modules/order/utils/system.d.ts +156 -26
- package/dist/modules/order/utils/system.js +103 -71
- package/dist/modules/order/utils/validate.d.ts +89 -41
- package/dist/modules/order/utils/validate.js +26 -24
- package/dist/modules/order/utils/withPoolRef.d.ts +6 -4
- package/dist/modules/order/utils/withPoolRef.js +18 -12
- package/dist/modules/order/working_grid.d.ts +14 -13
- package/dist/modules/paths.d.ts +1 -4
- package/dist/modules/paths.js +33 -61
- package/dist/modules/process_discovery.d.ts +1 -1
- package/dist/modules/process_discovery.js +2 -1
- package/dist/modules/runtime.d.ts +7 -3
- package/dist/modules/runtime.js +4 -3
- package/dist/modules/runtime_settings.d.ts +14 -11
- package/dist/modules/runtime_settings.js +30 -24
- package/dist/modules/settings_merge.d.ts +29 -3
- package/dist/modules/settings_merge.js +56 -78
- package/dist/modules/socket_json_client.d.ts +9 -5
- package/dist/modules/storage/browser_adapter.d.ts +38 -53
- package/dist/modules/storage/browser_adapter.js +7 -30
- package/dist/modules/storage/index.d.ts +1 -1
- package/dist/modules/storage/node_adapter.d.ts +45 -27
- package/dist/modules/storage/node_adapter.js +9 -9
- package/dist/modules/storage/types.d.ts +5 -4
- package/dist/modules/types.d.ts +1090 -6
- package/dist/modules/types.js +4 -1
- package/dist/modules/utils/asset_symbols.d.ts +2 -1
- package/dist/modules/utils/asset_symbols.js +2 -1
- package/dist/modules/utils/errors.d.ts +9 -0
- package/dist/modules/utils/errors.js +24 -4
- package/dist/modules/utils/time_range.d.ts +1 -12
- package/dist/modules/utils/time_range.js +1 -1
- package/dist/modules/validate_profiles.js +53 -43
- package/dist/modules/version_notice.d.ts +23 -11
- package/dist/modules/version_notice.js +6 -4
- package/dist/pm2.d.ts +28 -27
- package/dist/pm2.js +61 -40
- package/dist/scripts/analyze-credit.d.ts +2 -1
- package/dist/scripts/analyze-credit.js +23 -24
- package/dist/scripts/analyze-git.js +2 -15
- package/dist/scripts/analyze-orders.d.ts +77 -9
- package/dist/scripts/analyze-orders.js +65 -40
- package/dist/scripts/chart_command.d.ts +27 -18
- package/dist/scripts/chart_command.js +15 -13
- package/dist/scripts/check_any.d.ts +22 -0
- package/dist/scripts/check_any.js +158 -0
- package/dist/scripts/diagnose-pool-history.js +13 -13
- package/dist/scripts/print_grid.js +0 -1
- package/dist/scripts/run-tests.js +39 -8
- package/dist/scripts/runner.js +1 -1
- package/dist/scripts/sync-version.js +0 -10
- package/dist/scripts/test-credit-renewal.js +30 -21
- package/dist/scripts/update.d.ts +10 -5
- package/dist/scripts/update.js +168 -103
- package/dist/scripts/update_dist_freshness.js +1 -0
- package/dist/scripts/update_layout.d.ts +61 -0
- package/dist/scripts/update_layout.js +180 -0
- package/dist/scripts/validate_bots.js +2 -1
- package/dist/scripts/verify-browser-bundle.js +8 -0
- package/dist/unlock.d.ts +28 -3
- package/dist/unlock.js +63 -35
- package/docs/BITSHARES_ONBOARDING.md +10 -12
- package/docs/COPY_ON_WRITE_MASTER_PLAN.md +2 -2
- package/docs/COW_INVARIANTS.md +13 -6
- package/docs/DEXBOT_COMPARISON.md +8 -8
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +5 -5
- package/docs/GRID_RECALCULATION.md +4 -2
- package/docs/GRID_RECONCILE.md +14 -3
- package/docs/LIFECYCLE.md +11 -11
- package/docs/MPA_CREDIT_USAGE.md +22 -3
- package/docs/README.md +14 -3
- package/docs/WORKFLOW.md +1 -0
- package/docs/architecture.md +31 -19
- package/docs/developer_guide.md +6 -6
- package/market_adapter/README.md +12 -5
- package/modules/README.md +4 -1
- package/package.json +9 -9
- package/scripts/README.md +43 -38
- package/tests/README.md +25 -10
- package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
- package/dist/scripts/diagnose-kibana-candles.js +0 -82
- package/scripts/git-viewer.sh +0 -489
- package/scripts/postinstall.js +0 -16
|
@@ -1,90 +1,88 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
declare function shardKeyForTimestamp(tsMs: any): string;
|
|
9
|
-
declare function shardBoundsForKey(key: any): {
|
|
10
|
-
start: number;
|
|
11
|
-
end: number;
|
|
12
|
-
};
|
|
13
|
-
declare function shardKeysForRange(gteMs: any, lteMs: any): string[];
|
|
14
|
-
declare function shardPathFor(outPath: any, shardKey: any): string;
|
|
15
|
-
declare function readCacheChunk(chunkFile: any, requestKey: any, isMatch: (meta: any, requestKey: any) => boolean): {
|
|
16
|
-
candles: any;
|
|
17
|
-
fetchedAt: any;
|
|
18
|
-
file: any;
|
|
19
|
-
rangeGte: number | null;
|
|
20
|
-
rangeLte: number | null;
|
|
1
|
+
import { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges } from '../utils/month_shards.js';
|
|
2
|
+
import type { QueriedRange } from '../utils/month_shards.js';
|
|
3
|
+
type Candle = number[];
|
|
4
|
+
interface FileCover {
|
|
5
|
+
gte: number | null;
|
|
6
|
+
lte: number | null;
|
|
7
|
+
count: number;
|
|
21
8
|
queried: QueriedRange[];
|
|
22
|
-
}
|
|
23
|
-
|
|
9
|
+
}
|
|
10
|
+
interface GapRange {
|
|
24
11
|
gte: number;
|
|
25
12
|
lte: number;
|
|
26
|
-
|
|
27
|
-
|
|
13
|
+
hours: number;
|
|
14
|
+
}
|
|
15
|
+
interface BucketCache {
|
|
16
|
+
byTs: Map<number, Candle>;
|
|
28
17
|
files: number;
|
|
29
|
-
fileCover:
|
|
30
|
-
|
|
31
|
-
lte: number | null;
|
|
32
|
-
count: number;
|
|
33
|
-
queried: QueriedRange[];
|
|
34
|
-
}[];
|
|
35
|
-
shards: {
|
|
18
|
+
fileCover: FileCover[];
|
|
19
|
+
shards: Array<{
|
|
36
20
|
file: string;
|
|
37
21
|
shardKey: string;
|
|
38
|
-
candles:
|
|
22
|
+
candles: Candle[];
|
|
39
23
|
queried: QueriedRange[];
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
24
|
+
}>;
|
|
25
|
+
}
|
|
26
|
+
interface ShardMeta {
|
|
27
|
+
timeRange?: {
|
|
28
|
+
gte?: unknown;
|
|
29
|
+
lte?: unknown;
|
|
30
|
+
};
|
|
31
|
+
fetchedAt?: unknown;
|
|
32
|
+
queriedRanges?: unknown[];
|
|
33
|
+
[key: string]: unknown;
|
|
34
|
+
}
|
|
35
|
+
interface CacheChunk {
|
|
36
|
+
candles: Candle[];
|
|
37
|
+
fetchedAt: unknown;
|
|
38
|
+
file: string;
|
|
39
|
+
rangeGte: number | null;
|
|
40
|
+
rangeLte: number | null;
|
|
41
|
+
queried: QueriedRange[];
|
|
42
|
+
}
|
|
43
|
+
interface FetchRangeResult {
|
|
44
|
+
candles: Candle[];
|
|
45
|
+
complete?: boolean;
|
|
46
|
+
}
|
|
47
|
+
interface RunCachedWindowsOptions {
|
|
48
|
+
windows: Array<{
|
|
49
|
+
index: number;
|
|
50
|
+
gte: string;
|
|
51
|
+
lte: string;
|
|
52
|
+
}>;
|
|
53
|
+
outPath: string;
|
|
54
|
+
requestKey: unknown;
|
|
55
|
+
isMatch: (meta: unknown, requestKey: unknown) => boolean;
|
|
56
|
+
metaForWindow: (window: {
|
|
57
|
+
index: number;
|
|
58
|
+
gte: string;
|
|
59
|
+
lte: string;
|
|
60
|
+
}) => Record<string, unknown>;
|
|
61
|
+
fetchRange: (gteIso: string, lteIso: string, window: unknown, signal?: AbortSignal) => Promise<Candle[] | FetchRangeResult>;
|
|
62
|
+
bucketMs: number;
|
|
63
|
+
allowSubFetch?: boolean;
|
|
64
|
+
nowMs?: number;
|
|
65
|
+
fetchAttempts?: number;
|
|
66
|
+
fetchBackoffBaseMs?: number;
|
|
67
|
+
fetchTimeoutMs?: number;
|
|
68
|
+
onFetchRetry?: (info: {
|
|
69
|
+
attempt: number;
|
|
70
|
+
attempts: number;
|
|
71
|
+
backoffMs: number;
|
|
72
|
+
error: unknown;
|
|
73
|
+
gte: string;
|
|
74
|
+
lte: string;
|
|
75
|
+
}) => void;
|
|
76
|
+
}
|
|
77
|
+
declare const TAIL_REFRESH_HOURS = 48;
|
|
78
|
+
declare const TAIL_SETTLE_LAG_MS: number;
|
|
79
|
+
declare const GAP_SETTLE_HORIZON_MS: number;
|
|
80
|
+
declare const MAX_COVERAGE_SPANS = 64;
|
|
81
|
+
declare function readCacheChunk(chunkFile: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean): CacheChunk | null;
|
|
82
|
+
declare function loadBucketCache(outPath: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean, range?: {
|
|
73
83
|
gte: number;
|
|
74
84
|
lte: number;
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Change detection for shard writes: `have` already accounts for every span in
|
|
78
|
-
* `want`, in extent AND in verification time. Without the `at` half, a run
|
|
79
|
-
* whose only news is "these buckets were re-verified 1h later" would skip the
|
|
80
|
-
* write and the next run would fall back to the wide fixed refresh window.
|
|
81
|
-
* An unknown `at` on either side is never treated as fresh.
|
|
82
|
-
*/
|
|
83
|
-
declare function coverageSatisfied(have: QueriedRange[], want: QueriedRange[]): boolean;
|
|
84
|
-
/** Every recorded span across the loaded files, newest verification included. */
|
|
85
|
-
declare function allQueriedRanges(fileCover: {
|
|
86
|
-
queried?: QueriedRange[] | null;
|
|
87
|
-
}[]): QueriedRange[];
|
|
85
|
+
} | null): BucketCache;
|
|
88
86
|
/**
|
|
89
87
|
* How settled a window is, given the recorded coverage.
|
|
90
88
|
*
|
|
@@ -106,63 +104,30 @@ declare function settleCoverage(opts: {
|
|
|
106
104
|
hasTimestamps: boolean;
|
|
107
105
|
firstUnsettled: number | null;
|
|
108
106
|
};
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
* at the earliest of them. That is why the refresh is capped at
|
|
118
|
-
* TAIL_REFRESH_HOURS: the worst a dropped span can cost is the legacy 48h
|
|
119
|
-
* overlap, never weeks. The input is already sorted by extent, so the tail of
|
|
120
|
-
* the list is the newest.
|
|
121
|
-
*/
|
|
122
|
-
declare function compactCoverage(spans: QueriedRange[], maxSpans: number): QueriedRange[];
|
|
123
|
-
declare function buildFetchWindowsFromRange(timeRange: any, chunkMonths: any): any[];
|
|
124
|
-
declare function findMissingBucketRanges(gteMs: number, lteMs: number, bucketMs: number, haveTs: Set<number>): {
|
|
125
|
-
gte: number;
|
|
126
|
-
lte: number;
|
|
127
|
-
hours: number;
|
|
128
|
-
}[];
|
|
107
|
+
declare function buildFetchWindowsFromRange(timeRange: {
|
|
108
|
+
gte?: unknown;
|
|
109
|
+
lte?: unknown;
|
|
110
|
+
}, chunkMonths: number): Array<{
|
|
111
|
+
gte: string;
|
|
112
|
+
lte: string;
|
|
113
|
+
}>;
|
|
114
|
+
declare function findMissingBucketRanges(gteMs: number, lteMs: number, bucketMs: number, haveTs: Set<number>): GapRange[];
|
|
129
115
|
/**
|
|
130
116
|
* Merge gap ranges that sit closer together than `toleranceBuckets`, so a
|
|
131
117
|
* cluster of one-bucket holes costs one query instead of one per hole. Only
|
|
132
118
|
* the bounding span widens (the buckets in between get queried too — they are
|
|
133
119
|
* local anyway, so the over-claim is harmless), and input order is preserved.
|
|
134
120
|
*/
|
|
135
|
-
declare function mergeGapRanges(missing:
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
hours: number;
|
|
139
|
-
}[], bucketMs: number, toleranceBuckets: number): {
|
|
140
|
-
gte: number;
|
|
141
|
-
lte: number;
|
|
142
|
-
hours: number;
|
|
143
|
-
}[];
|
|
144
|
-
declare function pruneImmutableGaps(missing: {
|
|
145
|
-
gte: number;
|
|
146
|
-
lte: number;
|
|
147
|
-
hours: number;
|
|
148
|
-
}[], fileCover: {
|
|
149
|
-
gte: number | null;
|
|
150
|
-
lte: number | null;
|
|
151
|
-
count: number;
|
|
152
|
-
queried: QueriedRange[];
|
|
153
|
-
}[], nowMs?: number): {
|
|
154
|
-
gte: number;
|
|
155
|
-
lte: number;
|
|
156
|
-
hours: number;
|
|
157
|
-
}[];
|
|
158
|
-
declare function persistCacheChunk(chunkFile: any, meta: any, candles: any): void;
|
|
121
|
+
declare function mergeGapRanges(missing: GapRange[], bucketMs: number, toleranceBuckets: number): GapRange[];
|
|
122
|
+
declare function pruneImmutableGaps(missing: GapRange[], fileCover: FileCover[], nowMs?: number): GapRange[];
|
|
123
|
+
declare function persistCacheChunk(chunkFile: string, meta: ShardMeta, candles: Candle[]): void;
|
|
159
124
|
/**
|
|
160
125
|
* Reuse plan for one window: buckets already on disk + the ranges that still
|
|
161
126
|
* need querying. When `allowSubFetch` is false (data with cross-range state
|
|
162
127
|
* such as forward-filled crosses), the tail widening is skipped and the
|
|
163
128
|
* caller must take full-window fetches — reuse (zero queries) still applies.
|
|
164
129
|
*/
|
|
165
|
-
declare function planWindowReuse(localCache:
|
|
130
|
+
declare function planWindowReuse(localCache: BucketCache, opts: {
|
|
166
131
|
gteMs: number;
|
|
167
132
|
lteMs: number;
|
|
168
133
|
bucketMs: number;
|
|
@@ -170,12 +135,8 @@ declare function planWindowReuse(localCache: any, opts: {
|
|
|
170
135
|
allowSubFetch?: boolean;
|
|
171
136
|
nowMs?: number;
|
|
172
137
|
}): {
|
|
173
|
-
reusable:
|
|
174
|
-
missing:
|
|
175
|
-
gte: number;
|
|
176
|
-
lte: number;
|
|
177
|
-
hours: number;
|
|
178
|
-
}[];
|
|
138
|
+
reusable: Candle[];
|
|
139
|
+
missing: GapRange[];
|
|
179
140
|
missingHours: number;
|
|
180
141
|
windowHours: number;
|
|
181
142
|
inputsValid: boolean;
|
|
@@ -199,31 +160,7 @@ declare function formatWindowLine(unit: string, index: number, total: number, gt
|
|
|
199
160
|
* rewritten solely when it gains buckets or query coverage. Reuse-only runs
|
|
200
161
|
* touch nothing on disk.
|
|
201
162
|
*/
|
|
202
|
-
declare function runCachedWindows(opts:
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
requestKey: any;
|
|
206
|
-
isMatch: (meta: any, requestKey: any) => boolean;
|
|
207
|
-
metaForWindow: (window: any) => any;
|
|
208
|
-
fetchRange: (gteIso: string, lteIso: string, window: any, signal?: AbortSignal) => Promise<any[] | {
|
|
209
|
-
candles: any[];
|
|
210
|
-
complete?: boolean;
|
|
211
|
-
}>;
|
|
212
|
-
bucketMs: number;
|
|
213
|
-
allowSubFetch?: boolean;
|
|
214
|
-
nowMs?: number;
|
|
215
|
-
fetchAttempts?: number;
|
|
216
|
-
fetchBackoffBaseMs?: number;
|
|
217
|
-
fetchTimeoutMs?: number;
|
|
218
|
-
onFetchRetry?: (info: {
|
|
219
|
-
attempt: number;
|
|
220
|
-
attempts: number;
|
|
221
|
-
backoffMs: number;
|
|
222
|
-
error: any;
|
|
223
|
-
gte: string;
|
|
224
|
-
lte: string;
|
|
225
|
-
}) => void;
|
|
226
|
-
}): Promise<any[]>;
|
|
227
|
-
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, GAP_MERGE_TOLERANCE_BUCKETS, MAX_SUBFETCH_RANGES, MAX_SUBFETCH_SPAN_RATIO, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, cachedCandlesInRange, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
|
|
228
|
-
export type { QueriedRange };
|
|
163
|
+
declare function runCachedWindows(opts: RunCachedWindowsOptions): Promise<Candle[]>;
|
|
164
|
+
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
|
|
165
|
+
export type { QueriedRange } from '../utils/month_shards.js';
|
|
229
166
|
//# sourceMappingURL=window_cache.d.ts.map
|
|
@@ -27,16 +27,16 @@
|
|
|
27
27
|
* - `isMatch(meta, requestKey)` — same-pool/feed/interval/assets check,
|
|
28
28
|
* - `fetchRange(gteIso, lteIso)` — query one (sub-)range, gap-filled grid,
|
|
29
29
|
* - `metaForWindow(window)` — identity meta fields (source/feed/pool/...);
|
|
30
|
-
* the runner overrides timeRange with the shard bounds
|
|
31
|
-
* chunkIndex, which is meaningless for stable shards.
|
|
30
|
+
* the runner overrides timeRange with the shard bounds.
|
|
32
31
|
*
|
|
33
32
|
* Node-only (disk I/O via storage). Browser-safe code must not import this.
|
|
34
33
|
*/
|
|
35
34
|
import { path } from '../../modules/path_api.js';
|
|
36
35
|
import { getStorage } from '../../modules/storage/index.js';
|
|
37
36
|
import { writeJsonAtomic } from '../utils/atomic_write.js';
|
|
38
|
-
import { sleepMs } from '../../modules/utils/errors.js';
|
|
39
|
-
import { mergeCandles } from '../candle_utils.js';
|
|
37
|
+
import { sleepMs, getErrorName, getErrorMessage } from '../../modules/utils/errors.js';
|
|
38
|
+
import { mergeCandles, higherVolumeWins } from '../candle_utils.js';
|
|
39
|
+
import { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, shardKeyFromName, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges, clipRangeTo, } from '../utils/month_shards.js';
|
|
40
40
|
const storage = getStorage();
|
|
41
41
|
const { readJSON } = storage;
|
|
42
42
|
// Trailing overlap for the newest window. It is the refresh when the cache
|
|
@@ -87,49 +87,18 @@ const MAX_SUBFETCH_SPAN_RATIO = 0.5;
|
|
|
87
87
|
// current month accumulates.
|
|
88
88
|
const MAX_COVERAGE_SPANS = 64;
|
|
89
89
|
// ─── Month-shard naming ───────────────────────────────────────────────────────
|
|
90
|
-
// Shard key
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
function shardKeyForTimestamp(tsMs) {
|
|
94
|
-
const d = new Date(Number(tsMs));
|
|
95
|
-
return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
|
|
96
|
-
}
|
|
97
|
-
function shardBoundsForKey(key) {
|
|
98
|
-
const parts = String(key).split('-').map(Number);
|
|
99
|
-
const start = Date.UTC(parts[0], parts[1] - 1, 1);
|
|
100
|
-
const end = parts[1] === 12 ? Date.UTC(parts[0] + 1, 0, 1) : Date.UTC(parts[0], parts[1], 1);
|
|
101
|
-
return { start, end };
|
|
102
|
-
}
|
|
103
|
-
function shardKeysForRange(gteMs, lteMs) {
|
|
104
|
-
const keys = [];
|
|
105
|
-
let cursor = new Date(Date.UTC(new Date(Number(gteMs)).getUTCFullYear(), new Date(Number(gteMs)).getUTCMonth(), 1));
|
|
106
|
-
const last = new Date(Number(lteMs));
|
|
107
|
-
while (cursor <= last) {
|
|
108
|
-
keys.push(shardKeyForTimestamp(cursor.getTime()));
|
|
109
|
-
cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth() + 1, 1));
|
|
110
|
-
}
|
|
111
|
-
return keys;
|
|
112
|
-
}
|
|
113
|
-
function shardRangeOverlaps(shardKey, gteMs, lteMs) {
|
|
114
|
-
const { start, end } = shardBoundsForKey(shardKey);
|
|
115
|
-
return start <= lteMs && end > gteMs;
|
|
116
|
-
}
|
|
117
|
-
function shardPathFor(outPath, shardKey) {
|
|
118
|
-
const parsed = path.parse(outPath);
|
|
119
|
-
return path.join(parsed.dir, `${parsed.name}.shard_${shardKey}${parsed.ext}`);
|
|
120
|
-
}
|
|
90
|
+
// Shard key, bounds, path and coverage helpers live in
|
|
91
|
+
// `market_adapter/utils/month_shards.ts` so the candle and fills caches share
|
|
92
|
+
// one contract and cannot disagree on a month boundary.
|
|
121
93
|
function siblingCacheFiles(outPath) {
|
|
122
94
|
const resolved = path.resolve(outPath);
|
|
123
95
|
const parsed = path.parse(resolved);
|
|
124
96
|
if (!storage.exists(parsed.dir))
|
|
125
97
|
return [];
|
|
126
|
-
const shardPrefix = `${parsed.name}.shard_`;
|
|
127
98
|
const out = [];
|
|
128
99
|
for (const name of storage.readdir(parsed.dir)) {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
const key = name.slice(shardPrefix.length, name.length - parsed.ext.length);
|
|
132
|
-
if (!/^\d{4}-\d{2}$/.test(key))
|
|
100
|
+
const key = shardKeyFromName(name, parsed.name, parsed.ext);
|
|
101
|
+
if (!key)
|
|
133
102
|
continue;
|
|
134
103
|
out.push({ file: path.join(parsed.dir, name), shardKey: key });
|
|
135
104
|
}
|
|
@@ -144,7 +113,7 @@ function readCacheChunk(chunkFile, requestKey, isMatch) {
|
|
|
144
113
|
const meta = parsed?.meta || {};
|
|
145
114
|
if (!isMatch(meta, requestKey))
|
|
146
115
|
return null;
|
|
147
|
-
if (!Array.isArray(parsed.candles))
|
|
116
|
+
if (!parsed || !Array.isArray(parsed.candles))
|
|
148
117
|
return null;
|
|
149
118
|
const rangeGte = Date.parse(String(meta.timeRange?.gte || ''));
|
|
150
119
|
const rangeLte = Date.parse(String(meta.timeRange?.lte || ''));
|
|
@@ -188,11 +157,16 @@ function loadBucketCache(outPath, requestKey, isMatch, range) {
|
|
|
188
157
|
const shards = [];
|
|
189
158
|
let files = 0;
|
|
190
159
|
const scoped = range && Number.isFinite(range.gte) && Number.isFinite(range.lte);
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
160
|
+
// Scoped load: enumerate only the calendar-month shards that overlap the
|
|
161
|
+
// run and open exactly those — no directory listing, and out-of-range
|
|
162
|
+
// months are never even stat'd. Unscoped callers (no finite range) fall
|
|
163
|
+
// back to the full listing.
|
|
164
|
+
const entries = scoped
|
|
165
|
+
? shardKeysForRange(range.gte, range.lte)
|
|
166
|
+
.map((shardKey) => ({ file: shardPathFor(outPath, shardKey), shardKey }))
|
|
167
|
+
.filter((entry) => storage.exists(entry.file))
|
|
168
|
+
: siblingCacheFiles(outPath);
|
|
169
|
+
for (const entry of entries) {
|
|
196
170
|
const chunk = readCacheChunk(entry.file, requestKey, isMatch);
|
|
197
171
|
if (!chunk)
|
|
198
172
|
continue;
|
|
@@ -218,112 +192,15 @@ function cachedCandlesInRange(localCache, gteMs, lteMs) {
|
|
|
218
192
|
if (ts >= gteMs && ts <= lteMs)
|
|
219
193
|
out.push(c);
|
|
220
194
|
}
|
|
221
|
-
out.sort((a, b) => a[0] - b[0]);
|
|
222
|
-
return out;
|
|
223
|
-
}
|
|
224
|
-
/** Parse a verification timestamp: epoch ms or ISO. Anything else is unknown. */
|
|
225
|
-
function timestampMs(raw) {
|
|
226
|
-
// `null`/`undefined`/`''` mean "not recorded". They must NOT fall through
|
|
227
|
-
// to Number(): Number(null) === 0 is finite, which would read an unknown
|
|
228
|
-
// verification time as "verified at the epoch" — i.e. as maximally stale,
|
|
229
|
-
// silently disabling the tail refresh on every pre-`at` shard.
|
|
230
|
-
if (raw == null || raw === '')
|
|
231
|
-
return null;
|
|
232
|
-
if (typeof raw === 'string') {
|
|
233
|
-
const parsed = Date.parse(raw);
|
|
234
|
-
return Number.isFinite(parsed) ? parsed : null;
|
|
235
|
-
}
|
|
236
|
-
const at = Number(raw);
|
|
237
|
-
return Number.isFinite(at) ? at : null;
|
|
238
|
-
}
|
|
239
|
-
function verifiedAt(range) {
|
|
240
|
-
return timestampMs(range?.at);
|
|
241
|
-
}
|
|
242
|
-
/**
|
|
243
|
-
* Normalize recorded spans: sort, then merge overlapping / bucket-adjacent
|
|
244
|
-
* ones — but ONLY when they assert the same verification time. Merging spans
|
|
245
|
-
* with different `at` would have to pick one verdict for the whole union, and
|
|
246
|
-
* either pick loses (min → the merged span vouches for buckets the older
|
|
247
|
-
* query never saw; max → buckets verified long ago look freshly verified).
|
|
248
|
-
* Keeping them separate preserves exact per-bucket freshness, which is the
|
|
249
|
-
* whole point of `at`. Spans therefore accumulate ~1 per tail run, each a few
|
|
250
|
-
* dozen bytes — negligible next to the candles in the same file, and the
|
|
251
|
-
* same-`at` case (bulk fetches, and every pre-`at` shard) still collapses to
|
|
252
|
-
* one span as before.
|
|
253
|
-
*/
|
|
254
|
-
function unionQueriedRanges(ranges, bucketMs) {
|
|
255
|
-
const clean = (ranges || [])
|
|
256
|
-
.filter((q) => q && Number.isFinite(Number(q.gte)) && Number.isFinite(Number(q.lte)) && Number(q.lte) >= Number(q.gte))
|
|
257
|
-
.map((q) => ({ gte: Number(q.gte), lte: Number(q.lte), at: verifiedAt(q) }))
|
|
258
|
-
.sort((a, b) => a.gte - b.gte || a.lte - b.lte);
|
|
259
|
-
const merged = [];
|
|
260
|
-
const gap = Number.isFinite(Number(bucketMs)) && Number(bucketMs) > 0 ? Number(bucketMs) : 0;
|
|
261
|
-
for (const q of clean) {
|
|
262
|
-
const top = merged[merged.length - 1];
|
|
263
|
-
if (top && q.gte <= top.lte + gap && top.at === q.at) {
|
|
264
|
-
if (q.lte > top.lte)
|
|
265
|
-
top.lte = q.lte;
|
|
266
|
-
}
|
|
267
|
-
else {
|
|
268
|
-
merged.push({ gte: q.gte, lte: q.lte, at: q.at });
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
return merged;
|
|
272
|
-
}
|
|
273
|
-
function rangesCoveredBy(have, want) {
|
|
274
|
-
for (const w of want || []) {
|
|
275
|
-
let covered = false;
|
|
276
|
-
for (const h of have || []) {
|
|
277
|
-
if (h.gte <= w.gte && h.lte >= w.lte) {
|
|
278
|
-
covered = true;
|
|
279
|
-
break;
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
if (!covered)
|
|
283
|
-
return false;
|
|
284
|
-
}
|
|
285
|
-
return true;
|
|
286
|
-
}
|
|
287
|
-
/**
|
|
288
|
-
* Change detection for shard writes: `have` already accounts for every span in
|
|
289
|
-
* `want`, in extent AND in verification time. Without the `at` half, a run
|
|
290
|
-
* whose only news is "these buckets were re-verified 1h later" would skip the
|
|
291
|
-
* write and the next run would fall back to the wide fixed refresh window.
|
|
292
|
-
* An unknown `at` on either side is never treated as fresh.
|
|
293
|
-
*/
|
|
294
|
-
function coverageSatisfied(have, want) {
|
|
295
|
-
if (!rangesCoveredBy(have, want))
|
|
296
|
-
return false;
|
|
297
|
-
for (const w of want || []) {
|
|
298
|
-
const wAt = verifiedAt(w);
|
|
299
|
-
if (wAt == null)
|
|
300
|
-
continue;
|
|
301
|
-
let fresh = false;
|
|
302
|
-
for (const h of have || []) {
|
|
303
|
-
if (h.gte > w.gte || h.lte < w.lte)
|
|
304
|
-
continue;
|
|
305
|
-
const hAt = verifiedAt(h);
|
|
306
|
-
if (hAt != null && hAt >= wAt) {
|
|
307
|
-
fresh = true;
|
|
308
|
-
break;
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
if (!fresh)
|
|
312
|
-
return false;
|
|
313
|
-
}
|
|
314
|
-
return true;
|
|
315
|
-
}
|
|
316
|
-
/** Every recorded span across the loaded files, newest verification included. */
|
|
317
|
-
function allQueriedRanges(fileCover) {
|
|
318
|
-
const out = [];
|
|
319
|
-
for (const f of fileCover || []) {
|
|
320
|
-
for (const q of f?.queried || []) {
|
|
321
|
-
if (q && Number.isFinite(Number(q.gte)) && Number.isFinite(Number(q.lte)))
|
|
322
|
-
out.push(q);
|
|
323
|
-
}
|
|
324
|
-
}
|
|
195
|
+
out.sort((a, b) => Number(a[0]) - Number(b[0]));
|
|
325
196
|
return out;
|
|
326
197
|
}
|
|
198
|
+
// ─── Queried-range set ops ────────────────────────────────────────────────────
|
|
199
|
+
// Coverage provenance: normalize recorded spans (sort, merge overlapping or
|
|
200
|
+
// bucket-adjacent) so growth checks and absorption decisions are exact.
|
|
201
|
+
// A span also carries the time its covering query RAN (`at`), which is what
|
|
202
|
+
// makes the tail refresh incremental: extent says "we asked", `at` says "and
|
|
203
|
+
// the answer is still current".
|
|
327
204
|
/**
|
|
328
205
|
* How settled a window is, given the recorded coverage.
|
|
329
206
|
*
|
|
@@ -359,29 +236,6 @@ function settleCoverage(opts) {
|
|
|
359
236
|
}
|
|
360
237
|
return { hasTimestamps: true, firstUnsettled: null };
|
|
361
238
|
}
|
|
362
|
-
/**
|
|
363
|
-
* Bound the persisted span list. Because spans merge only when they assert
|
|
364
|
-
* the same `at`, a shard that is re-verified on every run gains one span per
|
|
365
|
-
* run and would grow without limit (and `settleCoverage` scans the list once
|
|
366
|
-
* per tail window). Dropping the OLDEST spans never invents emptiness, but it
|
|
367
|
-
* is not free: coverage is consulted by BOTH `pruneImmutableGaps` (to skip a
|
|
368
|
-
* query) and `settleCoverage` (to decide a bucket is fresh). A dropped span
|
|
369
|
-
* makes its buckets look unsettled, and the tail refresh would otherwise start
|
|
370
|
-
* at the earliest of them. That is why the refresh is capped at
|
|
371
|
-
* TAIL_REFRESH_HOURS: the worst a dropped span can cost is the legacy 48h
|
|
372
|
-
* overlap, never weeks. The input is already sorted by extent, so the tail of
|
|
373
|
-
* the list is the newest.
|
|
374
|
-
*/
|
|
375
|
-
function compactCoverage(spans, maxSpans) {
|
|
376
|
-
const list = spans || [];
|
|
377
|
-
const max = Number.isFinite(Number(maxSpans)) && Number(maxSpans) >= 1 ? Math.floor(Number(maxSpans)) : list.length;
|
|
378
|
-
return list.length <= max ? list : list.slice(list.length - max);
|
|
379
|
-
}
|
|
380
|
-
function clipRangeTo(q, gteMs, lteMs) {
|
|
381
|
-
const gte = Math.max(q.gte, gteMs);
|
|
382
|
-
const lte = Math.min(q.lte, lteMs);
|
|
383
|
-
return lte >= gte ? { gte, lte, at: verifiedAt(q) } : null;
|
|
384
|
-
}
|
|
385
239
|
function addUtcMonths(date, months) {
|
|
386
240
|
const result = new Date(date.getTime());
|
|
387
241
|
const day = result.getUTCDate();
|
|
@@ -391,6 +245,18 @@ function addUtcMonths(date, months) {
|
|
|
391
245
|
result.setUTCDate(Math.min(day, lastDay));
|
|
392
246
|
return result;
|
|
393
247
|
}
|
|
248
|
+
/** First instant of the UTC month after `date`. */
|
|
249
|
+
function nextUtcMonthStart(date) {
|
|
250
|
+
return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 1));
|
|
251
|
+
}
|
|
252
|
+
/** True when `date` sits exactly on a UTC month start (midnight, day 1). */
|
|
253
|
+
function isUtcMonthStart(date) {
|
|
254
|
+
return date.getUTCDate() === 1
|
|
255
|
+
&& date.getUTCHours() === 0
|
|
256
|
+
&& date.getUTCMinutes() === 0
|
|
257
|
+
&& date.getUTCSeconds() === 0
|
|
258
|
+
&& date.getUTCMilliseconds() === 0;
|
|
259
|
+
}
|
|
394
260
|
function normalizeDateInput(raw, label) {
|
|
395
261
|
const ms = Date.parse(String(raw || ''));
|
|
396
262
|
if (!Number.isFinite(ms)) {
|
|
@@ -404,10 +270,20 @@ function buildFetchWindowsFromRange(timeRange, chunkMonths) {
|
|
|
404
270
|
if (start >= end) {
|
|
405
271
|
throw new Error(`Invalid fetch range: ${start.toISOString()} must be earlier than ${end.toISOString()}`);
|
|
406
272
|
}
|
|
273
|
+
const step = Number.isFinite(Number(chunkMonths)) && Number(chunkMonths) >= 1
|
|
274
|
+
? Math.floor(Number(chunkMonths))
|
|
275
|
+
: 1;
|
|
276
|
+
// Interior window boundaries snap to UTC month starts — the same grid the
|
|
277
|
+
// storage shards use — so a window maps onto whole calendar months instead
|
|
278
|
+
// of straddling two of them. The requested start/end stay exact: only the
|
|
279
|
+
// first window can be short (a mid-month start), and the last is clipped
|
|
280
|
+
// to `end`.
|
|
407
281
|
const windows = [];
|
|
408
282
|
let cursor = start;
|
|
409
283
|
while (cursor < end) {
|
|
410
|
-
const next =
|
|
284
|
+
const next = isUtcMonthStart(cursor)
|
|
285
|
+
? addUtcMonths(cursor, step)
|
|
286
|
+
: nextUtcMonthStart(cursor);
|
|
411
287
|
const windowEnd = next < end ? next : end;
|
|
412
288
|
windows.push({
|
|
413
289
|
gte: cursor.toISOString(),
|
|
@@ -498,8 +374,8 @@ function persistCacheChunk(chunkFile, meta, candles) {
|
|
|
498
374
|
meta: {
|
|
499
375
|
...meta,
|
|
500
376
|
candleCount: candles.length,
|
|
501
|
-
firstTs: candles.length > 0 ? new Date(candles[0][0]).toISOString() : null,
|
|
502
|
-
lastTs: candles.length > 0 ? new Date(candles[candles.length - 1][0]).toISOString() : null,
|
|
377
|
+
firstTs: candles.length > 0 ? new Date(Number(candles[0][0])).toISOString() : null,
|
|
378
|
+
lastTs: candles.length > 0 ? new Date(Number(candles[candles.length - 1][0])).toISOString() : null,
|
|
503
379
|
},
|
|
504
380
|
candles,
|
|
505
381
|
};
|
|
@@ -629,7 +505,7 @@ async function fetchRangeWithRetry(fetchRange, opts) {
|
|
|
629
505
|
return { candles, attempts: attempt };
|
|
630
506
|
}
|
|
631
507
|
catch (err) {
|
|
632
|
-
lastErr = timedOut && (err
|
|
508
|
+
lastErr = timedOut && (getErrorName(err) === 'AbortError' || getErrorMessage(err) === timeoutMessage)
|
|
633
509
|
? new Error(timeoutMessage)
|
|
634
510
|
: err;
|
|
635
511
|
if (attempt < attempts) {
|
|
@@ -674,9 +550,6 @@ async function fetchRangeLogged(fetchRange, opts) {
|
|
|
674
550
|
console.log(formatWindowLine(opts.unit, opts.index, opts.total, opts.gte, opts.lte, `-> ${candles.length} candles (${((Date.now() - startMs) / 1000).toFixed(1)}s)${attemptNote}${partialNote}${note}`));
|
|
675
551
|
return { candles, complete };
|
|
676
552
|
}
|
|
677
|
-
function higherVolumeWins(existing, incoming) {
|
|
678
|
-
return incoming[5] > existing[5] ? incoming : existing;
|
|
679
|
-
}
|
|
680
553
|
function sortedCandles(byTs) {
|
|
681
554
|
return [...byTs.values()].sort((a, b) => Number(a[0]) - Number(b[0]));
|
|
682
555
|
}
|
|
@@ -849,7 +722,7 @@ async function runCachedWindows(opts) {
|
|
|
849
722
|
if (!fresh.complete)
|
|
850
723
|
windowComplete = false;
|
|
851
724
|
candles = reusable.length > 0
|
|
852
|
-
? mergeCandles(reusable, fresh.candles, { onCollision: higherVolumeWins }).filter((c) => Number(c[0]) >= gteMs && Number(c[0]) <= lteMs).sort((a, b) => a[0] - b[0])
|
|
725
|
+
? mergeCandles(reusable, fresh.candles, { onCollision: higherVolumeWins }).filter((c) => Number(c[0]) >= gteMs && Number(c[0]) <= lteMs).sort((a, b) => Number(a[0]) - Number(b[0]))
|
|
853
726
|
: fresh.candles;
|
|
854
727
|
// Full-window fetch. `at` is deliberately the WALL clock, never
|
|
855
728
|
// `nowMs`: `at` answers "when did this query run", so stamping a
|
|
@@ -872,6 +745,9 @@ async function runCachedWindows(opts) {
|
|
|
872
745
|
// else on disk is already current.
|
|
873
746
|
if (overall) {
|
|
874
747
|
const fetchedAt = new Date().toISOString();
|
|
748
|
+
// O(1) lookup of each shard's loaded candles for the write-on-change
|
|
749
|
+
// comparison below (instead of a linear scan per shard state).
|
|
750
|
+
const loadedShardCandles = new Map(localCache.shards.map((s) => [s.shardKey, s.candles]));
|
|
875
751
|
for (const state of shardStates.values()) {
|
|
876
752
|
const { start, end } = shardBoundsForKey(state.key);
|
|
877
753
|
// Fold every loaded bucket into its home shard. Presence on disk
|
|
@@ -884,7 +760,7 @@ async function runCachedWindows(opts) {
|
|
|
884
760
|
if (!prev || Number(c[5] || 0) > Number(prev[5] || 0))
|
|
885
761
|
state.candles.set(ts, c);
|
|
886
762
|
}
|
|
887
|
-
const before = sortedCandles(new Map((
|
|
763
|
+
const before = sortedCandles(new Map((loadedShardCandles.get(state.key) || [])
|
|
888
764
|
.filter((c) => Array.isArray(c) && Number.isFinite(Number(c[0])))
|
|
889
765
|
.map((c) => [Number(c[0]), c])));
|
|
890
766
|
const after = sortedCandles(state.candles);
|
|
@@ -903,7 +779,6 @@ async function runCachedWindows(opts) {
|
|
|
903
779
|
const meta = { ...metaForWindow(synthWindow), fetchedAt, queriedRanges: union };
|
|
904
780
|
meta.timeRange = { gte: synthWindow.gte, lte: synthWindow.lte };
|
|
905
781
|
meta.shard = state.key;
|
|
906
|
-
delete meta.chunkIndex;
|
|
907
782
|
persistCacheChunk(state.file, meta, after);
|
|
908
783
|
state.queried = union;
|
|
909
784
|
state.pendingQueried = [];
|
|
@@ -911,5 +786,5 @@ async function runCachedWindows(opts) {
|
|
|
911
786
|
}
|
|
912
787
|
return merged;
|
|
913
788
|
}
|
|
914
|
-
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS,
|
|
789
|
+
export { TAIL_REFRESH_HOURS, TAIL_SETTLE_LAG_MS, GAP_SETTLE_HORIZON_MS, MAX_COVERAGE_SPANS, shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, readCacheChunk, loadBucketCache, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, verifiedAt, timestampMs, allQueriedRanges, settleCoverage, buildFetchWindowsFromRange, findMissingBucketRanges, mergeGapRanges, pruneImmutableGaps, persistCacheChunk, planWindowReuse, formatWindowLine, runCachedWindows, };
|
|
915
790
|
//# sourceMappingURL=window_cache.js.map
|