dexbot 1.6.12 → 1.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -1
- package/README.md +2 -2
- package/analysis/README.md +9 -11
- package/analysis/ama_fitting/README.md +5 -3
- package/claw/README.md +1 -0
- package/claw/docs/AI_BOT_LIBRARY_API.md +3 -1
- package/claw/docs/DEXBOT2_TUNING_CHEAT_SHEET.md +4 -4
- package/claw/modules/claw_bridge.ts +0 -3
- package/claw/modules/claw_catalog.ts +8 -4
- package/claw/modules/claw_skill_md.ts +1 -0
- package/claw/modules/dexbot_profiles.ts +59 -15
- package/claw/modules/types.ts +2 -1
- package/claw/package.json +3 -3
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/claw/skills/margin-trading/SKILL.md +1 -1
- package/claw/skills/margin-trading/references/position-management.md +1 -1
- package/claw/tsconfig.json +2 -2
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +1 -14
- package/dist/analysis/ama_fitting/fetch_lp_candles.js +51 -41
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.d.ts +1 -24
- package/dist/analysis/ama_fitting/generate_unified_comparison_chart.js +36 -18
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js +18 -66
- package/dist/analysis/analyze_args.d.ts +12 -0
- package/dist/analysis/analyze_args.js +20 -0
- package/dist/analysis/bot_fitting/backtest_ama_sweep.d.ts +1 -16
- package/dist/analysis/bot_fitting/backtest_ama_sweep.js +20 -101
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js +26 -92
- package/dist/analysis/bot_fitting/backtest_lookback_drawdown.js +19 -34
- package/dist/analysis/bot_fitting/shared_utils.d.ts +71 -3
- package/dist/analysis/bot_fitting/shared_utils.js +124 -8
- package/dist/analysis/chart_ui.d.ts +30 -1
- package/dist/analysis/chart_ui.js +37 -1
- package/dist/analysis/fills_cache.d.ts +3 -23
- package/dist/analysis/fills_cache.js +3 -84
- package/dist/analysis/fills_source.d.ts +31 -1
- package/dist/analysis/fills_source.js +69 -1
- package/dist/analysis/grid_correction_check.d.ts +1 -23
- package/dist/analysis/grid_correction_check.js +15 -58
- package/dist/analysis/math_utils.d.ts +85 -1
- package/dist/analysis/math_utils.js +116 -1
- package/dist/analysis/price_sources.d.ts +22 -1
- package/dist/analysis/price_sources.js +23 -0
- package/dist/analysis/resolve_source.js +31 -10
- package/dist/analysis/trade_profitability.d.ts +6 -0
- package/dist/analysis/trade_profitability.js +23 -97
- package/dist/analysis/tradingview/analyze_tradingview.d.ts +1 -17
- package/dist/analysis/tradingview/analyze_tradingview.js +1 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +27 -40
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.d.ts +1 -101
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +22 -80
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +11 -37
- package/dist/analysis/trend_detection/volatility_chart_generator.js +9 -10
- package/dist/analysis/worker_pool.d.ts +38 -0
- package/dist/analysis/worker_pool.js +67 -0
- package/dist/dexbot.js +67 -48
- package/dist/market_adapter/ama_signal_runner.js +21 -54
- package/dist/market_adapter/candle_utils.d.ts +7 -1
- package/dist/market_adapter/candle_utils.js +11 -1
- package/dist/market_adapter/cli_args.d.ts +22 -0
- package/dist/market_adapter/cli_args.js +30 -0
- package/dist/market_adapter/core/config_normalizers.js +11 -6
- package/dist/market_adapter/core/kibana_candles.d.ts +3 -1
- package/dist/market_adapter/core/kibana_candles.js +3 -2
- package/dist/market_adapter/core/market_adapter_service.d.ts +85 -85
- package/dist/market_adapter/core/market_adapter_service.js +93 -158
- package/dist/market_adapter/core/signals/hurst_analyzer.d.ts +1 -6
- package/dist/market_adapter/core/signals/hurst_analyzer.js +1 -1
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.d.ts +1 -11
- package/dist/market_adapter/core/signals/permutation_entropy_analyzer.js +1 -1
- package/dist/market_adapter/inputs/fetch_book_data.js +0 -1
- package/dist/market_adapter/inputs/fetch_cex_synthetic_data.js +63 -107
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +8 -1
- package/dist/market_adapter/inputs/fetch_lp_data.js +25 -7
- package/dist/market_adapter/inputs/kibana_feed_source.js +4 -26
- package/dist/market_adapter/inputs/window_cache.d.ts +4 -72
- package/dist/market_adapter/inputs/window_cache.js +52 -177
- package/dist/market_adapter/interval_utils.d.ts +11 -1
- package/dist/market_adapter/interval_utils.js +16 -1
- package/dist/market_adapter/lp_chart_runner.d.ts +3 -0
- package/dist/market_adapter/lp_chart_runner.js +37 -19
- package/dist/market_adapter/market_adapter.js +27 -72
- package/dist/market_adapter/utils/data_discovery.d.ts +45 -1
- package/dist/market_adapter/utils/data_discovery.js +220 -27
- package/dist/market_adapter/utils/month_shards.d.ts +84 -0
- package/dist/market_adapter/utils/month_shards.js +237 -0
- package/dist/modules/account_bots.d.ts +8 -6
- package/dist/modules/account_bots.js +112 -131
- package/dist/modules/account_orders.d.ts +49 -4
- package/dist/modules/account_orders.js +114 -49
- package/dist/modules/bitshares-native/chain_client.d.ts +1 -1
- package/dist/modules/bitshares-native/chain_client.js +14 -13
- package/dist/modules/bitshares-native/crypto/ecc_core.d.ts +1 -2
- package/dist/modules/bitshares-native/crypto/ecc_core.js +1 -1
- package/dist/modules/bitshares-native/resolvers.js +22 -25
- package/dist/modules/bitshares-native/serial/chain_constants.d.ts +1 -2
- package/dist/modules/bitshares-native/serial/chain_constants.js +1 -2
- package/dist/modules/bitshares-native/serial/types.js +22 -50
- package/dist/modules/bitshares-native/signing_client.d.ts +1 -1
- package/dist/modules/bot_defaults.d.ts +18 -3
- package/dist/modules/bot_defaults.js +34 -8
- package/dist/modules/bot_settings.d.ts +5 -1
- package/dist/modules/bot_settings.js +55 -21
- package/dist/modules/broadcast_failure.d.ts +2 -1
- package/dist/modules/chain_orders.d.ts +1 -1
- package/dist/modules/chain_orders.js +51 -96
- package/dist/modules/cli_start_onboarding.d.ts +2 -1
- package/dist/modules/config.js +6 -1
- package/dist/modules/constants.d.ts +14 -6
- package/dist/modules/constants.js +32 -6
- package/dist/modules/cr_planner.d.ts +1 -1
- package/dist/modules/cr_planner.js +1 -5
- package/dist/modules/credit_pricing.d.ts +2 -1
- package/dist/modules/credit_pricing.js +36 -40
- package/dist/modules/credit_runtime.d.ts +7 -0
- package/dist/modules/credit_runtime.js +45 -73
- package/dist/modules/daemon_node_health.d.ts +4 -3
- package/dist/modules/dexbot_class.d.ts +33 -17
- package/dist/modules/dexbot_class.js +113 -61
- package/dist/modules/dexbot_cow_runtime.d.ts +62 -181
- package/dist/modules/dexbot_cow_runtime.js +380 -69
- package/dist/modules/dexbot_credential_client.d.ts +1 -23
- package/dist/modules/dexbot_credential_client.js +2 -3
- package/dist/modules/dexbot_fill_runtime.d.ts +2 -3
- package/dist/modules/dexbot_fill_runtime.js +17 -7
- package/dist/modules/dexbot_maintenance_runtime.d.ts +36 -81
- package/dist/modules/dexbot_maintenance_runtime.js +137 -133
- package/dist/modules/dexbot_startup_runtime.d.ts +0 -1
- package/dist/modules/dexbot_startup_runtime.js +25 -14
- package/dist/modules/dexbot_state_recovery.js +16 -0
- package/dist/modules/general_settings.d.ts +1 -1
- package/dist/modules/graceful_shutdown.js +17 -11
- package/dist/modules/key_store.d.ts +0 -5
- package/dist/modules/key_store.js +1 -2
- package/dist/modules/lazy_shim.d.ts +11 -0
- package/dist/modules/lazy_shim.js +16 -0
- package/dist/modules/node_connect_policy.d.ts +3 -2
- package/dist/modules/node_failure_ledger.d.ts +6 -5
- package/dist/modules/order/accounting.js +11 -11
- package/dist/modules/order/export.js +2 -2
- package/dist/modules/order/format.d.ts +26 -3
- package/dist/modules/order/format.js +33 -3
- package/dist/modules/order/genesis_policy.d.ts +4 -4
- package/dist/modules/order/genesis_policy.js +6 -3
- package/dist/modules/order/grid.d.ts +26 -12
- package/dist/modules/order/grid.js +468 -80
- package/dist/modules/order/grid_reconcile.d.ts +1 -1
- package/dist/modules/order/grid_reconcile.js +5 -5
- package/dist/modules/order/grid_reconcile_internal.js +48 -40
- package/dist/modules/order/index.d.ts +35 -8
- package/dist/modules/order/logger.d.ts +1 -8
- package/dist/modules/order/logger.js +2 -11
- package/dist/modules/order/manager.d.ts +18 -15
- package/dist/modules/order/manager.js +91 -74
- package/dist/modules/order/strategy.d.ts +2 -2
- package/dist/modules/order/strategy.js +51 -21
- package/dist/modules/order/sync_engine.js +4 -0
- package/dist/modules/order/utils/math.d.ts +3 -3
- package/dist/modules/order/utils/math.js +38 -27
- package/dist/modules/order/utils/order.d.ts +58 -58
- package/dist/modules/order/utils/order.js +158 -137
- package/dist/modules/order/utils/system.d.ts +202 -35
- package/dist/modules/order/utils/system.js +409 -31
- package/dist/modules/order/utils/validate.js +28 -6
- package/dist/modules/order/utils/withPoolRef.d.ts +2 -1
- package/dist/modules/paths.js +32 -34
- package/dist/modules/runtime_settings.d.ts +3 -5
- package/dist/modules/runtime_settings.js +3 -3
- package/dist/modules/settings_merge.d.ts +2 -2
- package/dist/modules/socket_json_client.d.ts +2 -1
- package/dist/modules/types.d.ts +89 -30
- package/dist/modules/utils/errors.js +1 -3
- package/dist/modules/version_notice.d.ts +11 -10
- package/dist/pm2.d.ts +0 -1
- package/dist/pm2.js +56 -35
- package/dist/scripts/analyze-git.js +2 -15
- package/dist/scripts/analyze-orders.d.ts +1 -1
- package/dist/scripts/analyze-orders.js +4 -4
- package/dist/scripts/chart_command.js +2 -2
- package/dist/scripts/check_any.d.ts +4 -0
- package/dist/scripts/check_any.js +53 -21
- package/dist/scripts/diagnose-pool-history.js +1 -6
- package/dist/scripts/divergence-calc.js +6 -6
- package/dist/scripts/run-tests.js +0 -1
- package/dist/scripts/sync-version.js +0 -10
- package/dist/scripts/validate_bots.d.ts +1 -1
- package/dist/scripts/validate_bots.js +3 -3
- package/dist/unlock.js +7 -2
- package/docs/BITSHARES_ONBOARDING.md +8 -12
- package/docs/COPY_ON_WRITE_MASTER_PLAN.md +2 -3
- package/docs/COW_INVARIANTS.md +6 -6
- package/docs/DEXBOT_COMPARISON.md +7 -7
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +8 -6
- package/docs/GRID_PRICE_INVARIANT.md +3 -3
- package/docs/GRID_RECALCULATION.md +4 -1
- package/docs/GRID_RECONCILE.md +4 -4
- package/docs/LIFECYCLE.md +11 -11
- package/docs/LOGGING.md +14 -0
- package/docs/MPA_CREDIT_USAGE.md +1 -1
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +12 -4
- package/docs/README.md +3 -3
- package/docs/WORKFLOW.md +1 -0
- package/docs/architecture.md +25 -16
- package/docs/developer_guide.md +7 -2
- package/market_adapter/README.md +6 -4
- package/package.json +6 -9
- package/scripts/README.md +43 -38
- package/tests/README.md +26 -11
- package/dist/scripts/diagnose-kibana-candles.d.ts +0 -3
- package/dist/scripts/diagnose-kibana-candles.js +0 -82
- package/scripts/git-viewer.sh +0 -489
- package/scripts/postinstall.js +0 -16
|
@@ -7,7 +7,7 @@ import { calculateAMA } from './core/strategies/ama.js';
|
|
|
7
7
|
import { MARKET_ADAPTER } from '../modules/constants.js';
|
|
8
8
|
import { generateHTML } from './lp_chart_core.js';
|
|
9
9
|
import { loadStrategiesForLpChart } from './lp_chart_strategy_loader.js';
|
|
10
|
-
import { findLatestLpData } from './utils/data_discovery.js';
|
|
10
|
+
import { findLatestLpData, loadLatestLpSeries, loadLpSeriesFromPath } from './utils/data_discovery.js';
|
|
11
11
|
import { PATHS } from '../modules/paths.js';
|
|
12
12
|
/**
|
|
13
13
|
* LP CHART RUNNER — Shared LP chart orchestration
|
|
@@ -30,7 +30,7 @@ import { PATHS } from '../modules/paths.js';
|
|
|
30
30
|
* - fetch/export of LP data (`market_adapter/inputs/fetch_lp_data.ts`)
|
|
31
31
|
*/
|
|
32
32
|
const storage = getStorage();
|
|
33
|
-
const { ensureDir
|
|
33
|
+
const { ensureDir } = storage;
|
|
34
34
|
const ANALYSIS_CHARTS_DIR = PATHS.ANALYSIS.CHARTS_DIR;
|
|
35
35
|
const AMA_PROFILES_FILE = PATHS.PROFILES.MARKET_PROFILES_JSON;
|
|
36
36
|
const DEFAULT_COMPARISON_COLORS = ['#26a69a', '#fb8c00', '#5c9ee6', '#ef5350'];
|
|
@@ -100,7 +100,11 @@ function normalizeLpCandle(candle, index) {
|
|
|
100
100
|
throw new Error(`Invalid candle row at index ${index}: unsupported format`);
|
|
101
101
|
}
|
|
102
102
|
function resolveLpDataFile(dataFile) {
|
|
103
|
-
|
|
103
|
+
// Explicit path wins. Otherwise resolve through the assembling loader so a
|
|
104
|
+
// shard-only cache yields a representative source path instead of null.
|
|
105
|
+
const resolved = dataFile
|
|
106
|
+
? path.resolve(dataFile)
|
|
107
|
+
: (loadLatestLpSeries()?.path ?? findLatestLpData());
|
|
104
108
|
if (!resolved) {
|
|
105
109
|
throw new Error('No LP data file found. Use --data <path> or run the fetch step first.');
|
|
106
110
|
}
|
|
@@ -109,16 +113,9 @@ function resolveLpDataFile(dataFile) {
|
|
|
109
113
|
}
|
|
110
114
|
return resolved;
|
|
111
115
|
}
|
|
112
|
-
function
|
|
113
|
-
const resolved = resolveLpDataFile(dataFile);
|
|
114
|
-
const raw = readJSON(resolved);
|
|
115
|
-
const meta = (raw.meta ?? null);
|
|
116
|
-
const candles = Array.isArray(raw?.candles) ? raw.candles : (Array.isArray(raw) ? raw : []);
|
|
117
|
-
if (!Array.isArray(candles) || candles.length === 0) {
|
|
118
|
-
throw new Error('No candles in data file.');
|
|
119
|
-
}
|
|
116
|
+
function bundleFrom(dataFile, meta, candles) {
|
|
120
117
|
return {
|
|
121
|
-
dataFile
|
|
118
|
+
dataFile,
|
|
122
119
|
meta,
|
|
123
120
|
candleArrays: candles.map((c, index) => normalizeLpCandle(c, index)),
|
|
124
121
|
candleObjects: candles.map((c, index) => {
|
|
@@ -134,6 +131,24 @@ function loadLpDataFile(dataFile) {
|
|
|
134
131
|
}),
|
|
135
132
|
};
|
|
136
133
|
}
|
|
134
|
+
function loadLpDataFile(dataFile) {
|
|
135
|
+
// Explicit path: read the file as before. Auto-discovery: assemble the
|
|
136
|
+
// complete series (merging month shards when they are the newest source),
|
|
137
|
+
// so a shard-only cache charts the whole history, not one month.
|
|
138
|
+
if (!dataFile) {
|
|
139
|
+
const series = loadLatestLpSeries();
|
|
140
|
+
if (!series || series.candles.length === 0) {
|
|
141
|
+
throw new Error('No LP data found. Use --data <path> or run the fetch step first.');
|
|
142
|
+
}
|
|
143
|
+
return bundleFrom(series.path, series.meta, series.candles);
|
|
144
|
+
}
|
|
145
|
+
const resolved = resolveLpDataFile(dataFile);
|
|
146
|
+
const series = loadLpSeriesFromPath(resolved);
|
|
147
|
+
if (!series || series.candles.length === 0) {
|
|
148
|
+
throw new Error('No candles in data file.');
|
|
149
|
+
}
|
|
150
|
+
return bundleFrom(series.path, series.meta, series.candles);
|
|
151
|
+
}
|
|
137
152
|
function openInBrowser(filePath) {
|
|
138
153
|
const url = `file://${filePath}`;
|
|
139
154
|
const cmd = process.platform === 'darwin'
|
|
@@ -203,7 +218,7 @@ function writeChartHtml({ meta, candleArrays, amaResults, outFile }) {
|
|
|
203
218
|
}
|
|
204
219
|
function generateMarketLpChart(options = {}) {
|
|
205
220
|
const logger = options.logger ?? console;
|
|
206
|
-
const { dataFile, meta, candleArrays } = loadLpDataFile(options.dataFile);
|
|
221
|
+
const { dataFile, meta, candleArrays } = options.dataBundle ?? loadLpDataFile(options.dataFile);
|
|
207
222
|
logger.log(`Reading: ${path.relative(process.cwd(), dataFile)}`);
|
|
208
223
|
const poolLabel = meta?.pool ? `pool ${meta.pool}` : `${meta?.assetA?.symbol || '?'}\/${meta?.assetB?.symbol || '?'}`;
|
|
209
224
|
logger.log(` ${candleArrays.length} candles · ${poolLabel}`);
|
|
@@ -241,7 +256,7 @@ function generateMarketLpChart(options = {}) {
|
|
|
241
256
|
}
|
|
242
257
|
function generateComparisonLpChart(options = {}) {
|
|
243
258
|
const logger = options.logger ?? console;
|
|
244
|
-
const { dataFile, meta, candleArrays, candleObjects } = loadLpDataFile(options.dataFile);
|
|
259
|
+
const { dataFile, meta, candleArrays, candleObjects } = options.dataBundle ?? loadLpDataFile(options.dataFile);
|
|
245
260
|
const defaultStrategies = options.defaultStrategies ?? DEFAULT_COMPARISON_STRATEGIES;
|
|
246
261
|
const strategies = loadStrategiesForLpChart({
|
|
247
262
|
dataFile,
|
|
@@ -284,23 +299,26 @@ function generateComparisonLpChart(options = {}) {
|
|
|
284
299
|
}
|
|
285
300
|
function generateLpChartBundle(options = {}) {
|
|
286
301
|
const logger = options.logger ?? console;
|
|
287
|
-
|
|
288
|
-
|
|
302
|
+
// Resolve/assemble ONCE, then hand the same loaded bundle to both charts —
|
|
303
|
+
// auto-discovery over shards would otherwise re-read and re-merge the whole
|
|
304
|
+
// family for each sub-chart.
|
|
305
|
+
const bundle = options.dataBundle ?? loadLpDataFile(options.dataFile);
|
|
306
|
+
logger.log(`Generating LP charts from ${path.relative(process.cwd(), bundle.dataFile)}`);
|
|
289
307
|
const marketChart = generateMarketLpChart({
|
|
290
|
-
|
|
308
|
+
dataBundle: bundle,
|
|
291
309
|
noOpen: options.noOpen,
|
|
292
310
|
logger,
|
|
293
311
|
profilesFile: options.profilesFile,
|
|
294
312
|
});
|
|
295
313
|
const comparisonChart = generateComparisonLpChart({
|
|
296
|
-
|
|
314
|
+
dataBundle: bundle,
|
|
297
315
|
noOpen: options.openComparison === true ? !!options.noOpen : true,
|
|
298
316
|
logger,
|
|
299
317
|
defaultStrategies: options.defaultStrategies ?? DEFAULT_COMPARISON_STRATEGIES,
|
|
300
318
|
profilesFile: options.comparisonProfilesFile ?? null,
|
|
301
319
|
});
|
|
302
320
|
return {
|
|
303
|
-
dataFile,
|
|
321
|
+
dataFile: bundle.dataFile,
|
|
304
322
|
marketChart,
|
|
305
323
|
comparisonChart,
|
|
306
324
|
};
|
|
@@ -12,7 +12,7 @@ import { normalizeBotEntry } from '../modules/bot_settings.js';
|
|
|
12
12
|
import * as kibanaSource from './inputs/kibana_source.js';
|
|
13
13
|
import * as kibanaMarketSource from './core/kibana_market_candles.js';
|
|
14
14
|
import { tradesToCandles, detectMissingCandleTimestamps, fillCandleGaps, detectStaleTail, pruneStaleTail, mergeCandles } from './candle_utils.js';
|
|
15
|
-
import { toIntervalLabel, bucketStartMs, latestClosedBucketStartMs } from './interval_utils.js';
|
|
15
|
+
import { toIntervalLabel, bucketStartMs, latestClosedBucketStartMs, parseChainTimeToMs } from './interval_utils.js';
|
|
16
16
|
import { loadMarketProfiles } from '../analysis/tradingview/tradingview_uplot_chart_generator.js';
|
|
17
17
|
import { candleFileForBot as candleFilePathForLabel } from '../analysis/bot_key_utils.js';
|
|
18
18
|
import { writeJsonAtomic } from './utils/atomic_write.js';
|
|
@@ -27,6 +27,7 @@ import { resetMarketAdapterWhitelistCache, isBotWhitelisted, isBotDynamicWeightW
|
|
|
27
27
|
import { normalizeAssetSymbol, normalizeMarketSource, resolveBotContext, isExactPair, isSamePair, isExactPairIds, isSamePairIds, getBitsharesClient, setBitsharesClientForTests, } from './utils/chain.js';
|
|
28
28
|
import { normalizeNativeMarketHistoryCandles, } from './utils/native_history.js';
|
|
29
29
|
import { formatLogPercent, buildWeightSummary, buildDynamicWeightInputsLog, buildDynamicWeightTuningLog, buildAsymmetricBoundsLog, buildStartupDefaultsLog, } from './log_format.js';
|
|
30
|
+
import { walkCliArgs } from './cli_args.js';
|
|
30
31
|
/**
|
|
31
32
|
* PRICE ADAPTER — standalone or auto-launched by dexbot runtime.
|
|
32
33
|
*
|
|
@@ -616,73 +617,33 @@ function parseArgs() {
|
|
|
616
617
|
const provided = {
|
|
617
618
|
deltaThresholdPercent: false,
|
|
618
619
|
};
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
620
|
+
walkCliArgs(args, {
|
|
621
|
+
onHelp: () => {
|
|
622
|
+
printHelp();
|
|
623
|
+
process.exit(0);
|
|
624
|
+
},
|
|
625
|
+
flag: {
|
|
626
|
+
'--once': () => { cfg.once = true; },
|
|
627
|
+
'--metricsJson': () => { cfg.metricsJson = true; },
|
|
628
|
+
'--quiet': () => { cfg.quiet = true; },
|
|
629
|
+
'--dryRun': () => { cfg.dryRun = true; },
|
|
630
|
+
'--whitelist-all': () => { cfg.whitelistAll = true; },
|
|
631
|
+
},
|
|
632
|
+
value: {
|
|
633
|
+
'--pollSeconds': (v) => { cfg.pollSeconds = Number(v); },
|
|
634
|
+
'--deltaPercent': (v) => {
|
|
631
635
|
cfg.deltaThresholdPercent = Number(v);
|
|
632
636
|
provided.deltaThresholdPercent = true;
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
i++;
|
|
644
|
-
break;
|
|
645
|
-
case '--maxStaleHours':
|
|
646
|
-
cfg.maxStaleHours = Number(v);
|
|
647
|
-
i++;
|
|
648
|
-
break;
|
|
649
|
-
case '--sourceRetries':
|
|
650
|
-
cfg.sourceRetries = Number(v);
|
|
651
|
-
i++;
|
|
652
|
-
break;
|
|
653
|
-
case '--retryDelayMs':
|
|
654
|
-
cfg.retryDelayMs = Number(v);
|
|
655
|
-
i++;
|
|
656
|
-
break;
|
|
657
|
-
case '--metricsJson':
|
|
658
|
-
cfg.metricsJson = true;
|
|
659
|
-
break;
|
|
660
|
-
case '--quiet':
|
|
661
|
-
cfg.quiet = true;
|
|
662
|
-
break;
|
|
663
|
-
case '--dryRun':
|
|
664
|
-
cfg.dryRun = true;
|
|
665
|
-
break;
|
|
666
|
-
case '--whitelist-all':
|
|
667
|
-
cfg.whitelistAll = true;
|
|
668
|
-
break;
|
|
669
|
-
case '--maxPages':
|
|
670
|
-
cfg.maxPages = Number(v);
|
|
671
|
-
i++;
|
|
672
|
-
break;
|
|
673
|
-
case '--pageLimit':
|
|
674
|
-
cfg.pageLimit = Number(v);
|
|
675
|
-
i++;
|
|
676
|
-
break;
|
|
677
|
-
case '--help':
|
|
678
|
-
case '-h':
|
|
679
|
-
printHelp();
|
|
680
|
-
process.exit(0);
|
|
681
|
-
break;
|
|
682
|
-
default:
|
|
683
|
-
throw new Error(`Unknown argument: ${a}`);
|
|
684
|
-
}
|
|
685
|
-
}
|
|
637
|
+
},
|
|
638
|
+
'--bootstrapHours': (v) => { cfg.bootstrapLookbackHours = Number(v); },
|
|
639
|
+
'--nativeBackfillHours': (v) => { cfg.nativeBackfillHours = Number(v); },
|
|
640
|
+
'--maxStaleHours': (v) => { cfg.maxStaleHours = Number(v); },
|
|
641
|
+
'--sourceRetries': (v) => { cfg.sourceRetries = Number(v); },
|
|
642
|
+
'--retryDelayMs': (v) => { cfg.retryDelayMs = Number(v); },
|
|
643
|
+
'--maxPages': (v) => { cfg.maxPages = Number(v); },
|
|
644
|
+
'--pageLimit': (v) => { cfg.pageLimit = Number(v); },
|
|
645
|
+
},
|
|
646
|
+
});
|
|
686
647
|
const merged = applyRuntimeDefaultsFromGeneralSettings(cfg, provided);
|
|
687
648
|
return validateConfig(merged);
|
|
688
649
|
}
|
|
@@ -808,12 +769,6 @@ function loadJson(filePath, defaultValue) {
|
|
|
808
769
|
function saveJson(filePath, data) {
|
|
809
770
|
writeJsonAtomic(filePath, data);
|
|
810
771
|
}
|
|
811
|
-
function parseChainTimeToMs(timeStr) {
|
|
812
|
-
if (!timeStr)
|
|
813
|
-
return Number.NaN;
|
|
814
|
-
const s = String(timeStr);
|
|
815
|
-
return Date.parse(s.endsWith('Z') ? s : `${s}Z`);
|
|
816
|
-
}
|
|
817
772
|
function candleFileForBot(botKey, intervalSeconds = RUNTIME_DEFAULTS.intervalSeconds) {
|
|
818
773
|
const label = intervalSeconds === RUNTIME_DEFAULTS.intervalSeconds
|
|
819
774
|
? RUNTIME_DEFAULTS.intervalLabel
|
|
@@ -1,6 +1,50 @@
|
|
|
1
1
|
interface FindLatestLpDataOptions {
|
|
2
2
|
dataDir?: string;
|
|
3
3
|
}
|
|
4
|
+
/**
|
|
5
|
+
* Result of auto-discovering the newest LP series. `candles` is always the
|
|
6
|
+
* complete series, whether it came from one whole-history export
|
|
7
|
+
* (`assembled: false`) or was merged across month shards (`assembled: true`).
|
|
8
|
+
*/
|
|
9
|
+
interface LpSeriesResult {
|
|
10
|
+
path: string;
|
|
11
|
+
meta: Record<string, unknown> | null;
|
|
12
|
+
candles: unknown[];
|
|
13
|
+
assembled: boolean;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Whole-history-export discovery only: returns the newest `lp_pool_*.json` that
|
|
17
|
+
* is a complete series, or null. Month shards and legacy `.chunk_*` files are
|
|
18
|
+
* excluded — a single shard is a slice, and (because the active shard is
|
|
19
|
+
* rewritten every run) by mtime it would otherwise always win. Use
|
|
20
|
+
* `loadLatestLpSeries` when a shard-only cache should still yield a series.
|
|
21
|
+
*/
|
|
4
22
|
declare function findLatestLpData(options?: FindLatestLpDataOptions): string | null;
|
|
5
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Auto-discover and load the newest LP series.
|
|
25
|
+
*
|
|
26
|
+
* Month shards are the cache and a whole-history export is a derived snapshot
|
|
27
|
+
* of the same shard family (both writers, `fetch_lp_data` and
|
|
28
|
+
* `fetch_lp_candles`, regenerate the export from the assembled shards after
|
|
29
|
+
* fetching). The two therefore agree in extent, and the ordinary mtime race
|
|
30
|
+
* decides — an export produced by an external tool can still be newer and win.
|
|
31
|
+
* A shard-only directory is assembled; an export-only directory is read as-is.
|
|
32
|
+
*/
|
|
33
|
+
declare function loadLatestLpSeries(options?: FindLatestLpDataOptions): LpSeriesResult | null;
|
|
34
|
+
/**
|
|
35
|
+
* Assemble the whole series for a base path directly from its month shards,
|
|
36
|
+
* regardless of any sibling whole-history export. Writers use this to derive
|
|
37
|
+
* the export from the cache, so the monolithic file can never become a
|
|
38
|
+
* narrower or staler snapshot than the shards beside it.
|
|
39
|
+
*/
|
|
40
|
+
declare function loadLpSeriesFromBasePath(basePath: string): LpSeriesResult | null;
|
|
41
|
+
/**
|
|
42
|
+
* Load one explicit LP path. A month shard is transparently upgraded to its
|
|
43
|
+
* whole family: pointing `--data` at `.shard_2026-10.json` charts the full
|
|
44
|
+
* cached history, never a silent one-month slice. A whole-history export (or
|
|
45
|
+
* a lone shard whose siblings are missing) is read as-is.
|
|
46
|
+
*/
|
|
47
|
+
declare function loadLpSeriesFromPath(filePath: string): LpSeriesResult | null;
|
|
48
|
+
export { findLatestLpData, loadLatestLpSeries, loadLpSeriesFromPath, loadLpSeriesFromBasePath };
|
|
49
|
+
export type { LpSeriesResult, FindLatestLpDataOptions };
|
|
6
50
|
//# sourceMappingURL=data_discovery.d.ts.map
|
|
@@ -2,47 +2,240 @@
|
|
|
2
2
|
import { path } from '../../modules/path_api.js';
|
|
3
3
|
import { getStorage } from '../../modules/storage/index.js';
|
|
4
4
|
import { PATHS } from '../../modules/paths.js';
|
|
5
|
+
import { mergeCandles, higherVolumeWins } from '../candle_utils.js';
|
|
6
|
+
import { shardKeyFromName, shardStemFromName } from './month_shards.js';
|
|
5
7
|
const storage = getStorage();
|
|
6
|
-
function
|
|
7
|
-
|
|
8
|
+
function isLpJsonName(name) {
|
|
9
|
+
return name.startsWith('lp_pool_') && name.endsWith('.json');
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* All shard files for one base path (`<base>.shard_YYYY-MM.json`), read from
|
|
13
|
+
* disk. Empty when the base is not a shard-addressed series.
|
|
14
|
+
*/
|
|
15
|
+
function readShardFamily(basePath) {
|
|
16
|
+
const resolved = path.resolve(basePath);
|
|
17
|
+
const parsed = path.parse(resolved);
|
|
18
|
+
if (!storage.exists(parsed.dir))
|
|
19
|
+
return [];
|
|
8
20
|
const out = [];
|
|
9
|
-
|
|
21
|
+
for (const name of storage.readdir(parsed.dir)) {
|
|
22
|
+
if (!shardKeyFromName(name, parsed.name, parsed.ext))
|
|
23
|
+
continue;
|
|
24
|
+
const full = path.join(parsed.dir, name);
|
|
25
|
+
try {
|
|
26
|
+
const info = storage.stat(full);
|
|
27
|
+
if (info.isFile())
|
|
28
|
+
out.push({ path: full, mtime: info.mtimeMs });
|
|
29
|
+
}
|
|
30
|
+
catch (_) { /* unreadable sibling: skip it, keep the rest */ }
|
|
31
|
+
}
|
|
32
|
+
return out.sort((a, b) => (a.path < b.path ? -1 : 1));
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Merge an ordered shard family into one series. `sourcePath` is the file the
|
|
36
|
+
* caller should report (the newest shard, or the explicitly requested one).
|
|
37
|
+
*/
|
|
38
|
+
function assembleLpShards(files, sourcePath) {
|
|
39
|
+
let merged = [];
|
|
40
|
+
let meta = null;
|
|
41
|
+
for (const file of files) {
|
|
42
|
+
const raw = storage.readJSON(file.path);
|
|
43
|
+
if (!meta)
|
|
44
|
+
meta = metaOf(raw);
|
|
45
|
+
merged = mergeCandles(merged, rawCandlesOf(raw), { onCollision: higherVolumeWins });
|
|
46
|
+
}
|
|
47
|
+
if (merged.length === 0)
|
|
10
48
|
return null;
|
|
11
|
-
const
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
49
|
+
const firstTs = Number(merged[0][0]);
|
|
50
|
+
const lastTs = Number(merged[merged.length - 1][0]);
|
|
51
|
+
// A shard's meta describes one month; publish the assembled extent instead
|
|
52
|
+
// and drop per-shard bookkeeping that would misrepresent a whole series.
|
|
53
|
+
const mergedMeta = {
|
|
54
|
+
...(meta || {}),
|
|
55
|
+
candleCount: merged.length,
|
|
56
|
+
firstTs: new Date(firstTs).toISOString(),
|
|
57
|
+
lastTs: new Date(lastTs).toISOString(),
|
|
58
|
+
timeRange: { gte: new Date(firstTs).toISOString(), lte: new Date(lastTs).toISOString() },
|
|
59
|
+
assembledFrom: 'month-shards',
|
|
60
|
+
};
|
|
61
|
+
delete mergedMeta.shard;
|
|
62
|
+
delete mergedMeta.queriedRanges;
|
|
63
|
+
return { path: sourcePath, meta: mergedMeta, candles: merged, assembled: true };
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Scan the LP data dir for whole-history exports and month-shard families.
|
|
67
|
+
*
|
|
68
|
+
* The on-disk layout is fixed at two levels (`<dataDir>/<pair>/<file>`), so the
|
|
69
|
+
* scan is deliberately NOT recursive: it stats each direct child once and only
|
|
70
|
+
* descends one level into pair folders. Legacy run-relative `.chunk_*` files
|
|
71
|
+
* are ignored — nothing reads them and they are not a series.
|
|
72
|
+
*/
|
|
73
|
+
function scanLpDataDir(dataDir) {
|
|
74
|
+
const whole = [];
|
|
75
|
+
const shardFamilies = new Map();
|
|
76
|
+
if (!storage.exists(dataDir))
|
|
77
|
+
return { whole, shardFamilies };
|
|
78
|
+
const consider = (full, stat) => {
|
|
79
|
+
const name = path.basename(full);
|
|
80
|
+
if (!isLpJsonName(name) || name.includes('.chunk_'))
|
|
81
|
+
return;
|
|
82
|
+
let info = stat;
|
|
83
|
+
if (!info) {
|
|
84
|
+
try {
|
|
85
|
+
info = storage.stat(full);
|
|
86
|
+
}
|
|
87
|
+
catch (_) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (!info.isFile())
|
|
92
|
+
return;
|
|
93
|
+
const stem = shardStemFromName(name, path.extname(name));
|
|
94
|
+
if (!stem) {
|
|
95
|
+
whole.push({ path: full, mtime: info.mtimeMs });
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const key = path.join(path.dirname(full), stem);
|
|
99
|
+
let family = shardFamilies.get(key);
|
|
100
|
+
if (!family) {
|
|
101
|
+
family = { stem, files: [], maxMtime: 0 };
|
|
102
|
+
shardFamilies.set(key, family);
|
|
103
|
+
}
|
|
104
|
+
family.files.push({ path: full, mtime: info.mtimeMs });
|
|
105
|
+
if (info.mtimeMs > family.maxMtime)
|
|
106
|
+
family.maxMtime = info.mtimeMs;
|
|
107
|
+
};
|
|
108
|
+
for (const name of storage.readdir(dataDir)) {
|
|
109
|
+
const full = path.join(dataDir, name);
|
|
110
|
+
let stat = null;
|
|
15
111
|
try {
|
|
16
|
-
|
|
112
|
+
stat = storage.stat(full);
|
|
17
113
|
}
|
|
18
114
|
catch (_) {
|
|
19
|
-
// Directory vanished or is unreadable — skip it instead of
|
|
20
|
-
// aborting the whole scan.
|
|
21
115
|
continue;
|
|
22
116
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
let stat = null;
|
|
117
|
+
if (stat.isDirectory()) {
|
|
118
|
+
let names = [];
|
|
26
119
|
try {
|
|
27
|
-
|
|
120
|
+
names = storage.readdir(full);
|
|
28
121
|
}
|
|
29
122
|
catch (_) {
|
|
30
|
-
// Dangling symlink or race-deleted entry — skip the entry.
|
|
31
|
-
continue;
|
|
32
|
-
}
|
|
33
|
-
if (stat.isDirectory()) {
|
|
34
|
-
stack.push(full);
|
|
35
123
|
continue;
|
|
36
124
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
125
|
+
for (const inner of names)
|
|
126
|
+
consider(path.join(full, inner));
|
|
127
|
+
}
|
|
128
|
+
else if (stat.isFile()) {
|
|
129
|
+
consider(full, stat);
|
|
42
130
|
}
|
|
43
131
|
}
|
|
44
|
-
|
|
45
|
-
|
|
132
|
+
return { whole, shardFamilies };
|
|
133
|
+
}
|
|
134
|
+
function newestOf(files) {
|
|
135
|
+
let best = null;
|
|
136
|
+
for (const f of files)
|
|
137
|
+
if (!best || f.mtime > best.mtime)
|
|
138
|
+
best = f;
|
|
139
|
+
return best;
|
|
140
|
+
}
|
|
141
|
+
function newestFamily(families) {
|
|
142
|
+
let best = null;
|
|
143
|
+
for (const fam of families.values())
|
|
144
|
+
if (!best || fam.maxMtime > best.maxMtime)
|
|
145
|
+
best = fam;
|
|
146
|
+
return best;
|
|
147
|
+
}
|
|
148
|
+
function rawCandlesOf(raw) {
|
|
149
|
+
if (Array.isArray(raw))
|
|
150
|
+
return raw;
|
|
151
|
+
const r = raw;
|
|
152
|
+
if (r && Array.isArray(r.candles))
|
|
153
|
+
return r.candles;
|
|
154
|
+
if (r && Array.isArray(r.data))
|
|
155
|
+
return r.data;
|
|
156
|
+
return [];
|
|
157
|
+
}
|
|
158
|
+
function metaOf(raw) {
|
|
159
|
+
if (Array.isArray(raw))
|
|
160
|
+
return null;
|
|
161
|
+
const meta = raw?.meta;
|
|
162
|
+
return meta && typeof meta === 'object' ? meta : null;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Whole-history-export discovery only: returns the newest `lp_pool_*.json` that
|
|
166
|
+
* is a complete series, or null. Month shards and legacy `.chunk_*` files are
|
|
167
|
+
* excluded — a single shard is a slice, and (because the active shard is
|
|
168
|
+
* rewritten every run) by mtime it would otherwise always win. Use
|
|
169
|
+
* `loadLatestLpSeries` when a shard-only cache should still yield a series.
|
|
170
|
+
*/
|
|
171
|
+
function findLatestLpData(options = {}) {
|
|
172
|
+
const dataDir = options.dataDir ? path.resolve(options.dataDir) : PATHS.MARKET_ADAPTER.LP_DATA_DIR;
|
|
173
|
+
const { whole } = scanLpDataDir(dataDir);
|
|
174
|
+
return newestOf(whole)?.path ?? null;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Auto-discover and load the newest LP series.
|
|
178
|
+
*
|
|
179
|
+
* Month shards are the cache and a whole-history export is a derived snapshot
|
|
180
|
+
* of the same shard family (both writers, `fetch_lp_data` and
|
|
181
|
+
* `fetch_lp_candles`, regenerate the export from the assembled shards after
|
|
182
|
+
* fetching). The two therefore agree in extent, and the ordinary mtime race
|
|
183
|
+
* decides — an export produced by an external tool can still be newer and win.
|
|
184
|
+
* A shard-only directory is assembled; an export-only directory is read as-is.
|
|
185
|
+
*/
|
|
186
|
+
function loadLatestLpSeries(options = {}) {
|
|
187
|
+
const dataDir = options.dataDir ? path.resolve(options.dataDir) : PATHS.MARKET_ADAPTER.LP_DATA_DIR;
|
|
188
|
+
const { whole, shardFamilies } = scanLpDataDir(dataDir);
|
|
189
|
+
const newestWhole = newestOf(whole);
|
|
190
|
+
const family = newestFamily(shardFamilies);
|
|
191
|
+
// Whole-history export wins ties/races; only a strictly newer shard family
|
|
192
|
+
// forces the merge path.
|
|
193
|
+
if (newestWhole && (!family || newestWhole.mtime >= family.maxMtime)) {
|
|
194
|
+
const raw = storage.readJSON(newestWhole.path);
|
|
195
|
+
const candles = rawCandlesOf(raw);
|
|
196
|
+
if (candles.length === 0)
|
|
197
|
+
return null;
|
|
198
|
+
return { path: newestWhole.path, meta: metaOf(raw), candles, assembled: false };
|
|
199
|
+
}
|
|
200
|
+
if (!family)
|
|
201
|
+
return null;
|
|
202
|
+
const ordered = [...family.files].sort((a, b) => (a.path < b.path ? -1 : 1));
|
|
203
|
+
return assembleLpShards(ordered, ordered[ordered.length - 1].path);
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Assemble the whole series for a base path directly from its month shards,
|
|
207
|
+
* regardless of any sibling whole-history export. Writers use this to derive
|
|
208
|
+
* the export from the cache, so the monolithic file can never become a
|
|
209
|
+
* narrower or staler snapshot than the shards beside it.
|
|
210
|
+
*/
|
|
211
|
+
function loadLpSeriesFromBasePath(basePath) {
|
|
212
|
+
const resolved = path.resolve(basePath);
|
|
213
|
+
const family = readShardFamily(resolved);
|
|
214
|
+
if (family.length === 0)
|
|
215
|
+
return null;
|
|
216
|
+
return assembleLpShards(family, family[family.length - 1].path);
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Load one explicit LP path. A month shard is transparently upgraded to its
|
|
220
|
+
* whole family: pointing `--data` at `.shard_2026-10.json` charts the full
|
|
221
|
+
* cached history, never a silent one-month slice. A whole-history export (or
|
|
222
|
+
* a lone shard whose siblings are missing) is read as-is.
|
|
223
|
+
*/
|
|
224
|
+
function loadLpSeriesFromPath(filePath) {
|
|
225
|
+
const resolved = path.resolve(filePath);
|
|
226
|
+
const parsed = path.parse(resolved);
|
|
227
|
+
const stem = shardStemFromName(parsed.base, parsed.ext);
|
|
228
|
+
if (stem) {
|
|
229
|
+
const family = readShardFamily(path.join(parsed.dir, `${stem}${parsed.ext}`));
|
|
230
|
+
const assembled = assembleLpShards(family, resolved);
|
|
231
|
+
if (assembled)
|
|
232
|
+
return assembled;
|
|
233
|
+
}
|
|
234
|
+
const raw = storage.readJSON(resolved);
|
|
235
|
+
const candles = rawCandlesOf(raw);
|
|
236
|
+
if (candles.length === 0)
|
|
237
|
+
return null;
|
|
238
|
+
return { path: resolved, meta: metaOf(raw), candles, assembled: false };
|
|
46
239
|
}
|
|
47
|
-
export { findLatestLpData };
|
|
240
|
+
export { findLatestLpData, loadLatestLpSeries, loadLpSeriesFromPath, loadLpSeriesFromBasePath };
|
|
48
241
|
//# sourceMappingURL=data_discovery.js.map
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/** A span that was actually queried, with the time the query ran. */
|
|
2
|
+
type QueriedRange = {
|
|
3
|
+
gte: number;
|
|
4
|
+
lte: number;
|
|
5
|
+
at: number | null;
|
|
6
|
+
};
|
|
7
|
+
/** A plain inclusive ms range. */
|
|
8
|
+
type Span = {
|
|
9
|
+
gte: number;
|
|
10
|
+
lte: number;
|
|
11
|
+
};
|
|
12
|
+
declare function shardKeyForTimestamp(tsMs: unknown): string;
|
|
13
|
+
declare function shardBoundsForKey(key: unknown): {
|
|
14
|
+
start: number;
|
|
15
|
+
end: number;
|
|
16
|
+
};
|
|
17
|
+
declare function shardKeysForRange(gteMs: unknown, lteMs: unknown): string[];
|
|
18
|
+
declare function shardPathFor(basePath: string, shardKey: string): string;
|
|
19
|
+
/**
|
|
20
|
+
* If `name` is a shard file for `baseName`, return its `YYYY-MM` key;
|
|
21
|
+
* otherwise null. Guards the `sibling`/family scans against stray files.
|
|
22
|
+
*/
|
|
23
|
+
declare function shardKeyFromName(name: string, baseName: string, ext?: string): string | null;
|
|
24
|
+
/**
|
|
25
|
+
* The base stem of a shard filename, or null when `name` is not a shard.
|
|
26
|
+
* `lp_pool_133_1h.shard_2026-10.json` → `lp_pool_133_1h`. Callers that must
|
|
27
|
+
* group or upgrade a family use this instead of re-deriving the marker regex.
|
|
28
|
+
*/
|
|
29
|
+
declare function shardStemFromName(name: string, ext?: string): string | null;
|
|
30
|
+
/** Parse a verification timestamp: epoch ms or ISO. Anything else is unknown. */
|
|
31
|
+
declare function timestampMs(raw: unknown): number | null;
|
|
32
|
+
declare function verifiedAt(range: unknown): number | null;
|
|
33
|
+
/**
|
|
34
|
+
* Normalize recorded spans: sort, then merge overlapping / bucket-adjacent
|
|
35
|
+
* ones — but ONLY when they assert the same verification time. Merging spans
|
|
36
|
+
* with different `at` would have to pick one verdict for the whole union, and
|
|
37
|
+
* either pick loses (min → the merged span vouches for buckets the older
|
|
38
|
+
* query never saw; max → buckets verified long ago look freshly verified).
|
|
39
|
+
* Keeping them separate preserves exact per-bucket freshness. The same-`at`
|
|
40
|
+
* case (bulk fetches, and every pre-`at` shard) collapses to one span.
|
|
41
|
+
*
|
|
42
|
+
* `bucketMs` widens adjacency: `0` merges only overlaps, `1` merges adjacent
|
|
43
|
+
* integer-ms spans (fills), a bucket size merges spans a bucket apart (candles).
|
|
44
|
+
*/
|
|
45
|
+
declare function unionQueriedRanges(ranges: {
|
|
46
|
+
gte: number;
|
|
47
|
+
lte: number;
|
|
48
|
+
at?: number | null;
|
|
49
|
+
}[], bucketMs?: number): QueriedRange[];
|
|
50
|
+
/**
|
|
51
|
+
* Bound the persisted span list. Because spans merge only when they assert the
|
|
52
|
+
* same `at`, a shard that is re-verified on every run gains one span per run
|
|
53
|
+
* and would otherwise grow without limit. Dropping the OLDEST spans never
|
|
54
|
+
* invents emptiness. The input is already sorted by extent, so the tail is
|
|
55
|
+
* newest.
|
|
56
|
+
*/
|
|
57
|
+
declare function compactCoverage(spans: QueriedRange[], maxSpans: number): QueriedRange[];
|
|
58
|
+
declare function rangesCoveredBy(have: {
|
|
59
|
+
gte: number;
|
|
60
|
+
lte: number;
|
|
61
|
+
}[], want: {
|
|
62
|
+
gte: number;
|
|
63
|
+
lte: number;
|
|
64
|
+
}[]): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Change detection for shard writes: `have` already accounts for every span in
|
|
67
|
+
* `want`, in extent AND in verification time. Without the `at` half, a run
|
|
68
|
+
* whose only news is "these buckets were re-verified 1h later" would skip the
|
|
69
|
+
* write and the next run would fall back to the wide fixed refresh window.
|
|
70
|
+
* An unknown `at` on either side is never treated as fresh.
|
|
71
|
+
*/
|
|
72
|
+
declare function coverageSatisfied(have: QueriedRange[], want: QueriedRange[]): boolean;
|
|
73
|
+
/** Every recorded span across the loaded files, newest verification included. */
|
|
74
|
+
declare function allQueriedRanges(fileCover: {
|
|
75
|
+
queried?: QueriedRange[] | null;
|
|
76
|
+
}[]): QueriedRange[];
|
|
77
|
+
declare function clipRangeTo(q: QueriedRange, gteMs: number, lteMs: number): QueriedRange | null;
|
|
78
|
+
/** Sort and merge overlapping / adjacent (integer-ms) spans. */
|
|
79
|
+
declare function mergeSpans(spans: Span[]): Span[];
|
|
80
|
+
/** `span` minus the union of `covered` (both as inclusive ms ranges). */
|
|
81
|
+
declare function subtractRanges(span: Span, covered: Span[]): Span[];
|
|
82
|
+
export { shardKeyForTimestamp, shardBoundsForKey, shardKeysForRange, shardPathFor, shardKeyFromName, shardStemFromName, timestampMs, verifiedAt, unionQueriedRanges, compactCoverage, rangesCoveredBy, coverageSatisfied, allQueriedRanges, clipRangeTo, mergeSpans, subtractRanges, };
|
|
83
|
+
export type { QueriedRange, Span };
|
|
84
|
+
//# sourceMappingURL=month_shards.d.ts.map
|