dexbot 1.6.15 → 1.6.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +4 -4
- package/analysis/tradingview/README.md +2 -2
- package/analysis/trend_detection/DYNAMIC_WEIGHT_RESEARCH.md +1 -1
- package/claw/docs/DEXBOT2_TUNING_CHEAT_SHEET.md +3 -2
- package/claw/modules/chain_queries.ts +3 -3
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/ama_fitting/analyze_lambda_vs_slow.js +5 -3
- package/dist/analysis/ama_fitting/fetch_lp_candles.js +8 -1
- package/dist/analysis/ama_fitting/optimizer_high_resolution.d.ts +50 -1
- package/dist/analysis/ama_fitting/optimizer_high_resolution.js +33 -7
- package/dist/analysis/bot_fitting/backtest_bot_fitting.d.ts +8 -6
- package/dist/analysis/bot_fitting/backtest_bot_fitting.js +8 -10
- package/dist/analysis/bot_key_utils.js +9 -7
- package/dist/analysis/chart_utils.d.ts +9 -2
- package/dist/analysis/chart_utils.js +51 -1
- package/dist/analysis/tradingview/grid_reset_sim.js +1 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.d.ts +2 -1
- package/dist/analysis/tradingview/tradingview_uplot_chart_generator.js +23 -9
- package/dist/analysis/trend_detection/backtest_ama_slope_huber.js +1 -1
- package/dist/analysis/trend_detection/dynamic_weight_chart_generator.js +21 -2
- package/dist/analysis/trend_detection/kalman_chart_generator.js +1 -0
- package/dist/analysis/trend_detection/regime_chart_generator.js +1 -1
- package/dist/analysis/trend_detection/volatility_chart_generator.js +1 -0
- package/dist/dexbot.js +89 -138
- package/dist/market_adapter/inputs/fetch_book_data.js +36 -12
- package/dist/market_adapter/inputs/fetch_lp_data.d.ts +6 -0
- package/dist/market_adapter/inputs/fetch_lp_data.js +65 -19
- package/dist/market_adapter/inputs/kibana_feed_source.js +45 -12
- package/dist/market_adapter/inputs/window_cache.d.ts +14 -1
- package/dist/market_adapter/inputs/window_cache.js +53 -17
- package/dist/market_adapter/lp_chart_core.js +1 -0
- package/dist/market_adapter/utils/chain.js +4 -3
- package/dist/market_adapter/utils/pair_orientation.d.ts +60 -0
- package/dist/market_adapter/utils/pair_orientation.js +142 -0
- package/dist/modules/account_bots.d.ts +40 -3
- package/dist/modules/account_bots.js +77 -37
- package/dist/modules/account_orders.d.ts +4 -1
- package/dist/modules/account_orders.js +4 -1
- package/dist/modules/bitshares-native/resolvers.js +3 -3
- package/dist/modules/bitshares_client.d.ts +12 -10
- package/dist/modules/bitshares_client.js +12 -10
- package/dist/modules/bot_settings.js +3 -3
- package/dist/modules/chain_keys.d.ts +6 -2
- package/dist/modules/chain_keys.js +5 -1
- package/dist/modules/chain_orders.d.ts +6 -2
- package/dist/modules/chain_orders.js +6 -2
- package/dist/modules/constants.d.ts +11 -0
- package/dist/modules/constants.js +41 -7
- package/dist/modules/credit_pricing.d.ts +20 -1
- package/dist/modules/credit_pricing.js +23 -1
- package/dist/modules/credit_runtime.js +33 -16
- package/dist/modules/dexbot_class.d.ts +6 -2
- package/dist/modules/dexbot_class.js +8 -3
- package/dist/modules/dexbot_cow_runtime.d.ts +28 -37
- package/dist/modules/dexbot_cow_runtime.js +47 -66
- package/dist/modules/dexbot_maintenance_runtime.js +68 -10
- package/dist/modules/graceful_shutdown.d.ts +6 -2
- package/dist/modules/graceful_shutdown.js +6 -2
- package/dist/modules/launcher/bot_supervisor.js +4 -0
- package/dist/modules/launcher/credential_daemon.d.ts +14 -2
- package/dist/modules/launcher/credential_daemon.js +60 -26
- package/dist/modules/launcher/launch_modes.js +6 -3
- package/dist/modules/launcher/monolithic_runtime.d.ts +39 -1
- package/dist/modules/launcher/monolithic_runtime.js +60 -1
- package/dist/modules/launcher/runtime_mode.d.ts +79 -0
- package/dist/modules/launcher/runtime_mode.js +223 -0
- package/dist/modules/market_adapter_whitelist.d.ts +8 -0
- package/dist/modules/market_adapter_whitelist.js +8 -0
- package/dist/modules/order/accounting.d.ts +15 -1
- package/dist/modules/order/accounting.js +15 -1
- package/dist/modules/order/async_lock.d.ts +9 -3
- package/dist/modules/order/async_lock.js +9 -3
- package/dist/modules/order/export.d.ts +0 -5
- package/dist/modules/order/export.js +0 -5
- package/dist/modules/order/format.d.ts +6 -4
- package/dist/modules/order/format.js +6 -4
- package/dist/modules/order/grid.d.ts +36 -18
- package/dist/modules/order/grid.js +186 -33
- package/dist/modules/order/index.d.ts +16 -1
- package/dist/modules/order/index.js +1 -1
- package/dist/modules/order/logger_state.d.ts +1 -1
- package/dist/modules/order/logger_state.js +1 -1
- package/dist/modules/order/manager.d.ts +30 -7
- package/dist/modules/order/manager.js +108 -12
- package/dist/modules/order/strategy.d.ts +5 -0
- package/dist/modules/order/strategy.js +23 -2
- package/dist/modules/order/sync_engine.d.ts +26 -11
- package/dist/modules/order/sync_engine.js +26 -11
- package/dist/modules/order/utils/math.d.ts +100 -22
- package/dist/modules/order/utils/math.js +106 -22
- package/dist/modules/order/utils/order.d.ts +77 -20
- package/dist/modules/order/utils/order.js +125 -27
- package/dist/modules/order/utils/system.d.ts +107 -11
- package/dist/modules/order/utils/system.js +176 -11
- package/dist/modules/order/utils/validate.d.ts +8 -3
- package/dist/modules/order/utils/validate.js +10 -3
- package/dist/modules/types.d.ts +35 -2
- package/dist/modules/utils/asset_symbols.d.ts +5 -5
- package/dist/modules/utils/asset_symbols.js +5 -5
- package/dist/modules/utils/errors.d.ts +2 -1
- package/dist/modules/utils/errors.js +2 -1
- package/dist/scripts/analyze-credit.js +64 -23
- package/dist/scripts/bts_distribution.d.ts +66 -0
- package/dist/scripts/bts_distribution.js +2102 -0
- package/dist/scripts/rich_list.d.ts +32 -0
- package/dist/scripts/rich_list.js +16 -8
- package/dist/scripts/token_distribution.d.ts +45 -0
- package/dist/scripts/token_distribution.js +1870 -0
- package/dist/unlock.js +91 -42
- package/docs/BITSHARES_ONBOARDING.md +7 -0
- package/docs/COW_INVARIANTS.md +7 -1
- package/docs/DEXBOT_COMPARISON.md +5 -5
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +5 -3
- package/docs/GRID_PRICE_INVARIANT.md +6 -0
- package/docs/GRID_RECALCULATION.md +18 -6
- package/docs/LIFECYCLE.md +8 -8
- package/docs/LOGGING.md +9 -0
- package/docs/MPA_CREDIT_USAGE.md +3 -2
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +121 -4
- package/docs/README.md +1 -1
- package/docs/WORKFLOW.md +5 -5
- package/docs/architecture.md +98 -1
- package/market_adapter/README.md +10 -3
- package/package.json +5 -1
- package/scripts/README.md +85 -148
- package/tests/README.md +29 -0
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
import { path } from '../../modules/path_api.js';
|
|
29
29
|
import { getStorage } from '../../modules/storage/index.js';
|
|
30
30
|
import * as kibanaSource from './kibana_source.js';
|
|
31
|
-
import { toIntervalLabel
|
|
31
|
+
import { toIntervalLabel } from '../interval_utils.js';
|
|
32
32
|
import { parseJsonWithComments } from '../../modules/order/utils/system.js';
|
|
33
33
|
import { MARKET_ADAPTER } from '../../modules/constants.js';
|
|
34
34
|
import { normalizePoolId, resolveAsset, findPoolByAssets } from '../utils/chain.js';
|
|
@@ -36,6 +36,7 @@ import { writeJsonAtomic } from '../utils/atomic_write.js';
|
|
|
36
36
|
import { buildFetchWindowsFromRange, formatWindowLine, runCachedWindows, } from './window_cache.js';
|
|
37
37
|
import { PATHS } from '../../modules/paths.js';
|
|
38
38
|
import { loadLpSeriesFromBasePath } from '../utils/data_discovery.js';
|
|
39
|
+
import { canonicalPair, pairFolderSlug, sameAssets, swappedAssets, inverseSiblingOutPath, invertCandleSeries } from '../utils/pair_orientation.js';
|
|
39
40
|
import { shardStemFromName } from '../utils/month_shards.js';
|
|
40
41
|
import * as bitsharesClient from '../../modules/bitshares_client.js';
|
|
41
42
|
import { getErrorMessage, resolveSeamMs } from '../../modules/utils/errors.js';
|
|
@@ -135,13 +136,14 @@ function parseArgs() {
|
|
|
135
136
|
return { poolId, botName, precA, precB, config };
|
|
136
137
|
}
|
|
137
138
|
// ─── Output path ──────────────────────────────────────────────────────────────
|
|
138
|
-
function pairFolderName(assetA, assetB) {
|
|
139
|
-
return `${slugPart(assetA?.symbol)}_${slugPart(assetB?.symbol)}`;
|
|
140
|
-
}
|
|
141
139
|
function outputPath(poolId, intervalSeconds, assetA, assetB) {
|
|
142
140
|
const label = toIntervalLabel(intervalSeconds);
|
|
143
141
|
const id = String(poolId).replace('1.19.', '');
|
|
144
|
-
|
|
142
|
+
// Canonical orientation: `bts/iob.xrp` and `iob.xrp/bts` resolve to ONE
|
|
143
|
+
// folder, so the cache is shared. The fetcher stores canonically and
|
|
144
|
+
// translates back to the requested side on the way out.
|
|
145
|
+
const pair = canonicalPair(assetA, assetB);
|
|
146
|
+
const pairFolder = pairFolderSlug(pair.assetA, pair.assetB);
|
|
145
147
|
return path.join(PATHS.MARKET_ADAPTER.LP_DATA_DIR, pairFolder, `lp_pool_${id}_${label}.json`);
|
|
146
148
|
}
|
|
147
149
|
function applyPrecisionOverrides(assetA, assetB, precA, precB) {
|
|
@@ -151,7 +153,7 @@ function applyPrecisionOverrides(assetA, assetB, precA, precB) {
|
|
|
151
153
|
};
|
|
152
154
|
}
|
|
153
155
|
function pairFolderPath(assetASymbol, assetBSymbol) {
|
|
154
|
-
return path.join(PATHS.MARKET_ADAPTER.LP_DATA_DIR,
|
|
156
|
+
return path.join(PATHS.MARKET_ADAPTER.LP_DATA_DIR, pairFolderSlug({ symbol: assetASymbol }, { symbol: assetBSymbol }));
|
|
155
157
|
}
|
|
156
158
|
function resolveChunkMonths(config) {
|
|
157
159
|
const months = Number(config.chunkMonths);
|
|
@@ -187,7 +189,7 @@ function buildRequestKey(config, fullPoolId, assetA, assetB, timeRange, outPath)
|
|
|
187
189
|
chunkMonths,
|
|
188
190
|
};
|
|
189
191
|
}
|
|
190
|
-
function
|
|
192
|
+
function scanCachedFetchContext(bot, intervalSeconds, dir) {
|
|
191
193
|
if (!storage.exists(dir))
|
|
192
194
|
return null;
|
|
193
195
|
const label = toIntervalLabel(intervalSeconds);
|
|
@@ -214,9 +216,12 @@ function loadCachedFetchContext(bot, intervalSeconds, dir = pairFolderPath(bot.a
|
|
|
214
216
|
continue;
|
|
215
217
|
if (meta.intervalSeconds !== intervalSeconds)
|
|
216
218
|
continue;
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
219
|
+
// Either orientation resolves the cached context; the assets are
|
|
220
|
+
// returned in the bot's own order so the rest of the CLI (and the
|
|
221
|
+
// derived export) stays in the requested orientation.
|
|
222
|
+
const direct = meta.assetA?.symbol === bot.assetA && meta.assetB?.symbol === bot.assetB;
|
|
223
|
+
const inverted = meta.assetA?.symbol === bot.assetB && meta.assetB?.symbol === bot.assetA;
|
|
224
|
+
if (!direct && !inverted)
|
|
220
225
|
continue;
|
|
221
226
|
if (!meta.pool || !meta.assetA?.id || !meta.assetB?.id)
|
|
222
227
|
continue;
|
|
@@ -224,8 +229,8 @@ function loadCachedFetchContext(bot, intervalSeconds, dir = pairFolderPath(bot.a
|
|
|
224
229
|
continue;
|
|
225
230
|
return {
|
|
226
231
|
poolId: meta.pool,
|
|
227
|
-
assetA: meta.assetA,
|
|
228
|
-
assetB: meta.assetB,
|
|
232
|
+
assetA: inverted ? meta.assetB : meta.assetA,
|
|
233
|
+
assetB: inverted ? meta.assetA : meta.assetB,
|
|
229
234
|
source: 'data',
|
|
230
235
|
path: path.join(dir, file),
|
|
231
236
|
};
|
|
@@ -234,6 +239,23 @@ function loadCachedFetchContext(bot, intervalSeconds, dir = pairFolderPath(bot.a
|
|
|
234
239
|
}
|
|
235
240
|
return null;
|
|
236
241
|
}
|
|
242
|
+
/**
|
|
243
|
+
* Locate a cached fetch context for a bot. Standard calls omit `dir` and both
|
|
244
|
+
* orientations are probed, so a bot whose pair is the inverse of the canonical
|
|
245
|
+
* folder still skips the blockchain connect. An explicit `dir` (tests, custom
|
|
246
|
+
* layouts) is scanned alone.
|
|
247
|
+
*/
|
|
248
|
+
function loadCachedFetchContext(bot, intervalSeconds, dir) {
|
|
249
|
+
const candidates = dir
|
|
250
|
+
? [dir]
|
|
251
|
+
: [pairFolderPath(bot.assetA, bot.assetB), pairFolderPath(bot.assetB, bot.assetA)];
|
|
252
|
+
for (const candidate of new Set(candidates)) {
|
|
253
|
+
const found = scanCachedFetchContext(bot, intervalSeconds, candidate);
|
|
254
|
+
if (found)
|
|
255
|
+
return found;
|
|
256
|
+
}
|
|
257
|
+
return null;
|
|
258
|
+
}
|
|
237
259
|
// ─── Local-first incremental reuse ────────────────────────────────────────────
|
|
238
260
|
// Bucket-level reuse lives in ./window_cache.js and is shared by all three
|
|
239
261
|
// candle fetchers (pool, book, feed) through the single runCachedWindows
|
|
@@ -246,21 +268,34 @@ function isLpShardMatch(metaInput, requestKeyInput) {
|
|
|
246
268
|
return false;
|
|
247
269
|
if (meta.intervalSeconds !== requestKey.intervalSeconds)
|
|
248
270
|
return false;
|
|
249
|
-
|
|
271
|
+
return sameAssets(meta, requestKey);
|
|
272
|
+
}
|
|
273
|
+
/** Same identity, opposite orientation (legacy pre-canonicalization shard). */
|
|
274
|
+
function isLpShardMatchInverse(metaInput, requestKeyInput) {
|
|
275
|
+
const meta = metaInput;
|
|
276
|
+
const requestKey = requestKeyInput;
|
|
277
|
+
if (meta.pool !== requestKey.pool)
|
|
250
278
|
return false;
|
|
251
|
-
if (meta.
|
|
279
|
+
if (meta.intervalSeconds !== requestKey.intervalSeconds)
|
|
252
280
|
return false;
|
|
253
|
-
return
|
|
281
|
+
return swappedAssets(meta, requestKey);
|
|
254
282
|
}
|
|
255
283
|
async function fetchCandlesSequentially(fullPoolId, assetA, assetB, config, outPath) {
|
|
256
284
|
// Pool, book and feed fetches share ONE cache function: runCachedWindows
|
|
257
285
|
// in window_cache.js. Stable shard files are the only cache format.
|
|
286
|
+
// Storage is canonical: the fetch queries (and caches) the lower-id leg as
|
|
287
|
+
// A, and the result is inverted back if the caller asked for the other
|
|
288
|
+
// side. That makes `bts/iob.xrp` and `iob.xrp/bts` one cache family.
|
|
289
|
+
const pair = canonicalPair(assetA, assetB);
|
|
290
|
+
const storeA = pair.assetA;
|
|
291
|
+
const storeB = pair.assetB;
|
|
292
|
+
const flipped = pair.flipped;
|
|
258
293
|
const chunkMonths = resolveChunkMonths(config);
|
|
259
294
|
const bucketMs = Number(config.intervalSeconds) * 1000;
|
|
260
295
|
const effectiveTimeRange = config.timeRange
|
|
261
296
|
? { gte: config.timeRange.gte, lte: config.timeRange.lte }
|
|
262
297
|
: normalizeLookbackRange(config);
|
|
263
|
-
const requestKey = buildRequestKey(config, fullPoolId,
|
|
298
|
+
const requestKey = buildRequestKey(config, fullPoolId, storeA, storeB, effectiveTimeRange, outPath);
|
|
264
299
|
const plainWindows = buildFetchWindowsFromRange(effectiveTimeRange, chunkMonths);
|
|
265
300
|
// Windows are fetch-planning splits only (query batching + progress);
|
|
266
301
|
// storage layout is fixed calendar-month shards, so no per-window file.
|
|
@@ -296,7 +331,7 @@ async function fetchCandlesSequentially(fullPoolId, assetA, assetB, config, outP
|
|
|
296
331
|
}
|
|
297
332
|
catch (_) { /* progress must never fail the fetch */ }
|
|
298
333
|
};
|
|
299
|
-
const candles = await kibanaSource.getLpCandlesForPool(fullPoolId,
|
|
334
|
+
const candles = await kibanaSource.getLpCandlesForPool(fullPoolId, storeA, storeB, {
|
|
300
335
|
...config,
|
|
301
336
|
// Per-request timeout is the single network timer here: the old
|
|
302
337
|
// second same-value window timeout was redundant with it (the
|
|
@@ -314,6 +349,12 @@ async function fetchCandlesSequentially(fullPoolId, assetA, assetB, config, outP
|
|
|
314
349
|
outPath,
|
|
315
350
|
requestKey,
|
|
316
351
|
isMatch: isLpShardMatch,
|
|
352
|
+
inverse: (() => {
|
|
353
|
+
const legacy = inverseSiblingOutPath(outPath, storeA, storeB);
|
|
354
|
+
return legacy
|
|
355
|
+
? { outPath: legacy, isMatch: isLpShardMatchInverse, toRequest: (c) => invertCandleSeries(c, { volume: 'scale' }) }
|
|
356
|
+
: undefined;
|
|
357
|
+
})(),
|
|
317
358
|
metaForWindow: (window) => ({
|
|
318
359
|
source: `https://kibana.bitshares.dev (bitshares-*, op_type 63, pool ${requestKey.pool})`,
|
|
319
360
|
pool: requestKey.pool,
|
|
@@ -335,7 +376,7 @@ async function fetchCandlesSequentially(fullPoolId, assetA, assetB, config, outP
|
|
|
335
376
|
},
|
|
336
377
|
});
|
|
337
378
|
console.log(` Merged ${merged.length} candles across ${windows.length} window(s) (month-shard cache)`);
|
|
338
|
-
return merged;
|
|
379
|
+
return flipped ? invertCandleSeries(merged, { volume: 'scale' }) : merged;
|
|
339
380
|
}
|
|
340
381
|
// ─── bots.json helper ─────────────────────────────────────────────────────────
|
|
341
382
|
function loadBotsJson() {
|
|
@@ -562,7 +603,12 @@ async function run() {
|
|
|
562
603
|
// a full-history file with a subset. Falls back to the fetched candles when
|
|
563
604
|
// the shards were withheld (e.g. a partial fetch).
|
|
564
605
|
const assembled = loadLpSeriesFromBasePath(outPath);
|
|
565
|
-
|
|
606
|
+
// Shards are stored in the canonical orientation; the export (and its meta)
|
|
607
|
+
// is the requested one, so translate when the caller asked for the inverse.
|
|
608
|
+
const assembledCandles = assembled && assembled.candles.length > 0
|
|
609
|
+
? (canonicalPair(assetA, assetB).flipped ? invertCandleSeries(assembled.candles, { volume: 'scale' }) : assembled.candles)
|
|
610
|
+
: null;
|
|
611
|
+
const exportCandles = assembledCandles ?? candles;
|
|
566
612
|
const output = {
|
|
567
613
|
meta: {
|
|
568
614
|
fetchedAt: new Date().toISOString(),
|
|
@@ -37,6 +37,7 @@ import { path } from '../../modules/path_api.js';
|
|
|
37
37
|
import { PATHS } from '../../modules/paths.js';
|
|
38
38
|
import { toIntervalLabel, slugPart } from '../interval_utils.js';
|
|
39
39
|
import { buildFetchWindowsFromRange, runCachedWindows } from './window_cache.js';
|
|
40
|
+
import { canonicalPair, pairFolderSlug, sameAssets, swappedAssets, inverseSiblingOutPath, compareAssetIds } from '../utils/pair_orientation.js';
|
|
40
41
|
import { isTransientNetworkError, sleepMs } from '../../modules/utils/errors.js';
|
|
41
42
|
// ─── Constants ────────────────────────────────────────────────────────────────
|
|
42
43
|
const OP_TYPE_FEED = 19; // asset_publish_feed
|
|
@@ -488,9 +489,21 @@ function feedCacheKey(feedCtx) {
|
|
|
488
489
|
}
|
|
489
490
|
function feedOutputPath(feedKey, intervalSeconds, assetA, assetB) {
|
|
490
491
|
const label = toIntervalLabel(intervalSeconds);
|
|
491
|
-
|
|
492
|
+
// Canonical orientation: the inverse pair shares one feed cache folder.
|
|
493
|
+
const canon = canonicalPair(assetA || {}, assetB || {});
|
|
494
|
+
const folder = pairFolderSlug(canon.assetA, canon.assetB);
|
|
492
495
|
return path.join(PATHS.MARKET_ADAPTER.FEED_DATA_DIR, folder, `feed_${slugPart(feedKey)}_${label}.json`);
|
|
493
496
|
}
|
|
497
|
+
/**
|
|
498
|
+
* Cross legs sorted by MPA id so the feed key is orientation-independent; a
|
|
499
|
+
* single leg is returned untouched.
|
|
500
|
+
*/
|
|
501
|
+
function normalizeFeedLegs(feedCtx) {
|
|
502
|
+
if (feedCtx?.kind !== 'cross' || !Array.isArray(feedCtx.legs))
|
|
503
|
+
return feedCtx;
|
|
504
|
+
const legs = [...feedCtx.legs].sort((a, b) => compareAssetIds(a?.mpa?.id, b?.mpa?.id));
|
|
505
|
+
return { ...feedCtx, legs };
|
|
506
|
+
}
|
|
494
507
|
function isFeedChunkMatch(metaInput, requestKeyInput) {
|
|
495
508
|
const meta = metaInput;
|
|
496
509
|
const requestKey = requestKeyInput;
|
|
@@ -498,11 +511,17 @@ function isFeedChunkMatch(metaInput, requestKeyInput) {
|
|
|
498
511
|
return false;
|
|
499
512
|
if (meta.intervalSeconds !== requestKey.intervalSeconds)
|
|
500
513
|
return false;
|
|
501
|
-
|
|
514
|
+
return sameAssets(meta, requestKey);
|
|
515
|
+
}
|
|
516
|
+
/** Same identity, opposite orientation (legacy pre-canonicalization shard). */
|
|
517
|
+
function isFeedChunkMatchInverse(metaInput, requestKeyInput) {
|
|
518
|
+
const meta = metaInput;
|
|
519
|
+
const requestKey = requestKeyInput;
|
|
520
|
+
if (meta.feed !== requestKey.feed)
|
|
502
521
|
return false;
|
|
503
|
-
if (meta.
|
|
522
|
+
if (meta.intervalSeconds !== requestKey.intervalSeconds)
|
|
504
523
|
return false;
|
|
505
|
-
return
|
|
524
|
+
return swappedAssets(meta, requestKey);
|
|
506
525
|
}
|
|
507
526
|
async function fetchFeedCandlesSequentially(feedCtx, assetA, assetB, opts = {}) {
|
|
508
527
|
const intervalSeconds = Number(opts.intervalSeconds) || 3600;
|
|
@@ -511,15 +530,22 @@ async function fetchFeedCandlesSequentially(feedCtx, assetA, assetB, opts = {})
|
|
|
511
530
|
if (!timeRange?.gte || !timeRange?.lte) {
|
|
512
531
|
throw new Error('fetchFeedCandlesSequentially requires opts.timeRange { gte, lte }');
|
|
513
532
|
}
|
|
514
|
-
|
|
515
|
-
|
|
533
|
+
// Canonical storage: fetch/cache the lower-id leg as A and invert the
|
|
534
|
+
// result back if the caller asked for the other side.
|
|
535
|
+
const canon = canonicalPair(assetA, assetB);
|
|
536
|
+
const storeA = canon.assetA;
|
|
537
|
+
const storeB = canon.assetB;
|
|
538
|
+
const flipped = canon.flipped;
|
|
539
|
+
const normalizedCtx = normalizeFeedLegs(feedCtx);
|
|
540
|
+
const feedKey = feedCacheKey(normalizedCtx);
|
|
541
|
+
const outPath = opts.outPath || feedOutputPath(feedKey, intervalSeconds, storeA, storeB);
|
|
516
542
|
const requestKey = {
|
|
517
543
|
feed: feedKey,
|
|
518
|
-
assetA: { id:
|
|
519
|
-
assetB: { id:
|
|
544
|
+
assetA: { id: storeA.id, precision: storeA.precision, symbol: storeA.symbol },
|
|
545
|
+
assetB: { id: storeB.id, precision: storeB.precision, symbol: storeB.symbol },
|
|
520
546
|
intervalSeconds,
|
|
521
547
|
};
|
|
522
|
-
const isCross =
|
|
548
|
+
const isCross = normalizedCtx?.kind === 'cross';
|
|
523
549
|
const plainWindows = buildFetchWindowsFromRange(timeRange, chunkMonths);
|
|
524
550
|
// Windows are fetch-planning splits only; storage is fixed month shards.
|
|
525
551
|
const windows = plainWindows.map((w, idx) => ({
|
|
@@ -528,13 +554,19 @@ async function fetchFeedCandlesSequentially(feedCtx, assetA, assetB, opts = {})
|
|
|
528
554
|
lte: w.lte,
|
|
529
555
|
}));
|
|
530
556
|
const fetchRange = isCross
|
|
531
|
-
? (gte, lte) => getFeedCandlesForMpaCross(
|
|
532
|
-
: (gte, lte) => getFeedCandlesForPair(
|
|
533
|
-
|
|
557
|
+
? (gte, lte) => getFeedCandlesForMpaCross(storeA, storeB, normalizedCtx.legs[0], normalizedCtx.legs[1], { intervalSeconds, timeRange: { gte, lte } })
|
|
558
|
+
: (gte, lte) => getFeedCandlesForPair(storeA, storeB, normalizedCtx.legs[0].mpa, normalizedCtx.legs[0].backing, { intervalSeconds, timeRange: { gte, lte } });
|
|
559
|
+
const merged = await runCachedWindows({
|
|
534
560
|
windows,
|
|
535
561
|
outPath,
|
|
536
562
|
requestKey,
|
|
537
563
|
isMatch: isFeedChunkMatch,
|
|
564
|
+
inverse: (() => {
|
|
565
|
+
const legacy = inverseSiblingOutPath(outPath, storeA, storeB);
|
|
566
|
+
return legacy
|
|
567
|
+
? { outPath: legacy, isMatch: isFeedChunkMatchInverse, toRequest: (c) => invertCandles(c) }
|
|
568
|
+
: undefined;
|
|
569
|
+
})(),
|
|
538
570
|
metaForWindow: (window) => ({
|
|
539
571
|
source: `https://kibana.bitshares.dev (bitshares-*, op_type 19, feed ${feedKey})`,
|
|
540
572
|
feed: feedKey,
|
|
@@ -550,6 +582,7 @@ async function fetchFeedCandlesSequentially(feedCtx, assetA, assetB, opts = {})
|
|
|
550
582
|
// would drop leading ratios, so only exact reuse applies to them.
|
|
551
583
|
allowSubFetch: !isCross,
|
|
552
584
|
});
|
|
585
|
+
return flipped ? invertCandles(merged) : merged;
|
|
553
586
|
}
|
|
554
587
|
export { OP_TYPE_FEED, backingPerMpa, hitToFeedPrice, buildFeedDocumentQuery, crossPointsToRatios, bucketPricesToCandles, invertCandles, getFeedCandlesForMpa, getFeedCandlesForPair, getFeedCandlesForMpaCross, fetchFeedCandlesSequentially, };
|
|
555
588
|
//# sourceMappingURL=kibana_feed_source.js.map
|
|
@@ -21,6 +21,7 @@ interface BucketCache {
|
|
|
21
21
|
shardKey: string;
|
|
22
22
|
candles: Candle[];
|
|
23
23
|
queried: QueriedRange[];
|
|
24
|
+
foreign?: boolean;
|
|
24
25
|
}>;
|
|
25
26
|
}
|
|
26
27
|
interface ShardMeta {
|
|
@@ -44,6 +45,17 @@ interface FetchRangeResult {
|
|
|
44
45
|
candles: Candle[];
|
|
45
46
|
complete?: boolean;
|
|
46
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Opposite-orientation shard family, read as a migration source. When a month's
|
|
50
|
+
* canonical shard is absent but this family has one, its candles are
|
|
51
|
+
* transformed into the canonical orientation and reused; the flush then writes
|
|
52
|
+
* them into the canonical shard (one-way migration, no re-fetch).
|
|
53
|
+
*/
|
|
54
|
+
interface InverseCacheSource {
|
|
55
|
+
outPath: string;
|
|
56
|
+
isMatch: (meta: unknown, requestKey: unknown) => boolean;
|
|
57
|
+
toRequest: (candles: Candle[]) => Candle[];
|
|
58
|
+
}
|
|
47
59
|
interface RunCachedWindowsOptions {
|
|
48
60
|
windows: Array<{
|
|
49
61
|
index: number;
|
|
@@ -53,6 +65,7 @@ interface RunCachedWindowsOptions {
|
|
|
53
65
|
outPath: string;
|
|
54
66
|
requestKey: unknown;
|
|
55
67
|
isMatch: (meta: unknown, requestKey: unknown) => boolean;
|
|
68
|
+
inverse?: InverseCacheSource;
|
|
56
69
|
metaForWindow: (window: {
|
|
57
70
|
index: number;
|
|
58
71
|
gte: string;
|
|
@@ -82,7 +95,7 @@ declare function readCacheChunk(chunkFile: string, requestKey: unknown, isMatch:
|
|
|
82
95
|
declare function loadBucketCache(outPath: string, requestKey: unknown, isMatch: (meta: unknown, requestKey: unknown) => boolean, range?: {
|
|
83
96
|
gte: number;
|
|
84
97
|
lte: number;
|
|
85
|
-
} | null): BucketCache;
|
|
98
|
+
} | null, inverse?: InverseCacheSource): BucketCache;
|
|
86
99
|
/**
|
|
87
100
|
* How settled a window is, given the recorded coverage.
|
|
88
101
|
*
|
|
@@ -151,25 +151,57 @@ function readCacheChunk(chunkFile, requestKey, isMatch) {
|
|
|
151
151
|
return null;
|
|
152
152
|
}
|
|
153
153
|
}
|
|
154
|
-
|
|
154
|
+
/**
|
|
155
|
+
* Shard files to open for a run, canonical first. A month whose canonical
|
|
156
|
+
* shard is absent falls back to the inverse family (flagged `foreign`) so a
|
|
157
|
+
* pre-canonicalization cache is reused instead of re-fetched.
|
|
158
|
+
*/
|
|
159
|
+
function cacheEntriesFor(outPath, inverse, range) {
|
|
160
|
+
const out = [];
|
|
161
|
+
const scoped = range && Number.isFinite(range.gte) && Number.isFinite(range.lte);
|
|
162
|
+
if (scoped) {
|
|
163
|
+
for (const shardKey of shardKeysForRange(range.gte, range.lte)) {
|
|
164
|
+
const primary = shardPathFor(outPath, shardKey);
|
|
165
|
+
if (storage.exists(primary)) {
|
|
166
|
+
out.push({ file: primary, shardKey, foreign: false });
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
if (!inverse)
|
|
170
|
+
continue;
|
|
171
|
+
const alt = shardPathFor(inverse.outPath, shardKey);
|
|
172
|
+
if (storage.exists(alt))
|
|
173
|
+
out.push({ file: alt, shardKey, foreign: true });
|
|
174
|
+
}
|
|
175
|
+
return out;
|
|
176
|
+
}
|
|
177
|
+
const seen = new Set();
|
|
178
|
+
for (const entry of siblingCacheFiles(outPath)) {
|
|
179
|
+
out.push({ ...entry, foreign: false });
|
|
180
|
+
seen.add(entry.shardKey);
|
|
181
|
+
}
|
|
182
|
+
if (inverse) {
|
|
183
|
+
for (const entry of siblingCacheFiles(inverse.outPath)) {
|
|
184
|
+
if (seen.has(entry.shardKey))
|
|
185
|
+
continue;
|
|
186
|
+
out.push({ ...entry, foreign: true });
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
out.sort((a, b) => (a.file < b.file ? -1 : 1));
|
|
190
|
+
return out;
|
|
191
|
+
}
|
|
192
|
+
function loadBucketCache(outPath, requestKey, isMatch, range, inverse) {
|
|
155
193
|
const byTs = new Map();
|
|
156
194
|
const fileCover = [];
|
|
157
195
|
const shards = [];
|
|
158
196
|
let files = 0;
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
// back to the full listing.
|
|
164
|
-
const entries = scoped
|
|
165
|
-
? shardKeysForRange(range.gte, range.lte)
|
|
166
|
-
.map((shardKey) => ({ file: shardPathFor(outPath, shardKey), shardKey }))
|
|
167
|
-
.filter((entry) => storage.exists(entry.file))
|
|
168
|
-
: siblingCacheFiles(outPath);
|
|
169
|
-
for (const entry of entries) {
|
|
170
|
-
const chunk = readCacheChunk(entry.file, requestKey, isMatch);
|
|
171
|
-
if (!chunk)
|
|
197
|
+
for (const entry of cacheEntriesFor(outPath, inverse, range)) {
|
|
198
|
+
const matcher = entry.foreign && inverse ? inverse.isMatch : isMatch;
|
|
199
|
+
const raw = readCacheChunk(entry.file, requestKey, matcher);
|
|
200
|
+
if (!raw)
|
|
172
201
|
continue;
|
|
202
|
+
const chunk = entry.foreign && inverse
|
|
203
|
+
? { ...raw, candles: inverse.toRequest(raw.candles) }
|
|
204
|
+
: raw;
|
|
173
205
|
files += 1;
|
|
174
206
|
fileCover.push({ gte: chunk.rangeGte, lte: chunk.rangeLte, count: chunk.candles.length, queried: chunk.queried });
|
|
175
207
|
for (const c of chunk.candles) {
|
|
@@ -182,7 +214,7 @@ function loadBucketCache(outPath, requestKey, isMatch, range) {
|
|
|
182
214
|
if (!prev || Number(c[5] || 0) > Number(prev[5] || 0))
|
|
183
215
|
byTs.set(ts, c);
|
|
184
216
|
}
|
|
185
|
-
shards.push({ file: entry.file, shardKey: entry.shardKey, candles: chunk.candles.filter((c) => Array.isArray(c)), queried: chunk.queried });
|
|
217
|
+
shards.push({ file: entry.file, shardKey: entry.shardKey, candles: chunk.candles.filter((c) => Array.isArray(c)), queried: chunk.queried, foreign: entry.foreign });
|
|
186
218
|
}
|
|
187
219
|
return { byTs, files, fileCover, shards };
|
|
188
220
|
}
|
|
@@ -591,7 +623,7 @@ async function runCachedWindows(opts) {
|
|
|
591
623
|
? { gte: Math.min(...bounds.map((b) => b.gte)), lte: Math.max(...bounds.map((b) => b.lte)) }
|
|
592
624
|
: null;
|
|
593
625
|
// Scoped load: only shards overlapping the run are opened.
|
|
594
|
-
const localCache = loadBucketCache(outPath, requestKey, isMatch, overall);
|
|
626
|
+
const localCache = loadBucketCache(outPath, requestKey, isMatch, overall, opts.inverse);
|
|
595
627
|
if (localCache.files > 0) {
|
|
596
628
|
console.log(` Local cache: ${localCache.files} file(s), ${localCache.byTs.size} buckets — fetching only what is missing`);
|
|
597
629
|
}
|
|
@@ -747,7 +779,11 @@ async function runCachedWindows(opts) {
|
|
|
747
779
|
const fetchedAt = new Date().toISOString();
|
|
748
780
|
// O(1) lookup of each shard's loaded candles for the write-on-change
|
|
749
781
|
// comparison below (instead of a linear scan per shard state).
|
|
750
|
-
|
|
782
|
+
// Foreign (inverse-orientation) shards deliberately excluded: their
|
|
783
|
+
// buckets are folded into the canonical state below, and counting them
|
|
784
|
+
// as the canonical shard's prior content would suppress the write that
|
|
785
|
+
// performs the migration.
|
|
786
|
+
const loadedShardCandles = new Map(localCache.shards.filter((s) => !s.foreign).map((s) => [s.shardKey, s.candles]));
|
|
751
787
|
for (const state of shardStates.values()) {
|
|
752
788
|
const { start, end } = shardBoundsForKey(state.key);
|
|
753
789
|
// Fold every loaded bucket into its home shard. Presence on disk
|
|
@@ -19,9 +19,10 @@ async function resolveAsset(symbol, bitsharesClient = null) {
|
|
|
19
19
|
if (!symbol || typeof symbol !== 'string') {
|
|
20
20
|
throw new Error(`Cannot resolve asset: invalid or missing symbol "${String(symbol)}"`);
|
|
21
21
|
}
|
|
22
|
-
// BitShares symbols are canonical UPPERCASE
|
|
23
|
-
//
|
|
24
|
-
// (cache keys, Kibana terms,
|
|
22
|
+
// BitShares symbols are canonical UPPERCASE and `lookup_asset_symbols` is
|
|
23
|
+
// case-sensitive (a lowercase symbol resolves to null), so normalize here
|
|
24
|
+
// and hand the canonical symbol downstream (cache keys, Kibana terms,
|
|
25
|
+
// chart labels) instead of the raw input.
|
|
25
26
|
const canonical = normalizeAssetSymbol(symbol);
|
|
26
27
|
const client = bitsharesClient || getBitsharesClient();
|
|
27
28
|
const results = await client.BitShares.db.lookup_asset_symbols([canonical]);
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
export interface OrientableAsset {
|
|
2
|
+
id?: string | null;
|
|
3
|
+
precision?: unknown;
|
|
4
|
+
symbol?: string | null;
|
|
5
|
+
}
|
|
6
|
+
/** The asset legs stored on every shard meta and cache request key. */
|
|
7
|
+
export interface AssetPairLike {
|
|
8
|
+
assetA?: {
|
|
9
|
+
id?: string | null;
|
|
10
|
+
precision?: unknown;
|
|
11
|
+
} | null;
|
|
12
|
+
assetB?: {
|
|
13
|
+
id?: string | null;
|
|
14
|
+
precision?: unknown;
|
|
15
|
+
} | null;
|
|
16
|
+
}
|
|
17
|
+
type Candle = number[];
|
|
18
|
+
/** Numeric-aware compare of dotted BitShares ids (`1.3.2` < `1.3.10`). */
|
|
19
|
+
declare function compareAssetIds(a: unknown, b: unknown): number;
|
|
20
|
+
/**
|
|
21
|
+
* Order a pair canonically (lower asset id first). Falls back to symbol order
|
|
22
|
+
* when ids are missing, and keeps the caller's order (flipped=false) when
|
|
23
|
+
* neither can order the legs.
|
|
24
|
+
*/
|
|
25
|
+
declare function canonicalPair<A extends OrientableAsset, B extends OrientableAsset>(assetA: A, assetB: B): {
|
|
26
|
+
assetA: A | B;
|
|
27
|
+
assetB: A | B;
|
|
28
|
+
flipped: boolean;
|
|
29
|
+
};
|
|
30
|
+
/** Slug of the pair cache folder (`bts_iob_xrp`), canonical-agnostic. */
|
|
31
|
+
declare function pairFolderSlug(assetA: OrientableAsset | null | undefined, assetB: OrientableAsset | null | undefined): string;
|
|
32
|
+
/** Same asset legs, same order (id + precision). */
|
|
33
|
+
declare function sameAssets(meta: AssetPairLike | null | undefined, requestKey: AssetPairLike | null | undefined): boolean;
|
|
34
|
+
/** Same asset legs, opposite order — a legacy pre-canonicalization shard. */
|
|
35
|
+
declare function swappedAssets(meta: AssetPairLike | null | undefined, requestKey: AssetPairLike | null | undefined): boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Opposite-orientation twin of a standard-layout cache path, or null for a
|
|
38
|
+
* custom path. Migration source only: the pre-canonicalization cache wrote the
|
|
39
|
+
* inverse pair to its own folder. The parent folder must be the canonical
|
|
40
|
+
* pair folder (so test temp dirs and `--out` targets are skipped), and callers
|
|
41
|
+
* that encode the folder in the filename (book) pass `inverseBaseName`.
|
|
42
|
+
*/
|
|
43
|
+
declare function inverseSiblingOutPath(outPath: string, canonicalA: OrientableAsset, canonicalB: OrientableAsset, inverseBaseName?: string): string | null;
|
|
44
|
+
/**
|
|
45
|
+
* Invert a candle series to the opposite pair orientation.
|
|
46
|
+
*
|
|
47
|
+
* `volume`:
|
|
48
|
+
* - 'scale' (LP / order-book): new base volume ≈ old base volume × close,
|
|
49
|
+
* the same approximation the TV chart uses for its inverse view.
|
|
50
|
+
* - 'preserve' (feed): volume is a publish count, unchanged.
|
|
51
|
+
*
|
|
52
|
+
* Rows whose OHLC are not all finite and positive pass through untouched
|
|
53
|
+
* (defensive; cache data should never hit this, and dropping rows would break
|
|
54
|
+
* gap-filled continuity).
|
|
55
|
+
*/
|
|
56
|
+
declare function invertCandleSeries(candles: unknown, opts?: {
|
|
57
|
+
volume?: 'scale' | 'preserve';
|
|
58
|
+
}): Candle[];
|
|
59
|
+
export { compareAssetIds, canonicalPair, pairFolderSlug, sameAssets, swappedAssets, inverseSiblingOutPath, invertCandleSeries };
|
|
60
|
+
//# sourceMappingURL=pair_orientation.d.ts.map
|