dexbot 1.6.2 → 1.6.4

package/analysis/chain_pool.ts

@@ -0,0 +1,47 @@
+'use strict';
+
+/**
+ * SHARED CHAIN POOL (analysis)
+ *
+ * Single source of truth for the node list and the ephemeral read-only
+ * connection used by analysis tools for small on-chain lookups (account name
+ * resolution, asset precision resolution).
+ *
+ * Analysis tools deliberately do NOT use the production singleton client
+ * (modules/bitshares_client.ts): they are short-lived processes and must never
+ * disturb a running bot's connection state. Instead each lookup opens a fresh
+ * read-only client over the built-in node pool, which the native transport
+ * races in parallel and then retries sequentially — so one dead node is not
+ * fatal.
+ */
+
+import * as C from '../modules/constants.js';
+
+/** Nodes for ephemeral read-only analysis connections (node management pool). */
+function defaultNodePool(): string[] {
+    return [...C.NODE_MANAGEMENT.DEFAULT_NODES];
+}
+
+/**
+ * Run `fn` against a fresh read-only client connected over the built-in pool.
+ * Transport INFO logging is silenced for the duration of the short-lived
+ * connection and the client is always disconnected afterwards.
+ */
+async function withReadOnlyClient<T>(fn: (client: any) => Promise<T>): Promise<T> {
+    const { createReadOnlyClient } = await import('../modules/bitshares-native/index.js');
+    const client = createReadOnlyClient({ nodes: defaultNodePool() });
+    // Suppress transport INFO logs during ephemeral connection:
+    // bitshares-native transport logger (new Logger('Transport')) writes
+    // "[timestamp] [INFO] [Transport] ..." — silence by raising log level.
+    const prevLevel = process.env.LOG_LEVEL;
+    process.env.LOG_LEVEL = 'warn';
+    try {
+        await client.connect();
+        return await fn(client);
+    } finally {
+        try { client.disconnect(); } catch (_) { /* best-effort cleanup */ }
+        process.env.LOG_LEVEL = prevLevel;
+    }
+}
+
+export { defaultNodePool, withReadOnlyClient };

package/analysis/chart_utils.ts

@@ -10,26 +10,39 @@ const { ensureDir } = getStorage();
  * Chart utilities for analysis HTML generators.
  */
 
-// Runtime assets every generated chart references as ../uplot/* relative to
-// its own directory (charts live in <root>/charts, assets in <root>/uplot).
-const UPLOT_RUNTIME_FILES = ['uPlot.iife.min.js', 'uPlot.min.css'];
+// Vendored uPlot runtime assets are inlined directly into every generated chart
+// (see uplotInlineTags below), so each chart is a single self-contained file
+// that renders anywhere: no DEXBot2 install, no sibling uplot/ dir, no CDN, no
+// network. The vendored originals always live with the code under the analysis
+// assets dir.
+let _uplotRuntime: { css: string; js: string } | null = null;
+
+function getUplotRuntime() {
+    if (!_uplotRuntime) {
+        const dir = PATHS.ANALYSIS.ASSETS_DIR;
+        _uplotRuntime = {
+            css: fs.readFileSync(path.join(dir, 'uPlot.min.css'), 'utf8'),
+            js: fs.readFileSync(path.join(dir, 'uPlot.iife.min.js'), 'utf8'),
+        };
+    }
+    return _uplotRuntime;
+}
 
 /**
- * Generated HTML loads uPlot from a sibling `uplot/` dir next to the charts
- * dir. In a source checkout that dir is the vendored original; anywhere else
- * (relocated analysis root under npm/home profiles, or a custom --chart path)
- * materialize a copy so the relative refs keep resolving.
+ * Inline <style>/<script> tags carrying the vendored uPlot runtime. Embedding
+ * the source (not a relative <script src>) makes a chart portable: it keeps
+ * rendering after being copied, mailed, or reopened on a machine without a
+ * DEXBot2 installation. The inline script must be the first script that runs,
+ * placed before any chart-initialization code, since uPlot must exist first.
+ * Both the minified JS (no `</script>`) and CSS (no `</style>`) are safe to
+ * embed verbatim.
  */
-function ensureSiblingUplotAssets(chartDir: any) {
-    const targetDir = path.join(path.dirname(path.resolve(chartDir)), 'uplot');
-    const sourceDir = PATHS.ANALYSIS.ASSETS_DIR;
-    if (targetDir === path.resolve(sourceDir)) return;
-    if (UPLOT_RUNTIME_FILES.every((f) => fs.existsSync(path.join(targetDir, f)))) return;
-    if (!fs.existsSync(sourceDir)) return;
-    fs.mkdirSync(targetDir, { recursive: true });
-    for (const f of UPLOT_RUNTIME_FILES) {
-        fs.copyFileSync(path.join(sourceDir, f), path.join(targetDir, f));
-    }
+function uplotInlineTags({ css = true, js = true } = {}) {
+    const { css: cssCode, js: jsCode } = getUplotRuntime();
+    const parts: string[] = [];
+    if (css && cssCode) parts.push(`<style>\n${cssCode}\n</style>`);
+    if (js && jsCode) parts.push(`<script>\n${jsCode}\n</script>`);
+    return parts.join('\n    ');
 }
 
 
@@ -58,9 +71,9 @@ function toFileUrl(filePath: any): string {
     return `file://${path.resolve(String(filePath))}`;
 }
 
-function writeChartFile(filePath: any, html: any) {    const chartDir = path.dirname(filePath);
+function writeChartFile(filePath: any, html: any) {
+    const chartDir = path.dirname(filePath);
     if (!fs.existsSync(chartDir)) ensureDir(chartDir);
-    ensureSiblingUplotAssets(chartDir);
     // Atomic write (tmp + rename, matching the production storage adapter
     // pattern) so a crash mid-write can never leave a truncated chart file.
     const tmpPath = `${filePath}.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2, 10)}.tmp`;
@@ -193,5 +206,5 @@ function bindPan(chart) {
 }
 `;
 
-export { escapeHtml, serializeJsonForScript, toEpochSeconds, writeChartFile, toFileUrl, embedFunctionSources, UPLOT_SHARED_SCRIPT }
+export { escapeHtml, serializeJsonForScript, toEpochSeconds, writeChartFile, toFileUrl, embedFunctionSources, UPLOT_SHARED_SCRIPT, uplotInlineTags }
 

package/analysis/derivative_chart_generator.ts

@@ -4,7 +4,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
-import { escapeHtml, serializeJsonForScript, toEpochSeconds, UPLOT_SHARED_SCRIPT } from './chart_utils.js';
+import { escapeHtml, serializeJsonForScript, toEpochSeconds, UPLOT_SHARED_SCRIPT, uplotInlineTags } from './chart_utils.js';
 import { getStorage } from '../modules/storage/index.js';
 const { ensureDir, readJSON } = getStorage();
 import { fixedTo } from '../modules/order/utils/math.js';
@@ -225,8 +225,7 @@ function generateHTML(data: any, title: string) {
     <meta name="darkreader-lock">
     <meta name="color-scheme" content="dark">
     <title>${escapeHtml(title)}</title>
-    <link rel="stylesheet" href="../uplot/uPlot.min.css">
-    <script src="../uplot/uPlot.iife.min.js"></script>
+    ${uplotInlineTags()}
     <style>
         :root { color-scheme: dark; }
         * { box-sizing: border-box; margin: 0; padding: 0; }

package/analysis/fills_source.ts

@@ -0,0 +1,234 @@
+'use strict';
+
+/**
+ * SHARED FILLS SOURCE (analysis)
+ *
+ * Single implementation of the Kibana `fill_order` pipeline and the asset
+ * precision table/cache used by the fill-based analysis tools. Both
+ * trade_profitability.ts and grid_correction_check.ts carried verbatim copies
+ * of this (43-entry ASSETS table, toReal/assetPrec helpers, on-chain precision
+ * resolution, query builder and paginated fetch); all of it lives here now.
+ */
+
+import * as KC from '../market_adapter/core/kibana_client.js';
+import { withReadOnlyClient } from './chain_pool.js';
+
+const { kibanaSearch, DEFAULT_CONFIG: BASE_CONFIG } = KC;
+
+/** operation_type of a fill_order operation in the chain history index. */
+const OP_FILL_ORDER = 4;
+const BTS_ID = '1.3.0';
+
+interface AssetInfo {
+    symbol: string;
+    precision: number;
+}
+
+interface AssetAmount {
+    amount: number;
+    asset_id: string;
+}
+
+interface FillRecord {
+    time: string;
+    blockNum: number;
+    opNum: number;
+    orderId: string;
+    accountId: string;
+    pays: AssetAmount;
+    receives: AssetAmount;
+    fee: AssetAmount;
+    isMaker: boolean;
+    sort: any[];
+}
+
+const ASSETS: Record<string, AssetInfo> = {
+    '1.3.0':    { symbol: 'BTS',          precision: 5 },
+    '1.3.118':  { symbol: 'GBP',          precision: 4 },
+    '1.3.119':  { symbol: 'JPY',          precision: 2 },
+    '1.3.120':  { symbol: 'EUR',          precision: 4 },
+    '1.3.1325': { symbol: 'RUBLE',        precision: 5 },
+    '1.3.2512': { symbol: 'EVRAZ',        precision: 4 },
+    '1.3.3291': { symbol: 'TWENTIX',      precision: 5 },
+    '1.3.4099': { symbol: 'XBTSX.STH',    precision: 6 },
+    '1.3.4156': { symbol: 'XBTSX.DOGE',   precision: 5 },
+    '1.3.4157': { symbol: 'XBTSX.BTC',    precision: 8 },
+    '1.3.4159': { symbol: 'XBTSX.LTC',    precision: 8 },
+    '1.3.4176': { symbol: 'XBTSX.DASH',   precision: 8 },
+    '1.3.4274': { symbol: 'XBTSX.BCH',    precision: 8 },
+    '1.3.4760': { symbol: 'XBTSX.ETH',    precision: 7 },
+    '1.3.5537': { symbol: 'IOB.XRP',      precision: 4 },
+    '1.3.5541': { symbol: 'XBTSX.BNB',    precision: 7 },
+    '1.3.5589': { symbol: 'XBTSX.USDT',   precision: 6 },
+    '1.3.5641': { symbol: 'HONEST.CNY',   precision: 4 },
+    '1.3.5649': { symbol: 'HONEST.USD',   precision: 4 },
+    '1.3.5650': { symbol: 'HONEST.BTC',   precision: 8 },
+    '1.3.5659': { symbol: 'HONEST.ETH',   precision: 6 },
+    '1.3.5870': { symbol: 'XBTSX.FIL',    precision: 6 },
+    '1.3.5887': { symbol: 'XBTSX.RUB',    precision: 4 },
+    '1.3.5902': { symbol: 'XBTSX.USDC',   precision: 6 },
+    '1.3.6013': { symbol: 'XBTSX.HIVE',   precision: 6 },
+    '1.3.6124': { symbol: 'XBTSX.AVAX',   precision: 6 },
+    '1.3.6139': { symbol: 'XBTSX.XAUT',   precision: 6 },
+    '1.3.6166': { symbol: 'XBTSX.MATIC',  precision: 5 },
+    '1.3.6241': { symbol: 'XBTSX.ETC',    precision: 7 },
+    '1.3.6268': { symbol: 'BTWTY.EOS',    precision: 4 },
+    '1.3.6301': { symbol: 'HONEST.MONEY', precision: 8 },
+    '1.3.6304': { symbol: 'HONEST.ADA',   precision: 8 },
+    '1.3.6305': { symbol: 'HONEST.DOT',   precision: 8 },
+    '1.3.6309': { symbol: 'HONEST.ATOM',  precision: 8 },
+    '1.3.6311': { symbol: 'HONEST.ALGO',  precision: 8 },
+    '1.3.6312': { symbol: 'HONEST.FIL',   precision: 8 },
+    '1.3.6313': { symbol: 'HONEST.EOS',   precision: 8 },
+    '1.3.6315': { symbol: 'HONEST.EUR',   precision: 4 },
+    '1.3.6316': { symbol: 'HONEST.GBP',   precision: 4 },
+    '1.3.6317': { symbol: 'HONEST.JPY',   precision: 4 },
+    '1.3.6444': { symbol: 'IOB.XLM',      precision: 4 },
+    '1.3.6573': { symbol: 'XBTSX.DAI',    precision: 6 },
+    '1.3.6620': { symbol: 'XBTSX.A',      precision: 6 },
+    '1.3.6627': { symbol: 'XBTSX.LINK',   precision: 6 },
+};
+
+/** Precisions learned on-chain this run (populated by resolveAssetPrecisions). */
+const resolvedPrecisions: Record<string, number> = {};
+
+function assetSymbol(id: string): string {
+    return ASSETS[id]?.symbol ?? id;
+}
+
+function assetPrec(id: string): number | undefined {
+    return ASSETS[id]?.precision ?? resolvedPrecisions[id];
+}
+
+function toReal(amount: number, assetId: string): number {
+    const p = assetPrec(assetId);
+    if (p === undefined) return NaN;
+    return amount / Math.pow(10, p);
+}
+
+/**
+ * Collect all unique non-BTS asset IDs from fills, resolve unknown precisions
+ * from the blockchain, and populate the runtime cache.
+ */
+async function resolveAssetPrecisions(fills: FillRecord[]): Promise<void> {
+    const unknownIds = new Set<string>();
+    for (const f of fills) {
+        for (const id of [f.pays.asset_id, f.receives.asset_id, f.fee.asset_id]) {
+            if (id !== BTS_ID && !(id in ASSETS) && !(id in resolvedPrecisions)) {
+                unknownIds.add(id);
+            }
+        }
+    }
+    if (unknownIds.size === 0) return;
+
+    const ids = [...unknownIds];
+    console.log(`  Resolving ${ids.length} unknown asset(s) from blockchain...`);
+    try {
+        await withReadOnlyClient(async (client) => {
+            const assets = await client.db('get_assets', [ids]);
+            if (Array.isArray(assets)) {
+                for (const asset of assets) {
+                    if (asset?.id && asset.precision != null) {
+                        resolvedPrecisions[asset.id] = asset.precision;
+                        console.log(`    ${asset.id} → ${asset.symbol || '?'} (precision ${asset.precision})`);
+                    }
+                }
+            }
+            const missing = ids.filter(id => !(id in resolvedPrecisions));
+            if (missing.length > 0) {
+                console.warn(`  [warn] ${missing.length} asset(s) not found on chain: ${missing.join(', ')}. Fills referencing them will be skipped.`);
+            }
+        });
+    } catch (e: any) {
+        console.warn(`  [warn] Asset resolution failed: ${e.message}. Fills with unknown assets will be skipped.`);
+    }
+}
+
+function buildFillQuery(accountId: string, gte: string, lte: string, size: number) {
+    return {
+        size,
+        track_total_hits: false,
+        _source: [
+            'block_data.block_time',
+            'block_data.block_num',
+            'operation_id_num',
+            'operation_history.op_object.pays',
+            'operation_history.op_object.receives',
+            'operation_history.op_object.fee',
+            'operation_history.op_object.order_id',
+            'operation_history.op_object.account_id',
+            'operation_history.op_object.is_maker',
+        ],
+        query: {
+            bool: {
+                filter: [
+                    { term: { operation_type: OP_FILL_ORDER } },
+                    { term: { 'operation_history.op_object.account_id.keyword': accountId } },
+                    { range: { 'block_data.block_time': { gte, lte } } },
+                ],
+            },
+        },
+        sort: [
+            { 'block_data.block_time': { order: 'asc' } },
+            { operation_id_num: { order: 'asc' } },
+        ],
+    };
+}
+
+/** Fetch every fill_order for an account in [gte, lte], paginated via search_after. */
+async function fetchAllFills(config: any, accountId: string, gte: string, lte: string): Promise<FillRecord[]> {
+    const pageSize = 10000;
+    const fills: FillRecord[] = [];
+    let searchAfter: any[] | null = null;
+    const cfg = { ...BASE_CONFIG, timeout: 60000, ...config };
+
+    while (true) {
+        const query = buildFillQuery(accountId, gte, lte, pageSize);
+        if (searchAfter) (query as any).search_after = searchAfter;
+
+        const result: any = await kibanaSearch(cfg, query);
+        const hits = result?.hits?.hits ?? [];
+        if (!hits.length) break;
+
+        for (const hit of hits) {
+            const src = hit?._source;
+            const op = src?.operation_history?.op_object;
+            if (!op || !op.pays || !op.receives) continue;
+
+            fills.push({
+                time: src.block_data?.block_time ?? '',
+                blockNum: src.block_data?.block_num ?? 0,
+                opNum: Number(src.operation_id_num ?? 0),
+                orderId: op.order_id ?? '',
+                accountId: op.account_id ?? '',
+                pays: { amount: Number(op.pays.amount ?? 0), asset_id: op.pays.asset_id ?? '' },
+                receives: { amount: Number(op.receives.amount ?? 0), asset_id: op.receives.asset_id ?? '' },
+                fee: { amount: Number(op.fee?.amount ?? 0), asset_id: op.fee?.asset_id ?? '' },
+                isMaker: op.is_maker ?? false,
+                sort: hit.sort,
+            });
+        }
+
+        if (hits.length < pageSize) break;
+        searchAfter = hits[hits.length - 1].sort;
+        if (!Array.isArray(searchAfter)) break;
+    }
+
+    return fills;
+}
+
+export {
+    BTS_ID,
+    OP_FILL_ORDER,
+    ASSETS,
+    resolvedPrecisions,
+    assetSymbol,
+    assetPrec,
+    toReal,
+    resolveAssetPrecisions,
+    buildFillQuery,
+    fetchAllFills,
+    AssetInfo,
+    AssetAmount,
+    FillRecord,
+};