dexbot 1.4.16 → 1.4.17

package/modules/README.md

@@ -0,0 +1,176 @@
+# Modules
+
+This is the core runtime of DEXBot2 — the part that actually places orders, tracks fills, and manages the grid on the BitShares blockchain. Entrypoint files (`dexbot.ts`, `bot.ts`, `pm2.ts`, `unlock.ts`, `credential-daemon.ts` in the repo root) launch and orchestrate what lives here.
+
+Everything below covers bot lifecycle, order management, blockchain connectivity, fill processing, grid maintenance, credential security, and MPA/credit workflows — all the code that isn't an entrypoint, adapter, analysis tool, or bridge.
+
+## Where to Start
+
+If you're new to the codebase, read these files in order (~20 minutes):
+
+1. **`constants.ts`** — all tuning parameters and defaults live here; gives you a map of the system's knobs
+2. **`dexbot_class.ts`** — the top-level orchestrator; shows how bot startup, fill processing, and maintenance connect
+3. **`order/manager.ts`** — the central controller for the grid; read the constructor and `_applySafeRebalanceCOW()`
+4. **`order/grid.ts`** — how the geometric grid is generated and sized
+
+After those four, branch out into `order/accounting.ts` (fund tracking), `order/strategy.ts` (rebalancing logic), and `dexbot_fill_runtime.ts` (fill processing pipeline).
+
+## Layout
+
+```
+modules/
+├── dexbot_class.ts                bot lifecycle orchestration
+├── dexbot_cow_runtime.ts          COW rebalance runtime
+├── dexbot_credential_client.ts    credential daemon client
+├── dexbot_fill_runtime.ts         fill processing pipeline
+├── dexbot_maintenance_runtime.ts  sync loops, grid maintenance, triggers
+├── dexbot_startup_runtime.ts      bot startup runtime
+├── dexbot_state_recovery.ts       state recovery after restart
+├── bitshares_client.ts            blockchain connection manager
+├── node_manager.ts                multi-node health and failover
+├── chain_orders.ts                blockchain order operations
+├── account_orders.ts              account order queries
+├── credit_runtime.ts              MPA/credit deal workflow executor
+├── cr_planner.ts                  collateral-ratio planning
+├── credential_runtime.ts          credential daemon lifecycle
+├── credential_policy.ts           signing policy validation
+├── credential_session_cache.ts    encrypted session cache
+├── constants.ts                   central config and tuning params
+├── config.ts                      load-time process.env snapshot
+├── env.ts                         isBrowser/hasProcess detection
+├── runtime.ts                     process abstraction singleton
+├── runtime_settings.ts            per-bot runtime settings resolver
+├── paths.ts                       guarded path resolution
+├── path_api.ts                    portable path abstraction for ESM/browser
+├── settings_merge.ts              consolidated settings merge
+├── fund_registry.ts               shared-account fund registry
+├── authority_resolver.ts          signing key resolution
+├── key_store.ts                   key storage (node-only)
+├── process_discovery.ts           Linux /proc/* filesystem reads
+├── validate_profiles.ts           profile validation
+├── general_settings.ts            global settings loader
+├── bot_settings.ts                per-bot settings loader
+├── graceful_shutdown.ts           signal handling and cleanup
+├── bots_file_lock.ts              profile write serialization
+├── account_bots.ts                bots.json read/write
+├── broadcast_failure.ts           broadcast failure classification
+├── chain_keys.ts                  blockchain key helpers
+├── daemon_node_health.ts          daemon-side node health checks
+├── node_failure_ledger.ts         node failure tracking
+├── node_health_cache.ts           node health tracking
+├── socket_json_client.ts          JSON-RPC socket client
+├── market_adapter_whitelist.ts    adapter whitelist I/O
+├── types.ts                       shared type definitions
+│
+├── crypto/                        cross-environment crypto providers
+│   ├── index.ts                   barrel export
+│   ├── provider.ts                abstract crypto provider interface
+│   ├── browser_provider.ts        Web Crypto based provider
+│   ├── node_provider.ts           Node crypto based provider
+│   ├── sync.ts                    crypto provider lazy loader
+│   ├── pure_ripemd160.ts          pure-JS ripemd160 fallback
+│   ├── pure_scrypt.ts             pure-JS scrypt fallback
+│   └── pure_secp256k1.ts          pure-JS secp256k1 fallback
+│
+├── storage/                       portable storage abstraction
+│   ├── index.ts                   barrel export
+│   ├── types.ts                   storage adapter interface
+│   ├── browser_adapter.ts         in-memory Map-based adapter
+│   └── node_adapter.ts            fs.*Sync based adapter
+│
+├── order/                         order lifecycle and grid management
+│   ├── manager.ts                 OrderManager — central controller, COW rebalance
+│   ├── grid.ts                    grid generation, sizing, health
+│   ├── working_grid.ts            COW grid wrapper for safe mutations
+│   ├── strategy.ts                grid rebalancing, consolidation, rotation
+│   ├── accounting.ts              fund tracking, fee accounting
+│   ├── sync_engine.ts             blockchain sync, fill detection, reconciliation
+│   ├── grid_reconcile.ts          startup grid reconciliation ([doc](../docs/GRID_RECONCILE.md))
+│   ├── grid_reconcile_internal.ts  internal grid reconciliation helpers
+│   ├── index.ts                   barrel export
+│   ├── logger.ts                  order-scoped logging
+│   ├── logger_state.ts            logger state management
+│   ├── processed_fill_store.ts    fill deduplication persistence
+│   ├── async_lock.ts              async mutex
+│   ├── format.ts                  numeric formatting helpers
+│   ├── export.ts                  trade history export
+│   └── utils/                     math, order predicates, validation, system helpers
+│
+├── launcher/                      process lifecycle (PM2, unlock, isolated)
+│   ├── bot_supervisor.ts          per-bot process supervision
+│   ├── child_env.ts               child process environment
+│   ├── credential_bootstrap.ts    daemon startup
+│   ├── credential_daemon.ts       daemon launch controller
+│   ├── credential_secret.ts       secret management
+│   ├── foreign_cred_daemon.ts     external daemon client
+│   ├── headless_password.ts       headless password storage
+│   ├── launch_modes.ts            monolithic vs isolated routing
+│   ├── market_adapter_runtime.ts  adapter process lifecycle
+│   ├── market_adapter_watchdog.ts adapter process watchdog
+│   ├── monolithic_runtime.ts      monolithic bot runtime
+│   ├── runtime_entry.ts           bot process entry
+│   ├── status_reporting.ts        process status reporter
+│   └── supervisor_control.ts      supervisor orchestration
+│
+├── bitshares-native/              native blockchain client
+│   ├── chain_client.ts            blockchain read API
+│   ├── index.ts                   barrel export
+│   ├── lru_cache.ts               LRU cache for blockchain reads
+│   ├── signing_client.ts          transaction signing
+│   ├── transport.ts               WebSocket transport
+│   ├── subscriptions.ts           event subscriptions
+│   ├── resolvers.ts               operation/object resolvers
+│   ├── tx/                        transaction building
+│   ├── serial/                    serialization
+│   └── crypto/                    cryptographic primitives
+│
+└── utils/
+    ├── base58check.ts             Base58Check encoding
+    ├── build_dir.ts               BUILD_DIR constant helper
+    └── errors.ts                  getErrorMessage helper
+```
+
+## Key Relationships
+
+```
+dexbot.ts / bot.ts / pm2.ts / unlock.ts  (entrypoints)
+    │
+    ├── launcher/           process lifecycle, credential daemon
+    │
+    └── dexbot_class.ts     orchestrates:
+         ├── dexbot_fill_runtime.ts        order/ (manager, accounting, grid, sync)
+         ├── dexbot_maintenance_runtime.ts order/ (sync engine, grid reconcile)
+         ├── credit_runtime.ts             cr_planner.ts
+         ├── bitshares_client.ts           node_manager.ts
+         │                                  └── bitshares-native/ (chain_client, signing_client)
+         └── credential_runtime.ts         credential_policy.ts
+```
+
+In plain terms: the **entrypoints** (in the repo root) start processes and hand off to the **launcher**, which boots the credential daemon and bot supervisor. **`dexbot_class.ts`** is the main orchestrator — it wires together the fill runtime (reacts to fills as they happen), the maintenance runtime (periodic sync and grid health checks), and the credit runtime (MPA/debt workflows). The **order subsystem** (`order/`) owns the grid: generating it, syncing it with the blockchain, tracking funds, and rebalancing. The **native client** (`bitshares-native/`) handles all direct blockchain communication so no external trading libraries are needed.
+
+External consumers — other parts of DEXBot2 interact with modules through these touchpoints:
+
+- **`market_adapter/`** — the price adapter reads tuning constants from `constants.ts` and writes trigger files that `dexbot_maintenance_runtime.ts` picks up to decide when to regenerate the grid
+- **`claw/`** — the AI bridge layer reads bot profiles and uses the credential client to sign transactions on behalf of the decision loop
+- **`analysis/`** — research scripts and operational tools: AMA parameter fitting, trend/volatility/regime charting, bot parameter sweep backtests, bot activity queries, and FIFO-based trade PnL analysis
+
+## Design Principles
+
+- **Zero runtime dependencies** — no npm trading/blockchain libraries; the native client, crypto, and serialization are all in `bitshares-native/`. This eliminates supply-chain risk and keeps the bot fully self-contained.
+- **Copy-on-write** — `order/working_grid.ts` provides isolated COW mutations; master grid is immutable during rebalance. The grid is never modified in-place; a working copy is built, mutated, and committed atomically only after blockchain operations succeed.
+- **Fund-driven boundary sync** — the grid rebalances based on available funds, not arbitrary triggers; no forced allocations. When a fill arrives, the system calculates what it can actually afford and adjusts the grid around that.
+- **Replay-safe accounting** — fill processing in `dexbot_fill_runtime.ts` uses `processed_fill_store.ts` to prevent double-counting. If the bot restarts mid-fill, it can safely replay without creating duplicate orders.
+- **Daemon-backed signing** — the credential daemon holds decrypted keys; modules never handle raw private keys. If the main bot crashes, keys stay encrypted on disk — only the small daemon process sees them.
+- **Fixed-cap batch processing** — fill batches are capped (default 4) to keep blockchain broadcasts predictable. Even if 20 fills arrive at once, they're processed in small chunks to avoid overwhelming the chain.
+
+## Related
+
+- [Architecture overview](../docs/architecture.md) — high-level system design and data flows
+- [Developer guide](../docs/developer_guide.md) — glossary, code reading roadmap, and common tasks
+- [Copy-on-write master plan](../docs/COPY_ON_WRITE_MASTER_PLAN.md) — COW grid architecture rationale
+- [COW invariants](../docs/COW_INVARIANTS.md) — safety guarantees for grid mutations
+- [Fund movement & accounting](../docs/FUND_MOVEMENT_AND_ACCOUNTING.md) — fund tracking algorithms and formulas
+- [Credential security](../docs/CREDENTIAL_SECURITY.md) — key handling and daemon-backed signing details
+- [Grid recalculation](../docs/GRID_RECALCULATION.md) — when and how the grid resets or regenerates
+- [Market adapter](../market_adapter/README.md) — AMA pricing, dynamic weights, and signal pipeline
+- [Claw bridge](../claw/README.md) — AI layer integration and command dispatch

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dexbot",
-  "version": "1.4.16",
+  "version": "1.4.17",
   "description": "The first open source trading bot with zero runtime dependencies and a fully adaptive market making strategy.",
   "main": "dist/modules/dexbot_class.js",
   "exports": {
@@ -14,16 +14,50 @@
   },
   "files": [
     "dist",
+    "!dist/analysis",
     "ecosystem.config.cjs",
+    "scripts/bots",
+    "scripts/clean-dist.js",
     "scripts/clear-all.sh",
     "scripts/clear-logs.sh",
     "scripts/clear-market-adapter.sh",
     "scripts/clear-orders.sh",
     "scripts/create-bot-symlinks.sh",
+    "scripts/dexbot",
+    "scripts/git-viewer.sh",
+    "scripts/keys",
     "scripts/lib/dexbot-paths.sh",
+    "scripts/pdev.sh",
+    "scripts/pm2",
+    "scripts/pmain.sh",
     "scripts/postinstall.js",
+    "scripts/ptest.sh",
+    "scripts/README.md",
+    "scripts/repo-stats.html",
     "scripts/reset-settings.sh",
+    "scripts/unlock",
+    "scripts/update.js",
     "claw",
+    "docs/BITSHARES_ONBOARDING.md",
+    "docs/CREDENTIAL_SECURITY.md",
+    "docs/GRID_RECALCULATION.md",
+    "docs/GRID_RECONCILE.md",
+    "docs/LOGGING.md",
+    "docs/MPA_CREDIT_USAGE.md",
+    "docs/COPY_ON_WRITE_MASTER_PLAN.md",
+    "docs/COW_INVARIANTS.md",
+    "docs/FUND_MOVEMENT_AND_ACCOUNTING.md",
+    "docs/DEXBOT_COMPARISON.md",
+    "docs/EVOLUTION.md",
+    "docs/WORKFLOW.md",
+    "docs/architecture.md",
+    "docs/developer_guide.md",
+    "docs/docker.md",
+    "market_adapter/README.md",
+    "modules/README.md",
+    "analysis",
+    "tests/README.md",
+    "CHANGELOG.md",
     "README.md",
     "LICENSE"
   ],
@@ -127,7 +161,6 @@
     "./dist/modules/daemon_node_health.js": false,
     "./dist/modules/storage/node_adapter.js": false,
     "./dist/modules/key_store.js": false,
-    "./dist/modules/logger.js": false,
     "./dist/modules/paths.js": false,
     "./dist/modules/order/logger.js": false,
     "./dist/modules/order/export.js": false,
@@ -144,9 +177,11 @@
   "tsx": {
     "tsconfig": true
   },
+  "dependencies": {
+    "tsx": "^4.22.3"
+  },
   "devDependencies": {
     "@types/node": "^25.9.1",
-    "tsx": "^4.22.3",
     "typescript": "^6.0.3"
   }
 }

package/scripts/README.md

@@ -238,7 +238,6 @@ npm run verify:browser-bundle
 ```bash
 bash scripts/create-bot-symlinks.sh
 ```
-Auto-runs on `npm postinstall` for global installs.
 
 ### Version Sync
 **File:** `sync-version.ts`
@@ -254,6 +253,10 @@ npm run version:sync
 
 ## 🌳 BRANCH SYNCHRONIZATION
 
+> **Git-only:** These scripts require a `.git` directory and will not work with
+> global npm package installs (`npm install -g dexbot`). They are intended for
+> source checkout workflows only.
+
 ### Synchronize test → dev → main
 **File:** `pmain.sh` (also: `npm run pmain`)
 **Purpose:** Sync local test branch through dev to main remote.
@@ -292,41 +295,61 @@ The following scripts allow you to call `dexbot` commands directly from the `scr
 
 | Wrapper | Target Command | Usage |
 |:---|:---|:---|
-| `scripts/bots` | `dexbot bot` | `./scripts/bots` |
-| `scripts/keys` | `dexbot key` | `./scripts/keys` |
-| `scripts/dexbot` | `dexbot` | `./dexbot <cmd>` |
-| `scripts/unlock` | `./unlock` | `./unlock` |
-| `scripts/pm2` | `./pm2` | `./pm2` |
+| `scripts/bots` | `dexbot bots` | `./scripts/bots` |
+| `scripts/keys` | `dexbot keys` | `./scripts/keys` |
+| `scripts/dexbot` | `dexbot` | `./scripts/dexbot <cmd>` |
+| `scripts/unlock` | `unlock.ts` / `dist/unlock.js` | `./scripts/unlock` |
+| `scripts/pm2` | `pm2.ts` / `dist/pm2.js` | `./scripts/pm2` |
 
 ---
 
 ## 📦 NPM SCRIPTS
 
-| Command | File | Purpose |
-|:---|:---|:---|
-| `npm test` | `scripts/run-tests.ts` | Run full test suite (excludes live-chain tests) |
-| `npm run test:live` | `scripts/run-tests.ts` | Run full test suite including live-chain tests |
-| `npm run typecheck` | — | TypeScript type checking (`tsc --noEmit`) |
-| `npm run build` | — | Clean + compile TypeScript |
-| `npm run clean` | `scripts/clean-dist.js` | Remove compiled `dist/` output |
-| `npm run ptest` | `scripts/ptest.sh` | Sync local test → origin/test |
-| `npm run pdev` | `scripts/pdev.sh` | Sync local test → dev |
-| `npm run pmain` | `scripts/pmain.sh` | Sync local test → dev → main |
-| `npm run unlock` | `unlock.ts` | Build + single-prompt credential unlock (full bot) |
-| `npm run claw:unlock` | `unlock.ts` | Build + single-prompt unlock (claw-only mode) |
-| `npm run pm2:unlock` | `pm2.ts` | Build + launch full bot via PM2 ecosystem |
-| `npm run pm2:claw-only` | `pm2.ts` | Build + launch claw-only PM2 process |
-| `npm run lp:chart` | `scripts/generate_lp_chart.ts` | Generate uPlot LP chart |
-| `npm run market-adapter:whitelist` | `scripts/generate_market_adapter_whitelist.ts` | Generate/update whitelist from AMA-configured bots |
-| `npm run analysis:derivatives` | `analysis/analyze_derivatives.ts` | Derivative analysis report |
-| `npm run version:sync` | `scripts/sync-version.ts` | Rewrite plugin/manifest versions from root `package.json` |
-| `npm run version:check` | `scripts/sync-version.ts --check` | Verify all version manifests match root `package.json` |
-| `npm run native:release-gates` | `scripts/native_release_gates.ts` | Verify native library build, linkage, and mainnet corpus round-trips |
-| `npm run native:serial-snapshots` | `tests/test_native_serial_ops.ts` | Pin wire-format bytes for signed operation types |
-| `npm run native:ecc-invariants` | `tests/test_native_ecc.ts` | Validate ECDH key derivation, ECDSA signing, WIF, brain key, and hash functions |
-| `npm run native:corpus` | `scripts/generate_mainnet_corpus_report.ts` | Generate `profiles/native_validation/mainnet_corpus_report.json` by reserializing recent mainnet transactions through the native serializer and diffing against the chain's own `get_transaction_hex` output (and `block.transaction_ids` when the node exposes them) |
-| `npm run test:credit-renewal` | `scripts/test-credit-renewal.ts` | Test credit offer renewal for a specific bot |
-| `npm run verify:browser-bundle` | `scripts/verify-browser-bundle.ts` | Verify browser-safe surface bundles correctly |
+### Build & Test
+| Command | Purpose |
+|:---|:---|
+| `npm run build` | Clean + compile TypeScript |
+| `npm run clean` | Remove compiled `dist/` output |
+| `npm run typecheck` | TypeScript type checking (`tsc --noEmit`) |
+| `npm run build:watch` | TypeScript incremental build watcher |
+| `npm test` | Run full test suite (excludes live-chain tests) |
+| `npm run test:live` | Run full test suite including live-chain tests |
+
+### Runtime
+| Command | Purpose |
+|:---|:---|
+| `npm run unlock` | Build + single-prompt credential unlock (full bot) |
+| `npm run claw:unlock` | Build + single-prompt unlock (claw-only mode) |
+| `npm run pm2:unlock` | Build + launch full bot via PM2 ecosystem |
+| `npm run pm2:claw-only` | Build + launch claw-only PM2 process |
+| `npm run pm2:start` | Start PM2 processes from ecosystem config |
+| `npm run pm2:stop` | Stop PM2 processes from ecosystem config |
+
+### Branch Sync *(git checkout only)*
+| Command | Purpose |
+|:---|:---|
+| `npm run ptest` | Push local test → origin/test |
+| `npm run pdev` | Sync local test → dev |
+| `npm run pmain` | Sync local test → dev → main |
+
+### Version
+| Command | Purpose |
+|:---|:---|
+| `npm run version:sync` | Rewrite plugin/manifest versions from root `package.json` |
+| `npm run version:check` | Verify all version manifests match root `package.json` |
+
+### Analysis
+| Command | Purpose |
+|:---|:---|
+| `npm run market-adapter:whitelist` | Generate/update whitelist from AMA-configured bots |
+| `npm run market-adapter:fetch-cex-synthetic` | Fetch CEX synthetic data for market adapter |
+| `npm run analysis:derivatives` | Derivative analysis report |
+| `npm run analysis:tradingview` | TradingView-style chart export |
+| `npm run analysis:trade-pnl` | Trade PnL analysis from fill data |
+| `npm run ama:chart:lp-local` | Generate local LP comparison chart |
+| `npm run lp:chart` | Generate uPlot LP chart |
+| `npm run test:credit-renewal` | Test credit offer renewal for a specific bot |
+| `npm run verify:browser-bundle` | Verify browser-safe surface bundles correctly |
 
 ---
 
@@ -363,7 +386,7 @@ npm run analysis:derivatives -- --source json --file <file.json>
 ## 📚 DOCUMENTATION REFERENCES
 
 For understanding the systems these scripts interact with:
-- **Module Architecture**: See [root README 📦 Modules section](../README.md#-modules)
+- **Module Architecture**: See [modules/README.md](../modules/README.md)
 - **Copy-on-Write Pattern**: See [docs/COPY_ON_WRITE_MASTER_PLAN.md](../docs/COPY_ON_WRITE_MASTER_PLAN.md) for rebalancing architecture
 - **Fund Accounting**: See [docs/FUND_MOVEMENT_AND_ACCOUNTING.md](../docs/FUND_MOVEMENT_AND_ACCOUNTING.md)
 - **Grid Divergence**: See [docs](../docs/README.md) for RMS threshold explanations

package/scripts/bots

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+process.argv[2] = 'bots';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { createRequire } from 'node:module';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+const root = path.resolve(__dirname, '..');
+const distTarget = path.join(root, 'dist', 'dexbot.js');
+
+if (existsSync(distTarget)) {
+  await import(pathToFileURL(distTarget).href);
+} else {
+  require('tsx/cjs');
+  require(path.join(root, 'dexbot.ts'));
+}

package/scripts/clean-dist.js

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+import { rmSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const distDir = join(__dirname, '..', 'dist');
+
+rmSync(distDir, { recursive: true, force: true });

package/scripts/create-bot-symlinks.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # create-bot-symlinks.sh - Create ecosystem config symlinks for each bot
 #
-# This allows: pm2 start bot-name (or: pm2 start bot-name.config.js)
+# This allows: pm2 start bot-name (or: pm2 start bot-name.config.cjs)
 #
 # Usage: ./scripts/create-bot-symlinks.sh
 
@@ -35,7 +35,7 @@ const ecoConfig = '$ECOSYSTEM_CONFIG';
 
 (botsConfig.bots || []).forEach(bot => {
     const botName = bot.name;
-    const symlink = path.join(profilesDir, botName + '.config.js');
+    const symlink = path.join(profilesDir, botName + '.config.cjs');
 
     // Remove old symlink if exists
     if (fs.existsSync(symlink)) {
@@ -45,7 +45,7 @@ const ecoConfig = '$ECOSYSTEM_CONFIG';
     // Create symlink
     try {
         fs.symlinkSync(ecoConfig, symlink);
-        console.log('✓ Created symlink: profiles/' + botName + '.config.cjs -> ecosystem.config.cjs');
+        console.log('✓ Created symlink: profiles/' + botName + '.config.cjs -> ' + path.basename(ecoConfig));
     } catch (err) {
         console.error('✗ Error creating symlink for ' + botName + ':', err.message);
     }

package/scripts/dexbot

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { createRequire } from 'node:module';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+const root = path.resolve(__dirname, '..');
+const distTarget = path.join(root, 'dist', 'dexbot.js');
+
+if (existsSync(distTarget)) {
+  await import(pathToFileURL(distTarget).href);
+} else {
+  require('tsx/cjs');
+  require(path.join(root, 'dexbot.ts'));
+}