openbroker 1.2.0 → 1.3.0

package/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 All notable changes to Open Broker will be documented in this file.
 
+## [1.3.0] - 2026-05-07
+
+### Added
+- **`openbroker auto prune`**: New subcommand for trimming the local audit DB
+  - `--older-than <duration>` accepts `7d`, `24h`, `30m`, etc.
+  - `--status <list>` filters by run status (default: `stopped,error,stale`)
+  - `--keep-last <N>` retains the N most-recent runs per `automation_id`
+  - `--all` deletes everything except runs whose process is still alive
+  - `--vacuum` reclaims disk after deletion
+  - `--dry` previews matches without writing — also reconcile is dry-safe
+
+### Changed
+- **`openbroker auto clean`**: Now also reconciles the audit DB. Runs whose pid is dead but whose row still says `status='running'` get marked `stopped` with `stop_reason='reconciled (process exited)'`, fixing dashboards that previously kept rendering them as live/stale long after `auto stop`.
+
 ## [1.0.59] - 2026-03-11
 
 ### Fixed

package/bin/openbroker.js

@@ -25,6 +25,9 @@ const child = spawn(
       OPENBROKER_CWD: process.cwd(),
       // Suppress Node.js experimental warnings
       NODE_NO_WARNINGS: '1',
+      // node:sqlite is stable in Node 24+, experimental-with-flag in 22/23.
+      // The flag is accepted (as a no-op) on 24+, so it's safe to set unconditionally.
+      NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ''} --experimental-sqlite`.trim(),
     },
   }
 );
@@ -42,6 +45,7 @@ child.on('error', (err) => {
           ...process.env,
           OPENBROKER_CWD: process.cwd(),
           NODE_NO_WARNINGS: '1',
+          NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ''} --experimental-sqlite`.trim(),
         },
       }
     );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openbroker",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Hyperliquid trading CLI - execute orders, manage positions, and run trading strategies",
   "type": "module",
   "bin": {
@@ -49,6 +49,7 @@
   "dependencies": {
     "@nktkas/hyperliquid": "^0.30.3",
     "dotenv": "^17.2.3",
+    "openbroker-monitoring": "file:../openbroker-monitoring",
     "tsx": "^4.19.0",
     "viem": "^2.21.0"
   },

package/scripts/auto/cli.ts

@@ -23,7 +23,8 @@ Usage:
   openbroker auto stop <id>                 Unregister an automation (won't restart)
   openbroker auto list                      List available automations
   openbroker auto status                    Show running automations
-  openbroker auto clean                     Remove stale entries from registry
+  openbroker auto clean                     Remove stale entries from registry + reconcile audit DB
+  openbroker auto prune [options]           Delete stale runs from the audit DB
 
 Options (for run):
   --example <name>   Run a bundled example (dca, grid, funding-arb, mm-spread, mm-maker)
@@ -34,6 +35,14 @@ Options (for run):
   --poll <ms>        Poll interval in milliseconds (default: 10000)
   --no-ws            Disable WebSocket; fall back to REST-only polling
 
+Options (for prune):
+  --older-than <d>   Only prune runs started before this duration ago (e.g. 7d, 24h)
+  --status <list>    CSV of statuses to consider (default: stopped,error,stale)
+  --keep-last <N>    Keep the N most-recent runs per automation_id
+  --all              Prune everything except runs that are still alive
+  --vacuum           VACUUM the DB after deletion to reclaim disk space
+  --dry              Preview what would be deleted without writing
+
 Scripts are loaded from:
   1. Absolute or relative path
   2. ~/.openbroker/automations/<name>.ts
@@ -288,9 +297,83 @@ function stopCommand(positional: string[]) {
   console.log(`Unregistered: ${id} (will not restart on next gateway start)`);
 }
 
-function cleanCommand() {
+async function cleanCommand() {
   cleanRegistry();
   console.log('Cleaned stale entries from registry');
+
+  // Also reconcile the audit DB so dead processes whose rows still say
+  // 'running' get marked 'stopped'. Without this, the dashboard keeps showing
+  // 'stale' badges for automations the operator already cleaned out of the
+  // registry.
+  try {
+    const { prune } = await import('./prune.js');
+    const result = prune({ reconcileOnly: true });
+    if (result.reconciled > 0) {
+      console.log(`Reconciled ${result.reconciled} orphan run row${result.reconciled === 1 ? '' : 's'} in audit DB (status: running → stopped)`);
+    } else {
+      console.log('Audit DB already consistent');
+    }
+  } catch (err) {
+    console.warn(`Could not reconcile audit DB: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+async function pruneCommand(args: Record<string, string | boolean>) {
+  const { fmtBytes, parseDuration, prune } = await import('./prune.js');
+  const olderThanRaw = args['older-than'];
+  const olderThanMs = typeof olderThanRaw === 'string' ? parseDuration(olderThanRaw) : undefined;
+  const statusesRaw = typeof args.status === 'string' ? args.status : undefined;
+  const statuses = statusesRaw
+    ? new Set(statusesRaw.split(',').map((s) => s.trim()).filter(Boolean))
+    : undefined;
+  const keepLastRaw = args['keep-last'];
+  const keepLast = typeof keepLastRaw === 'string' ? Number(keepLastRaw)
+    : typeof keepLastRaw === 'number' ? keepLastRaw
+    : undefined;
+  if (keepLast !== undefined && (!Number.isFinite(keepLast) || keepLast < 0)) {
+    console.error('Error: --keep-last must be a non-negative integer');
+    process.exit(1);
+  }
+
+  const opts = {
+    olderThanMs,
+    statuses,
+    keepLastPerAutomation: keepLast,
+    all: args.all === true,
+    vacuum: args.vacuum === true,
+    dryRun: args.dry === true,
+  };
+
+  const result = prune(opts);
+
+  if (result.reconciled > 0) {
+    const verb = result.dryRun ? 'Would reconcile' : 'Reconciled';
+    console.log(`${verb} ${result.reconciled} orphan run row${result.reconciled === 1 ? '' : 's'} (running → stopped)`);
+  }
+
+  const verb = result.dryRun ? 'Would delete' : 'Deleted';
+  if (result.candidateRunIds.length === 0) {
+    console.log('No runs matched pruning filters.');
+    return;
+  }
+  console.log(`${verb} ${result.candidateRunIds.length} automation run${result.candidateRunIds.length === 1 ? '' : 's'}:`);
+  for (const id of result.candidateRunIds.slice(0, 25)) {
+    console.log(`  · ${id}`);
+  }
+  if (result.candidateRunIds.length > 25) {
+    console.log(`  … and ${result.candidateRunIds.length - 25} more`);
+  }
+  if (!result.dryRun) {
+    const totalChild = Object.values(result.deletedRows).reduce((a, b) => a + b, 0);
+    console.log(`Removed ${totalChild.toLocaleString()} child rows across ${Object.keys(result.deletedRows).length} tables`);
+    if (result.freedBytes > 0) {
+      console.log(`Reclaimed ~${fmtBytes(result.freedBytes)}${opts.vacuum ? ' (post-VACUUM)' : ''}`);
+    } else if (opts.vacuum) {
+      console.log('No disk reclaimed (VACUUM completed)');
+    } else {
+      console.log('Run again with --vacuum to reclaim disk space.');
+    }
+  }
 }
 
 function reportCommand(rawArgs: string[]) {
@@ -364,7 +447,10 @@ async function main() {
       statusCommand();
       break;
     case 'clean':
-      cleanCommand();
+      await cleanCommand();
+      break;
+    case 'prune':
+      await pruneCommand(args);
       break;
     case 'report':
       reportCommand(restArgs);

package/scripts/auto/prune.ts

@@ -0,0 +1,252 @@
+// Audit DB pruning — delete stale automation runs and their child rows.
+//
+// The audit DB is the same SQLite file the daemon writes to and the dashboard
+// reads from. WAL mode lets us open it from another process for delete writes
+// without blocking the daemon. We always protect runs whose status is 'running'
+// AND whose pid is alive.
+//
+// Used by: `openbroker auto prune` and as a sub-step of `openbroker auto clean`.
+
+import os from 'os';
+import path from 'path';
+import { existsSync } from 'fs';
+import { DatabaseSync } from 'node:sqlite';
+import { ensureConfigDir } from '../core/config.js';
+
+export const AUDIT_DB_PATH = process.env.OPENBROKER_AUDIT_DB_PATH
+  || path.join(ensureConfigDir(), 'automation-audit.sqlite');
+
+export interface PruneFilters {
+  /** Delete runs whose started_at < (now - olderThanMs). Falsy = no age filter. */
+  olderThanMs?: number;
+  /** Delete runs whose status is in this set. Default: stopped, error, stale. */
+  statuses?: Set<string>;
+  /** For each automation_id, keep the N most recent runs regardless of other filters. */
+  keepLastPerAutomation?: number;
+  /** Delete every run that is not currently alive (overrides status/age). */
+  all?: boolean;
+}
+
+export interface PruneOptions extends PruneFilters {
+  dbPath?: string;
+  dryRun?: boolean;
+  vacuum?: boolean;
+  /**
+   * When true, skip the deletion phase and only update status of orphaned
+   * 'running' rows whose pid is dead — used by `auto clean` to reconcile state
+   * without losing history.
+   */
+  reconcileOnly?: boolean;
+}
+
+export interface PruneResult {
+  reconciled: number;
+  candidateRunIds: string[];
+  deletedRows: Record<string, number>;
+  freedBytes: number;
+  dryRun: boolean;
+}
+
+const CHILD_TABLES = [
+  'automation_logs',
+  'automation_events',
+  'automation_actions',
+  'automation_snapshots',
+  'automation_order_updates',
+  'automation_fills',
+  'automation_user_events',
+  'automation_state_changes',
+  'automation_publishes',
+  'automation_errors',
+  'automation_notes',
+  'automation_metrics',
+] as const;
+
+const DEFAULT_STATUSES = new Set(['stopped', 'error', 'stale']);
+
+function isProcessAlive(pid: number | null | undefined): boolean {
+  if (typeof pid !== 'number' || !Number.isFinite(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Parse human-friendly durations like `7d`, `24h`, `30m`, `45s`. */
+export function parseDuration(input: string): number {
+  const m = /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h|d|w)?\s*$/.exec(input);
+  if (!m) throw new Error(`invalid duration: ${input}`);
+  const n = Number(m[1]);
+  const unit = m[2] ?? 'ms';
+  const mult: Record<string, number> = {
+    ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000, w: 7 * 86_400_000,
+  };
+  return n * mult[unit];
+}
+
+/** Reconcile orphan-running rows in the DB (process dead → mark stopped). */
+export function reconcileStaleRuns(db: DatabaseSync, opts: { dryRun?: boolean; now?: number } = {}): number {
+  const now = opts.now ?? Date.now();
+  const runningRows = db.prepare(`
+    SELECT run_id, pid FROM automation_runs WHERE status = 'running'
+  `).all() as { run_id: string; pid: number | null }[];
+
+  const orphans = runningRows.filter((r) => !isProcessAlive(r.pid));
+  if (orphans.length === 0) return 0;
+  if (opts.dryRun) return orphans.length;
+
+  const update = db.prepare(`
+    UPDATE automation_runs
+    SET status = 'stopped',
+        stop_reason = COALESCE(stop_reason, 'reconciled (process exited)'),
+        stopped_at = COALESCE(stopped_at, ?)
+    WHERE run_id = ?
+  `);
+  let n = 0;
+  for (const o of orphans) {
+    update.run(now, o.run_id);
+    n++;
+  }
+  return n;
+}
+
+export function prune(opts: PruneOptions = {}): PruneResult {
+  const dbPath = opts.dbPath ?? AUDIT_DB_PATH;
+  if (!existsSync(dbPath)) {
+    return {
+      reconciled: 0,
+      candidateRunIds: [],
+      deletedRows: {},
+      freedBytes: 0,
+      dryRun: !!opts.dryRun,
+    };
+  }
+
+  const db = new DatabaseSync(dbPath);
+  db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;');
+
+  try {
+    const reconciled = reconcileStaleRuns(db, { dryRun: opts.dryRun });
+    if (opts.reconcileOnly) {
+      return {
+        reconciled,
+        candidateRunIds: [],
+        deletedRows: {},
+        freedBytes: 0,
+        dryRun: !!opts.dryRun,
+      };
+    }
+
+    const allRuns = db.prepare(`
+      SELECT run_id, automation_id, status, pid, started_at
+      FROM automation_runs
+      ORDER BY started_at DESC
+    `).all() as {
+      run_id: string;
+      automation_id: string;
+      status: string;
+      pid: number | null;
+      started_at: number;
+    }[];
+
+    const statuses = opts.statuses ?? DEFAULT_STATUSES;
+    const cutoff = opts.olderThanMs && opts.olderThanMs > 0 ? Date.now() - opts.olderThanMs : null;
+
+    // group runs per automation_id (already sorted DESC by started_at)
+    const byAuto = new Map<string, typeof allRuns>();
+    for (const r of allRuns) {
+      const arr = byAuto.get(r.automation_id) ?? [];
+      arr.push(r);
+      byAuto.set(r.automation_id, arr);
+    }
+
+    const candidates: typeof allRuns = [];
+
+    for (const [, runs] of byAuto) {
+      const protectedIdx = new Set<number>();
+      if (opts.keepLastPerAutomation && opts.keepLastPerAutomation > 0) {
+        for (let i = 0; i < Math.min(opts.keepLastPerAutomation, runs.length); i++) {
+          protectedIdx.add(i);
+        }
+      }
+      runs.forEach((r, i) => {
+        if (protectedIdx.has(i)) return;
+        // never delete a truly-running automation
+        if (r.status === 'running' && isProcessAlive(r.pid)) return;
+        if (opts.all) {
+          candidates.push(r);
+          return;
+        }
+        if (!statuses.has(r.status)) {
+          // 'running' rows that aren't actually alive were just reconciled to
+          // 'stopped' above, so the status check catches them.
+          return;
+        }
+        if (cutoff !== null && r.started_at >= cutoff) return;
+        candidates.push(r);
+      });
+    }
+
+    const candidateRunIds = candidates.map((r) => r.run_id);
+    const deletedRows: Record<string, number> = {};
+    let freedBytes = 0;
+
+    if (!opts.dryRun && candidateRunIds.length > 0) {
+      const sizeBefore = db.prepare('PRAGMA page_count').get() as { page_count: number };
+      const pageSize = db.prepare('PRAGMA page_size').get() as { page_size: number };
+
+      db.exec('BEGIN');
+      try {
+        for (const table of CHILD_TABLES) {
+          let n = 0;
+          const stmt = db.prepare(`DELETE FROM ${table} WHERE run_id = ?`);
+          for (const id of candidateRunIds) {
+            const info = stmt.run(id) as { changes?: number };
+            n += Number(info.changes ?? 0);
+          }
+          deletedRows[table] = n;
+        }
+        const runStmt = db.prepare('DELETE FROM automation_runs WHERE run_id = ?');
+        let runChanges = 0;
+        for (const id of candidateRunIds) {
+          const info = runStmt.run(id) as { changes: number };
+          runChanges += Number(info.changes ?? 0);
+        }
+        deletedRows.automation_runs = runChanges;
+        db.exec('COMMIT');
+      } catch (err) {
+        db.exec('ROLLBACK');
+        throw err;
+      }
+
+      if (opts.vacuum) {
+        // VACUUM cannot run inside a transaction
+        db.exec('VACUUM');
+      }
+
+      const sizeAfter = db.prepare('PRAGMA page_count').get() as { page_count: number };
+      freedBytes = (Number(sizeBefore.page_count) - Number(sizeAfter.page_count)) * Number(pageSize.page_size);
+    }
+
+    return {
+      reconciled,
+      candidateRunIds,
+      deletedRows,
+      freedBytes: Math.max(0, freedBytes),
+      dryRun: !!opts.dryRun,
+    };
+  } finally {
+    db.close();
+  }
+}
+
+export function fmtBytes(n: number): string {
+  if (!Number.isFinite(n) || n <= 0) return '0 B';
+  const u = ['B', 'KB', 'MB', 'GB'];
+  let i = 0;
+  let v = n;
+  while (v >= 1024 && i < u.length - 1) { v /= 1024; i++; }
+  return `${v.toFixed(v >= 100 || i === 0 ? 0 : 1)} ${u[i]}`;
+}

package/scripts/auto/runtime.ts

@@ -288,6 +288,12 @@ function createAuditedClient(
 async function buildSnapshot(
   client: HyperliquidClient,
 ): Promise<AutomationSnapshot> {
+  // `metaAndAssetCtxs` contains both mostly-static market metadata and live
+  // funding/premium values. The client caches it for market lookups, so clear
+  // it before each automation poll snapshot or funding_update events freeze at
+  // the first fetched value.
+  client.invalidateMetaCache();
+
   const [state, mids, metaCtxs] = await Promise.all([
     client.getUserStateAll(),
     client.getAllMids(),
@@ -321,20 +327,29 @@ async function buildSnapshot(
 
   // Build funding rates from asset contexts
   const fundingRates = new Map<string, { rate: number; premium: number }>();
+  const addFundingRates = (
+    universe: Array<{ name?: string }> | undefined,
+    assetCtxs: Array<{ funding?: string | number | null; premium?: string | number | null }> | undefined,
+  ) => {
+    if (!universe || !assetCtxs) return;
+    for (let i = 0; i < universe.length; i++) {
+      const meta = universe[i];
+      const ctx = assetCtxs[i];
+      if (ctx && meta?.name) {
+        fundingRates.set(meta.name, {
+          rate: parseFloat(String(ctx.funding || '0')),
+          premium: parseFloat(String(ctx.premium || '0')),
+        });
+      }
+    }
+  };
+
   if (metaCtxs && Array.isArray(metaCtxs)) {
     for (const group of metaCtxs) {
-      if (!group.universe || !group.assetCtxs) continue;
-      for (let i = 0; i < group.universe.length; i++) {
-        const meta = group.universe[i];
-        const ctx = group.assetCtxs[i];
-        if (ctx && meta) {
-          fundingRates.set(meta.name, {
-            rate: parseFloat(ctx.funding || '0'),
-            premium: parseFloat(ctx.premium || '0'),
-          });
-        }
-      }
+      addFundingRates(group.universe, group.assetCtxs);
     }
+  } else if (metaCtxs) {
+    addFundingRates(metaCtxs.meta?.universe, metaCtxs.assetCtxs);
   }
 
   return {

package/scripts/core/client.ts

@@ -1067,7 +1067,11 @@ export class HyperliquidClient {
 
     const response = await fetch(baseUrl + '/info', {
       method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
+      headers: {
+        'Content-Type': 'application/json',
+        'Cache-Control': 'no-cache, no-store, max-age=0',
+        Pragma: 'no-cache',
+      },
       body: JSON.stringify({ type: 'predictedFundings' }),
     });
     const data = await response.json() as Array<[