@senpi-ai/runtime 3.0.104 → 3.0.106

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/cli/guide-commands.d.ts.map +1 -1
  2. package/dist/cli/guide-commands.js +38 -0
  3. package/dist/cli/guide-commands.js.map +1 -1
  4. package/dist/cli/senpi-commands.d.ts.map +1 -1
  5. package/dist/cli/senpi-commands.js +29 -1
  6. package/dist/cli/senpi-commands.js.map +1 -1
  7. package/dist/cli/update-command.d.ts +69 -0
  8. package/dist/cli/update-command.d.ts.map +1 -0
  9. package/dist/cli/update-command.js +153 -0
  10. package/dist/cli/update-command.js.map +1 -0
  11. package/dist/dsl/plugin/index.d.ts +35 -1
  12. package/dist/dsl/plugin/index.d.ts.map +1 -1
  13. package/dist/dsl/plugin/index.js +62 -7
  14. package/dist/dsl/plugin/index.js.map +1 -1
  15. package/dist/index.d.ts +13 -0
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +212 -11
  18. package/dist/index.js.map +1 -1
  19. package/dist/runtime/effective-config.d.ts +67 -0
  20. package/dist/runtime/effective-config.d.ts.map +1 -0
  21. package/dist/runtime/effective-config.js +324 -0
  22. package/dist/runtime/effective-config.js.map +1 -0
  23. package/dist/runtime/lifecycle-events.d.ts +19 -0
  24. package/dist/runtime/lifecycle-events.d.ts.map +1 -1
  25. package/dist/runtime/lifecycle-events.js +27 -0
  26. package/dist/runtime/lifecycle-events.js.map +1 -1
  27. package/dist/runtime/rebuild-errors.d.ts +26 -0
  28. package/dist/runtime/rebuild-errors.d.ts.map +1 -0
  29. package/dist/runtime/rebuild-errors.js +36 -0
  30. package/dist/runtime/rebuild-errors.js.map +1 -0
  31. package/dist/runtime/run.d.ts +10 -0
  32. package/dist/runtime/run.d.ts.map +1 -1
  33. package/dist/runtime/run.js +13 -1
  34. package/dist/runtime/run.js.map +1 -1
  35. package/dist/runtime/runtime.d.ts +140 -1
  36. package/dist/runtime/runtime.d.ts.map +1 -1
  37. package/dist/runtime/runtime.js +494 -149
  38. package/dist/runtime/runtime.js.map +1 -1
  39. package/dist/runtime/strategy-registry.d.ts +1 -0
  40. package/dist/runtime/strategy-registry.d.ts.map +1 -1
  41. package/dist/runtime/strategy-registry.js +19 -0
  42. package/dist/runtime/strategy-registry.js.map +1 -1
  43. package/dist/runtime/update-apply-deps.d.ts +101 -0
  44. package/dist/runtime/update-apply-deps.d.ts.map +1 -0
  45. package/dist/runtime/update-apply-deps.js +159 -0
  46. package/dist/runtime/update-apply-deps.js.map +1 -0
  47. package/dist/runtime/update-handler.d.ts +197 -0
  48. package/dist/runtime/update-handler.d.ts.map +1 -0
  49. package/dist/runtime/update-handler.js +381 -0
  50. package/dist/runtime/update-handler.js.map +1 -0
  51. package/dist/runtime/update-plan.d.ts +60 -0
  52. package/dist/runtime/update-plan.d.ts.map +1 -0
  53. package/dist/runtime/update-plan.js +203 -0
  54. package/dist/runtime/update-plan.js.map +1 -0
  55. package/dist/runtime/update-register.d.ts +67 -0
  56. package/dist/runtime/update-register.d.ts.map +1 -0
  57. package/dist/runtime/update-register.js +173 -0
  58. package/dist/runtime/update-register.js.map +1 -0
  59. package/dist/runtime/update-report.d.ts +125 -0
  60. package/dist/runtime/update-report.d.ts.map +1 -0
  61. package/dist/runtime/update-report.js +215 -0
  62. package/dist/runtime/update-report.js.map +1 -0
  63. package/dist/utils/event-catalog.d.ts +26 -0
  64. package/dist/utils/event-catalog.d.ts.map +1 -1
  65. package/dist/utils/event-catalog.js +1 -0
  66. package/dist/utils/event-catalog.js.map +1 -1
  67. package/dist/widgets/errors.d.ts +11 -0
  68. package/dist/widgets/errors.d.ts.map +1 -0
  69. package/dist/widgets/errors.js +16 -0
  70. package/dist/widgets/errors.js.map +1 -0
  71. package/dist/widgets/index.d.ts +32 -0
  72. package/dist/widgets/index.d.ts.map +1 -0
  73. package/dist/widgets/index.js +33 -0
  74. package/dist/widgets/index.js.map +1 -0
  75. package/dist/widgets/registry.d.ts +24 -0
  76. package/dist/widgets/registry.d.ts.map +1 -0
  77. package/dist/widgets/registry.js +33 -0
  78. package/dist/widgets/registry.js.map +1 -0
  79. package/dist/widgets/tool.d.ts +48 -0
  80. package/dist/widgets/tool.d.ts.map +1 -0
  81. package/dist/widgets/tool.js +140 -0
  82. package/dist/widgets/tool.js.map +1 -0
  83. package/dist/widgets/types.d.ts +53 -0
  84. package/dist/widgets/types.d.ts.map +1 -0
  85. package/dist/widgets/types.js +10 -0
  86. package/dist/widgets/types.js.map +1 -0
  87. package/dist/widgets/widgets/fund-user-wallet.d.ts +12 -0
  88. package/dist/widgets/widgets/fund-user-wallet.d.ts.map +1 -0
  89. package/dist/widgets/widgets/fund-user-wallet.js +38 -0
  90. package/dist/widgets/widgets/fund-user-wallet.js.map +1 -0
  91. package/openclaw.plugin.json +13 -1
  92. package/package.json +1 -1
@@ -18,13 +18,15 @@ import { resolveStrategyConfig } from "./map-strategies.js";
18
18
  import { mapRuntimeExitToDslConfig } from "./map-exit-config.js";
19
19
  import { buildActionScanPayload } from "./build-action-scan-payload.js";
20
20
  import { createScannerModuleFromRuntimeConfig } from "./create-scanner-module.js";
21
- import { createRuntimeContext, createStrategyStatesMap } from "./create-runtime-context.js";
21
+ import { createRuntimeContext, createStrategyState, createStrategyStatesMap, } from "./create-runtime-context.js";
22
22
  import { createActions } from "./create-actions.js";
23
23
  import { ActionRuntimeModule } from "../actions/action-runtime-module.js";
24
24
  import { validateDslPositionTracking } from "./validate-dsl-position-tracking.js";
25
25
  import { ScannerRegistry } from "./scanner-registry.js";
26
26
  import { ActionRegistry } from "./action-registry.js";
27
27
  import { initRisk } from "../risk/index.js";
28
+ import { parseDslPluginConfig } from "../dsl/config/parse-plugin-config.js";
29
+ import { RuntimeRebuildRollbackFailedError } from "./rebuild-errors.js";
28
30
  import { createHealthStateModule, } from "../health/index.js";
29
31
  import { DslPlugin as DslPluginClass } from "../dsl/plugin/index.js";
30
32
  import { HOOK_EVENT } from "../types/hooks.js";
@@ -58,6 +60,7 @@ const SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD = 10;
58
60
  /** Default liveness-heartbeat window (~15 min) when `health.window_seconds` is unset. */
59
61
  const DEFAULT_LIVENESS_WINDOW_SECONDS = 900;
60
62
  export class Runtime {
63
+ /** Not readonly: {@link Runtime.rebuildComponents} swaps it, which is the whole point. */
61
64
  config;
62
65
  options;
63
66
  bus;
@@ -91,6 +94,25 @@ export class Runtime {
91
94
  livenessHeartbeatTimer = null;
92
95
  runActionsForSignal;
93
96
  strategyConfig;
97
+ /**
98
+ * The supervision fact source this runtime was handed, kept so a rebuilt scanner module inherits
99
+ * it.
100
+ *
101
+ * Every caller of {@link attachScannerSupervisionSource} sits in the plugin's boot and
102
+ * hot-install paths, and an in-place update runs neither. A module built by a rebuild therefore
103
+ * starts with no source and no way to be given one, which is not a cosmetic gap: liveness for
104
+ * push-driven scanners is derived entirely from it.
105
+ */
106
+ supervisionSource = null;
107
+ /**
108
+ * The context the CURRENT recipe's actions hold as `this.context`.
109
+ *
110
+ * Retained so the configuration the trading path is actually using can be read back. An update's
111
+ * report is a claim about what should be live; this is the thing that is, and the two are worth
112
+ * being able to compare — the failure this guards against is a report that says `slots: 1` over a
113
+ * strategy still entering three.
114
+ */
115
+ currentRuntimeContext;
94
116
  actionRuntimeModule;
95
117
  /** Stored so we can call off() in stop() and avoid retaining Runtime via scanner EventEmitter. */
96
118
  onScannerRunComplete = null;
@@ -109,6 +131,8 @@ export class Runtime {
109
131
  this.onScannerRunError = null;
110
132
  }
111
133
  }
134
+ /** Set at the end of start()'s dependency phase; read by every build and rebuild. */
135
+ componentDeps;
112
136
  /** Cached strategy id resolved from the runtime wallet on first {@link auditQuery} call. */
113
137
  cachedStrategyId = null;
114
138
  /** In-flight resolution promise; ensures concurrent first-burst callers reuse one `listStrategies`. */
@@ -194,6 +218,9 @@ export class Runtime {
194
218
  * which joins liveness/supervisor verdicts onto its health and state rows.
195
219
  */
196
220
  attachScannerSupervisionSource(source) {
221
+ // Remembered as well as forwarded: a rebuild replaces the module this was attached to, and the
222
+ // runtime is the only thing that outlives both sides of the swap.
223
+ this.supervisionSource = source;
197
224
  this.scannerModule.attachSupervisionSource(source);
198
225
  }
199
226
  /**
@@ -417,9 +444,7 @@ export class Runtime {
417
444
  senpi_auth_token: this.options.apiKey,
418
445
  ...this.config.notifications,
419
446
  });
420
- const riskBlock = this.config.risk;
421
447
  const strategyAddress = this.strategyConfig.address;
422
- const auditLog = riskBlock ? createRiskAuditLog(stateDir) : undefined;
423
448
  // Local domain-event ring. Always on (independent of the OTLP collector) so the agent can query
424
449
  // the trade narrative locally; the singleton logger taps it by wallet via this registration.
425
450
  registerEventSink(strategyAddress, createEventStore(stateDir, strategyAddress));
@@ -432,63 +457,7 @@ export class Runtime {
432
457
  notifications,
433
458
  runtimeName: this.config.name,
434
459
  });
435
- const riskGuard = riskBlock && senpi
436
- ? createRiskGuard({
437
- senpi,
438
- notifications,
439
- strategies: new Map([[strategyAddress, this.strategyConfig]]),
440
- // Human strategy label for alerts; risk-guard falls back to the
441
- // truncated address when an address has no name here.
442
- strategyNames: { [strategyAddress]: notificationLabel },
443
- auditLog,
444
- lifecycleTracker: lifecycle,
445
- })
446
- : createRiskGuardStub();
447
460
  logger.info("Runtime.start() creating scanner module");
448
- const layout = buildStateLayout(stateDir);
449
- // Augment caller-supplied providers with StateManager-backed DSL state access
450
- const augmentedProviders = {
451
- ...scannerProviders,
452
- getDslStatesForStrategy: scannerProviders?.getDslStatesForStrategy
453
- ?? (async (address) => this.stateManager.listDslStates(address)),
454
- };
455
- this.scannerModule = createScannerModuleFromRuntimeConfig(this.config, layout.scannerRoot, augmentedProviders, undefined, buildIdentityEnvelope);
456
- logger.info("Runtime.start() scanner module created, calling start()");
457
- // Pre-load max leverage cache (non-blocking — log warning on failure)
458
- const maxLeverageCache = new MaxLeverageCache(senpi);
459
- try {
460
- await maxLeverageCache.load();
461
- }
462
- catch (err) {
463
- logger.warn("failed to pre-load max leverage cache, will retry on first use", { err });
464
- }
465
- // Build RuntimeContext once per trading recipe and reuse for all actions.
466
- const runtimeContext = createRuntimeContext(this.strategyConfig, {
467
- senpi,
468
- hookSystem: this.hookSystem,
469
- llmDecision,
470
- contextBuilder,
471
- riskGuard,
472
- notifications,
473
- scannerModule: this.scannerModule,
474
- maxLeverageCache,
475
- positionIdRegistry: this.positionIdRegistry,
476
- notificationLabel,
477
- });
478
- this.actionRuntimeModule = new ActionRuntimeModule({ stateDir: layout.actionRoot });
479
- const { runActionsForSignal } = await createActions(this.config, {
480
- bus: this.bus,
481
- senpi,
482
- stateManager: this.stateManager,
483
- hookSystem: this.hookSystem,
484
- llmDecision,
485
- contextBuilder,
486
- riskGuard,
487
- notifications,
488
- runtimeContext,
489
- actionModule: this.actionRuntimeModule,
490
- });
491
- this.runActionsForSignal = runActionsForSignal;
492
461
  // Create the reconcile gate BEFORE DslPlugin.create() so the monitor's
493
462
  // first synchronous tick (fired inside create()) sees a gate that already exists.
494
463
  this.reconcileGate = (() => {
@@ -505,25 +474,22 @@ export class Runtime {
505
474
  })();
506
475
  // Build strategyStates map once — shared by DslPlugin and the reconcile loop.
507
476
  const strategyStatesMap = createStrategyStatesMap(this.strategyConfig, senpi);
508
- if (hasDsl) {
509
- this.dslPlugin = await DslPluginClass.create({
510
- config: exitConfig,
511
- bus: this.bus,
512
- stateDir,
513
- stateManager: this.stateManager,
514
- senpiClient: senpi,
515
- strategyStates: strategyStatesMap,
516
- notifications,
517
- // Monitor runs the unified periodic reconcile (adopt/resize/gone), serialized
518
- // against the trade loop on the same reconcileMutex key, and can resolve the boot gate.
519
- reconcileSerialize: (fn) => this.reconcileMutex.withKeyLock("reconcile", fn),
520
- reconcileGateResolve: () => this.resolveReconcileGate(),
521
- identity: buildIdentityEnvelope,
522
- });
523
- }
524
- else {
525
- this.dslPlugin = null;
526
- }
477
+ // Everything a rebuild reuses. Config-derived values are deliberately absent: those are
478
+ // recomputed from the config each build, which is what makes a rebuild a rebuild.
479
+ this.componentDeps = {
480
+ senpi,
481
+ llmDecision,
482
+ contextBuilder,
483
+ lifecycle,
484
+ stateDir,
485
+ scannerProviders,
486
+ buildIdentityEnvelope,
487
+ strategyStates: strategyStatesMap,
488
+ };
489
+ // Once per runtime, not once per build: a rebuild must not re-run it. (It is a no-op today,
490
+ // kept for backward compatibility, but the placement is the contract.)
491
+ initRisk(this.bus);
492
+ await this.buildComponents(this.config);
527
493
  // Launch detached boot-reconcile loop (does NOT block start()).
528
494
  // Last-resort guard: an unexpected throw here would otherwise become an
529
495
  // unhandled rejection and silently kill the loop. Do NOT resolve the gate
@@ -532,77 +498,8 @@ export class Runtime {
532
498
  void this.runBootReconcileLoop(strategyStatesMap).catch((err) => {
533
499
  logger.error("[runtime][reconcile] boot reconcile loop crashed", { err });
534
500
  });
535
- initRisk(this.bus);
536
- this.healthStateModule = createHealthStateModule({
537
- runtimeName: this.config.name,
538
- stateDir,
539
- startedAt: this.startedAt,
540
- providers: {
541
- scanners: this.scannerModule,
542
- dsl: this.dslPlugin ?? createDisabledDslProvider(),
543
- actions: this.actionRuntimeModule,
544
- risk: riskBlock
545
- ? createRiskHealthProvider(riskGuard, strategyAddress)
546
- : undefined,
547
- },
548
- });
549
- this.onScannerRunComplete = (event) => {
550
- // Per-scan-run id: stamped on the scanner_run emit and echoed through the pass so scan +
551
- // pass land in one trace. Prefer the id minted at the scanner.run span so scan + pass
552
- // share one senpi.run.id.
553
- const runId = event.result.runId ?? randomUUID();
554
- this.signalQueue.push({ result: event.result, enqueuedAt: Date.now(), runId });
555
- // Edge-triggered backpressure alarm: depth rises one enqueue at a time, so `=== threshold`
556
- // fires exactly once as it crosses up and re-arms only after the consumer drains back below.
557
- if (this.signalQueue.length === SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD) {
558
- emitEvent(logger, () => buildQueueBackpressureEvent({
559
- depth: this.signalQueue.length,
560
- threshold: SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD,
561
- scannerId: event.result.scannerId,
562
- address: this.strategyConfig.address,
563
- }));
564
- }
565
- if (this.signalQueueWait) {
566
- const w = this.signalQueueWait;
567
- this.signalQueueWait = null;
568
- w();
569
- }
570
- // Scanner-run summary payload (startedAt is data stamping for telemetry durations).
571
- const scannerRunPayload = {
572
- address: event.result.address,
573
- data: {
574
- ...event.result,
575
- startedAt: event.startedAt,
576
- runId,
577
- },
578
- timestamp: new Date().toISOString(),
579
- };
580
- // First-class event log. The build runs inside emitEvent's guard — telemetry never
581
- // disturbs the scan path.
582
- emitEvent(logger, () => buildScannerSummaryEvent(scannerRunPayload, Date.now()));
583
- recordScannerRun({ scannerId: event.result.scannerId, outcome: event.result.status });
584
- // TODO: Market-scanner alerts are default-OFF and MUST NOT be delivered
585
- // until per-user default-mute-list support exists (today we have mute-list-only
586
- // settings, which can't honour a default-OFF notification). When that lands,
587
- // forward `event.result.summary?.alerts` as a MARKET_SCANNER notification with:
588
- // in-app: `🔔 <b>${n} market alert(s)</b><br>${message}<br>Signals only — no trade placed yet.`
589
- // push: no title (the notification service resolves it), body `🔔 ${message}`
590
- // Address-less by design (the scanner watches the market, not one strategy), so
591
- // it opens a chat by category alone and the strategy wallet must not leak into
592
- // the seed. Re-add per-(scannerId, message) dedupe at that time.
593
- // Resolve this TODO when default mute list functionality is available.
594
- };
595
- this.scannerModule.on("scanner:run:complete", this.onScannerRunComplete);
596
- // An ERROR run-result is routed to `scanner:run:error`, never to `run:complete`, so the
597
- // counter above never saw a failed run of any scanner — including a scaffold-reported
598
- // tick failure from `/errors`. Count it here so `scanner.runs{outcome=error}` is a real
599
- // series (the wedged-scanner alert is bridged from it); the health/telemetry side is
600
- // already handled by the scanner module's own listener.
601
- this.onScannerRunError = (event) => {
602
- recordScannerRun({ scannerId: event.scannerId, outcome: "error" });
603
- };
604
- this.scannerModule.on("scanner:run:error", this.onScannerRunError);
605
- this.signalConsumerDone = this.runSignalConsumer();
501
+ this.wireScannerRunListener();
502
+ this.startSignalConsumer();
606
503
  logger.info("Runtime.start() invoking scannerModule.start()");
607
504
  await this.scannerModule.start();
608
505
  logger.info("Runtime.start() scannerModule.start() returned");
@@ -720,6 +617,443 @@ export class Runtime {
720
617
  * Cancel exchange SLs and archive DSL state for the strategy, then log removal.
721
618
  */
722
619
  /** @returns Count of active DSL positions archived during teardown. */
620
+ /**
621
+ * Build every component derived from the recipe: scanners, actions, the DSL plugin, and the
622
+ * health surface that holds references to all three.
623
+ *
624
+ * Called by `start()` and again by {@link rebuildComponents}. Deliberately does NOT create the
625
+ * reconcile gate, launch the boot-reconcile loop, or call `initRisk` — those are once-per-runtime
626
+ * and re-running them would re-block trading on an already-converged runtime, duplicate a
627
+ * detached loop, and double-subscribe the bus respectively.
628
+ */
629
+ async buildComponents(config) {
630
+ const constructed = await this.constructComponents(config);
631
+ await this.commitComponents(config, constructed);
632
+ }
633
+ /**
634
+ * Phase one of a build: everything the recipe determines, constructed but NOT installed on `this`
635
+ * and NOT started.
636
+ *
637
+ * The split exists so a rebuild can do all of its fallible work while the OLD components are
638
+ * still running. A recipe that names a scanner which will not construct, or an action class that
639
+ * does not exist, throws here — with the live runtime untouched, still scanning and still holding
640
+ * its stops. The alternative (the previous shape) tore everything down first and then discovered
641
+ * the new config was unbuildable, leaving a funded wallet with an open book, no scanners and no
642
+ * exit engine.
643
+ *
644
+ * Nothing in here may subscribe to the bus, start a timer, or touch disk. That is what makes it
645
+ * safe to run twice over — once for the attempt, once more for a rollback. `DslPlugin.create` is
646
+ * deliberately NOT called here for exactly that reason: it subscribes position handlers and
647
+ * starts the monitor loop, so pre-building it would put two plugins on one bus, both ticking the
648
+ * same positions. Its config is parsed here instead, which is where a bad preset actually throws.
649
+ */
650
+ async constructComponents(config) {
651
+ const { senpi, llmDecision, contextBuilder, lifecycle, stateDir, scannerProviders, buildIdentityEnvelope, } = this.componentDeps;
652
+ const exitConfig = mapRuntimeExitToDslConfig(config.exit);
653
+ const hasDsl = exitConfig.dsl_preset != null &&
654
+ typeof exitConfig.dsl_preset === "object" &&
655
+ !Array.isArray(exitConfig.dsl_preset);
656
+ // Boot enforces this and a rebuild must too. Without it an update could keep `exit.dsl_preset`
657
+ // while dropping the position-tracker scanner (a pure removal, which the plan does not refuse)
658
+ // or its POSITION_TRACKER action: the DSL plugin is created, but nothing confirms position
659
+ // openings from clearing-house state, so every position opened afterwards gets no DSL state —
660
+ // no floor, no trailing stop, no exit — while the apply reports success. It also leaves the
661
+ // runtime in a config `start()` would throw on, so the next restart fails to boot at all.
662
+ //
663
+ // In the fallible phase deliberately: this throws while the OLD components are still running
664
+ // and still holding their stops.
665
+ validateDslPositionTracking(config, hasDsl);
666
+ // Parse the preset now rather than inside `DslPlugin.create`, so a malformed exit block is
667
+ // rejected while the old plugin is still holding its stops. The result is discarded — this is
668
+ // a validation, and `create` re-parses from the same input.
669
+ if (hasDsl)
670
+ parseDslPluginConfig(exitConfig);
671
+ const layout = buildStateLayout(stateDir);
672
+ // Augment caller-supplied providers with StateManager-backed DSL state access
673
+ const augmentedProviders = {
674
+ ...scannerProviders,
675
+ getDslStatesForStrategy: scannerProviders?.getDslStatesForStrategy
676
+ ?? (async (address) => this.stateManager.listDslStates(address)),
677
+ };
678
+ const scannerModule = createScannerModuleFromRuntimeConfig(config, layout.scannerRoot, augmentedProviders, undefined, buildIdentityEnvelope);
679
+ logger.info("Runtime.start() scanner module created, calling start()");
680
+ // Pre-load max leverage cache (non-blocking — log warning on failure)
681
+ const maxLeverageCache = new MaxLeverageCache(senpi);
682
+ try {
683
+ await maxLeverageCache.load();
684
+ }
685
+ catch (err) {
686
+ logger.warn("failed to pre-load max leverage cache, will retry on first use", { err });
687
+ }
688
+ // Resolved from the incoming recipe. On boot this reproduces what `start()` already assigned;
689
+ // on a rebuild it is the only thing that moves the new sizing into the actions.
690
+ const strategyConfig = resolveStrategyConfig(config);
691
+ // Both are recipe-derived, so both are rebuilt with it. Kept in the boot bag they were a third
692
+ // instance of the same silent no-op: `diffRecipes` reports `notifications.*`, `group` and
693
+ // `name` as changed and the apply returns success, while the running strategy went on notifying
694
+ // through the boot-time service under the boot-time label until the process restarted.
695
+ //
696
+ // Constructed here rather than at commit because `createNotificationService` THROWS when its
697
+ // token cannot be resolved — that belongs in the fallible phase, while the old components are
698
+ // still running, not in the window after teardown.
699
+ const notificationLabel = config.group ?? config.name;
700
+ const notifications = createNotificationService({
701
+ senpi_auth_token: this.options.apiKey,
702
+ ...config.notifications,
703
+ });
704
+ // The guard is derived from `risk:`, so it is rebuilt with the recipe rather than kept in the
705
+ // boot bag. Kept there, three things were wrong at once and all of them silently: adding a
706
+ // `risk:` block to a running strategy registered health reporting around the boot-time STUB,
707
+ // which answers "enabled, healthy, OPEN" while enforcing nothing; changing a threshold moved
708
+ // the report and not the gate; and removing the block hid the health row while the real guard
709
+ // carried on refusing entries.
710
+ //
711
+ // Safe to rebuild because the guard is stateless by construction — every gate is derived from
712
+ // live venue history per call. The self-pause state lives in `lifecycle`, which is deliberately
713
+ // NOT rebuilt. The one thing lost is the in-memory gate-transition map, so a gate that is
714
+ // already closed may announce itself once more after an update.
715
+ const riskBlock = config.risk;
716
+ const auditLog = riskBlock ? createRiskAuditLog(stateDir) : undefined;
717
+ const riskGuard = riskBlock && senpi
718
+ ? createRiskGuard({
719
+ senpi,
720
+ notifications,
721
+ strategies: new Map([[strategyConfig.address, strategyConfig]]),
722
+ // Human strategy label for alerts; risk-guard falls back to the
723
+ // truncated address when an address has no name here.
724
+ strategyNames: { [strategyConfig.address]: notificationLabel },
725
+ auditLog,
726
+ lifecycleTracker: lifecycle,
727
+ })
728
+ : createRiskGuardStub();
729
+ // Build RuntimeContext once per trading recipe and reuse for all actions.
730
+ const runtimeContext = createRuntimeContext(strategyConfig, {
731
+ senpi,
732
+ hookSystem: this.hookSystem,
733
+ llmDecision,
734
+ contextBuilder,
735
+ riskGuard,
736
+ notifications,
737
+ scannerModule,
738
+ maxLeverageCache,
739
+ positionIdRegistry: this.positionIdRegistry,
740
+ notificationLabel,
741
+ });
742
+ const actionRuntimeModule = new ActionRuntimeModule({ stateDir: layout.actionRoot });
743
+ const { runActionsForSignal } = await createActions(config, {
744
+ bus: this.bus,
745
+ senpi,
746
+ stateManager: this.stateManager,
747
+ hookSystem: this.hookSystem,
748
+ llmDecision,
749
+ contextBuilder,
750
+ riskGuard,
751
+ notifications,
752
+ runtimeContext,
753
+ actionModule: actionRuntimeModule,
754
+ });
755
+ return {
756
+ scannerModule,
757
+ actionRuntimeModule,
758
+ runActionsForSignal,
759
+ exitConfig,
760
+ hasDsl,
761
+ strategyConfig,
762
+ runtimeContext,
763
+ riskGuard,
764
+ notifications,
765
+ notificationLabel,
766
+ };
767
+ }
768
+ /**
769
+ * Phase two of a build: install the constructed components on `this`, then create the one
770
+ * component that cannot be constructed without side effects.
771
+ *
772
+ * Everything fallible that COULD be done earlier already has been, in
773
+ * {@link constructComponents}. What remains is assignment — which cannot throw — plus
774
+ * `DslPlugin.create`, whose config this build already parsed and whose state manager, senpi
775
+ * client and strategy-state map are all passed in (each of those resolvers returns immediately
776
+ * when supplied). That is why the teardown window in {@link rebuildComponents} is narrow enough
777
+ * to be worth taking, and why its rollback is a backstop rather than the plan.
778
+ */
779
+ async commitComponents(config, built) {
780
+ const { senpi, stateDir, buildIdentityEnvelope, strategyStates: strategyStatesMap, } = this.componentDeps;
781
+ const riskBlock = config.risk;
782
+ // Install the recipe-derived strategy config before anything below reads it.
783
+ //
784
+ // The map is refreshed by replacing its entry rather than by minting a new Map: the boot
785
+ // reconcile loop and the DSL plugin were both handed this exact instance, and swapping the
786
+ // Map would leave them reading a container nobody writes to any more. Every consumer does
787
+ // `strategyStates.get(address)` at call time, so replacing the value is enough. The dropped
788
+ // StrategyState carried only venue caches (clearinghouse, open orders, past trades), which
789
+ // refetch on demand — and the swap runs under the reconcile lock, so no read is in flight.
790
+ this.strategyConfig = built.strategyConfig;
791
+ this.currentRuntimeContext = built.runtimeContext;
792
+ const strategyAddress = built.strategyConfig.address;
793
+ strategyStatesMap.set(strategyAddress, createStrategyState(built.strategyConfig, senpi));
794
+ this.scannerModule = built.scannerModule;
795
+ // Carried across the swap. Without it `derivePushDrivenScannerHealth` sees `lastAliveAt: null`
796
+ // for every external scanner, so each row reads `unknown` and the runtime's overall status
797
+ // follows — permanently, until the process restarts, rather than for the "next minute or so"
798
+ // an applied update's report promises while the scanners report back in.
799
+ if (this.supervisionSource) {
800
+ this.scannerModule.attachSupervisionSource(this.supervisionSource);
801
+ }
802
+ this.actionRuntimeModule = built.actionRuntimeModule;
803
+ this.runActionsForSignal = built.runActionsForSignal;
804
+ if (built.hasDsl) {
805
+ this.dslPlugin = await DslPluginClass.create({
806
+ config: built.exitConfig,
807
+ bus: this.bus,
808
+ stateDir,
809
+ stateManager: this.stateManager,
810
+ senpiClient: senpi,
811
+ strategyStates: strategyStatesMap,
812
+ notifications: built.notifications,
813
+ // Monitor runs the unified periodic reconcile (adopt/resize/gone), serialized
814
+ // against the trade loop on the same reconcileMutex key, and can resolve the boot gate.
815
+ reconcileSerialize: (fn) => this.reconcileMutex.withKeyLock("reconcile", fn),
816
+ reconcileGateResolve: () => this.resolveReconcileGate(),
817
+ identity: buildIdentityEnvelope,
818
+ });
819
+ }
820
+ else {
821
+ this.dslPlugin = null;
822
+ }
823
+ this.healthStateModule = createHealthStateModule({
824
+ runtimeName: config.name,
825
+ stateDir,
826
+ startedAt: this.startedAt ?? undefined,
827
+ providers: {
828
+ scanners: this.scannerModule,
829
+ dsl: this.dslPlugin ?? createDisabledDslProvider(),
830
+ actions: this.actionRuntimeModule,
831
+ risk: riskBlock
832
+ ? createRiskHealthProvider(built.riskGuard, strategyAddress)
833
+ : undefined,
834
+ },
835
+ });
836
+ }
837
+ /**
838
+ * Register the scan-complete listener on the CURRENT scanner module.
839
+ *
840
+ * Separate from {@link buildComponents} because the listener is held against a specific module
841
+ * instance: a rebuild replaces the module, so the old registration goes with it and a fresh one
842
+ * has to be made against the new one.
843
+ */
844
+ wireScannerRunListener() {
845
+ const { senpi } = this.componentDeps;
846
+ this.onScannerRunComplete = (event) => {
847
+ // Per-scan-run id: stamped on the scanner_run emit and echoed through the pass so scan +
848
+ // pass land in one trace. Prefer the id minted at the scanner.run span so scan + pass
849
+ // share one senpi.run.id.
850
+ const runId = event.result.runId ?? randomUUID();
851
+ this.signalQueue.push({ result: event.result, enqueuedAt: Date.now(), runId });
852
+ // Edge-triggered backpressure alarm: depth rises one enqueue at a time, so `=== threshold`
853
+ // fires exactly once as it crosses up and re-arms only after the consumer drains back below.
854
+ if (this.signalQueue.length === SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD) {
855
+ emitEvent(logger, () => buildQueueBackpressureEvent({
856
+ depth: this.signalQueue.length,
857
+ threshold: SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD,
858
+ scannerId: event.result.scannerId,
859
+ address: this.strategyConfig.address,
860
+ }));
861
+ }
862
+ if (this.signalQueueWait) {
863
+ const w = this.signalQueueWait;
864
+ this.signalQueueWait = null;
865
+ w();
866
+ }
867
+ // Scanner-run summary payload (startedAt is data stamping for telemetry durations).
868
+ const scannerRunPayload = {
869
+ address: event.result.address,
870
+ data: {
871
+ ...event.result,
872
+ startedAt: event.startedAt,
873
+ runId,
874
+ },
875
+ timestamp: new Date().toISOString(),
876
+ };
877
+ // First-class event log. The build runs inside emitEvent's guard — telemetry never
878
+ // disturbs the scan path.
879
+ emitEvent(logger, () => buildScannerSummaryEvent(scannerRunPayload, Date.now()));
880
+ recordScannerRun({ scannerId: event.result.scannerId, outcome: event.result.status });
881
+ // TODO: Market-scanner alerts are default-OFF and MUST NOT be delivered
882
+ // until per-user default-mute-list support exists (today we have mute-list-only
883
+ // settings, which can't honour a default-OFF notification). When that lands,
884
+ // forward `event.result.summary?.alerts` as a MARKET_SCANNER notification with:
885
+ // in-app: `🔔 <b>${n} market alert(s)</b><br>${message}<br>Signals only — no trade placed yet.`
886
+ // push: no title (the notification service resolves it), body `🔔 ${message}`
887
+ // Address-less by design (the scanner watches the market, not one strategy), so
888
+ // it opens a chat by category alone and the strategy wallet must not leak into
889
+ // the seed. Re-add per-(scannerId, message) dedupe at that time.
890
+ // Resolve this TODO when default mute list functionality is available.
891
+ };
892
+ this.scannerModule.on("scanner:run:complete", this.onScannerRunComplete);
893
+ // An ERROR run-result is routed to `scanner:run:error`, never to `run:complete`, so the
894
+ // counter above never saw a failed run of any scanner — including a scaffold-reported
895
+ // tick failure from `/errors`. Count it here so `scanner.runs{outcome=error}` is a real
896
+ // series (the wedged-scanner alert is bridged from it); the health/telemetry side is
897
+ // already handled by the scanner module's own listener.
898
+ //
899
+ // Registered here rather than in `start()` so a rebuilt scanner module gets it too: the
900
+ // listener is bound to a module instance, and a rebuild replaces the module.
901
+ this.onScannerRunError = (event) => {
902
+ recordScannerRun({ scannerId: event.scannerId, outcome: "error" });
903
+ };
904
+ this.scannerModule.on("scanner:run:error", this.onScannerRunError);
905
+ }
906
+ /**
907
+ * Launch the single serial signal consumer. Called ONCE, from `start()`.
908
+ *
909
+ * Deliberately not part of {@link wireScannerRunListener}: that has to run again on every rebuild,
910
+ * because the listener is held against a specific scanner-module instance and the rebuild replaces
911
+ * it. Launching the consumer alongside it started a SECOND infinite `runSignalConsumer()` loop on
912
+ * every apply — nothing stopped the first, `signalConsumerStopped` is only set by `stop()`, and a
913
+ * rollback made it three.
914
+ *
915
+ * Two things broke. `signalQueueWait` holds exactly one resolver, so whichever consumer parked
916
+ * last owned the wake-up and `signalConsumerDone` tracked only the newest — `stop()` woke one
917
+ * consumer and then awaited a different one's promise, hanging `runtime delete`, gateway shutdown
918
+ * and SIGTERM forever on any runtime that had been updated once. And with a non-empty queue at
919
+ * swap time both consumers ran concurrently: the slot check is read BEFORE the reconcile lock, so
920
+ * two passes could each read `used = 0` and each open a position on a one-slot strategy.
921
+ *
922
+ * Serial signal processing is the invariant the whole trade path is built on; there must be
923
+ * exactly one of these per runtime.
924
+ */
925
+ startSignalConsumer() {
926
+ if (this.signalConsumerDone)
927
+ return;
928
+ this.signalConsumerDone = this.runSignalConsumer();
929
+ }
930
+ /**
931
+ * Swap this runtime onto a new recipe without restarting it or touching its state.
932
+ *
933
+ * Teardown is the reverse of the build, and both halves matter. The scan listener is released
934
+ * before the module it is registered on; `scannerModule.stop()` clears its timers and awaits
935
+ * in-flight runs; `dslPlugin.stop()` is AWAITED, because it releases its bus subscriptions and
936
+ * drains the tick that may still be mid-flight at the venue — without that the outgoing monitor
937
+ * keeps writing state while the incoming one is already ticking the same positions, and the two
938
+ * hold independent StateManagers so nothing else serialises them.
939
+ *
940
+ * State on disk is untouched throughout: DSL position files, scanner stores and action history
941
+ * all survive, which is the entire reason an update is worth having over a delete-and-recreate.
942
+ */
943
+ async rebuildComponents(config) {
944
+ // One rebuild at a time, for this runtime, whoever asks.
945
+ //
946
+ // Without it two concurrent applies interleave catastrophically: A tears down (setting
947
+ // `dslPlugin = null`), B's teardown then finds nothing to stop, and both commit. A's DSL plugin
948
+ // is never stopped — it keeps its bus subscriptions and its monitor loop, ticking the same
949
+ // positions through an independent StateManager and racing B over venue stops and closes. A's
950
+ // scanner module leaks the same way. `stop()` only ever tears down whatever `this.dslPlugin`
951
+ // and `this.scannerModule` point at, so the loser of the race is unreachable for the life of
952
+ // the process.
953
+ //
954
+ // A DIFFERENT key from "reconcile": `swapInComponents` takes that one inside this, and
955
+ // `KeyedMutex` is non-reentrant, so sharing the key would deadlock every rebuild.
956
+ return this.reconcileMutex.withKeyLock("rebuild", () => this.rebuildComponentsLocked(config));
957
+ }
958
+ async rebuildComponentsLocked(config) {
959
+ // `stop()` holds this same "rebuild" key, so a stop can never land inside the window below —
960
+ // but it can have COMPLETED while this call sat in the lock queue. Rebuilding then would
961
+ // resurrect components on a runtime that already reported stopped (over a disconnected senpi
962
+ // client), and `stop()`'s own `started` guard would make them unreachable forever.
963
+ if (!this.started) {
964
+ throw new Error("Runtime is stopped; there is nothing to rebuild in place.");
965
+ }
966
+ // Everything fallible happens FIRST, while the old components are still running. A recipe that
967
+ // will not build throws here, and the live runtime never notices: it is still scanning, still
968
+ // ticking, still holding the stops it has already promised.
969
+ const next = await this.constructComponents(config);
970
+ const previousConfig = this.config;
971
+ await this.teardownComponentsForRebuild();
972
+ this.config = config;
973
+ try {
974
+ await this.swapInComponents(config, next);
975
+ }
976
+ catch (err) {
977
+ // Backstop, not the plan. Reaching here means something threw in the narrow window that
978
+ // construction could not cover — realistically `DslPlugin.create` failing to resolve state
979
+ // it has already been handed. The runtime is currently torn down with an open book, so the
980
+ // only acceptable outcome is to get the PREVIOUS recipe running again.
981
+ logger.error("[runtime] rebuild failed after teardown — restoring the previous recipe", { err });
982
+ try {
983
+ await this.teardownComponentsForRebuild();
984
+ const restored = await this.constructComponents(previousConfig);
985
+ this.config = previousConfig;
986
+ await this.swapInComponents(previousConfig, restored);
987
+ }
988
+ catch (rollbackErr) {
989
+ // Both the new recipe and the old one failed to build. Nothing is managing the book, and
990
+ // no further automatic recovery is available — say so in the loudest terms the process has,
991
+ // naming the wallet, because a human has to act.
992
+ logger.error("[runtime] ROLLBACK FAILED — runtime is down with an open book", {
993
+ address: this.strategyConfig.address,
994
+ rebuildError: err,
995
+ rollbackError: rollbackErr,
996
+ });
997
+ throw new RuntimeRebuildRollbackFailedError(this.strategyConfig.address, err, rollbackErr);
998
+ }
999
+ // The runtime is healthy again on its previous recipe; the update itself did not apply.
1000
+ throw err;
1001
+ }
1002
+ }
1003
+ /**
1004
+ * Install the freshly built components, serialized against the trade loop.
1005
+ *
1006
+ * The signal consumer runs every action pass inside
1007
+ * `reconcileMutex.withKeyLock("reconcile", () => this.runActionsForSignal(...))`, reading
1008
+ * `runActionsForSignal` at call time from inside the lock. Taking the same lock here means a
1009
+ * signal already in flight finishes against the OLD actions before the swap happens, and the next
1010
+ * one picks up the new ones — no pass can straddle the change.
1011
+ *
1012
+ * The lock is scoped to the SWAP and deliberately NOT to the teardown that precedes it.
1013
+ * `teardownComponentsForRebuild` awaits the DSL plugin's in-flight tick, and that tick takes this
1014
+ * very lock for its periodic reconcile (`runReconcile` is handed `reconcileSerialize`). Holding
1015
+ * the lock across the drain would deadlock the runtime — permanently, with an open book and
1016
+ * nothing managing it — because `KeyedMutex` is a plain FIFO and grants no reentrancy.
1017
+ *
1018
+ * Creating the DSL plugin inside the lock is safe for the mirror-image reason: `create()` starts
1019
+ * the monitor loop but does not await its first tick, so that tick queues behind this lock
1020
+ * instead of blocking the call that holds it.
1021
+ */
1022
+ async swapInComponents(config, built) {
1023
+ await this.reconcileMutex.withKeyLock("reconcile", async () => {
1024
+ await this.commitComponents(config, built);
1025
+ // Listener only. The consumer already running is the one that must keep running — see
1026
+ // `startSignalConsumer`.
1027
+ this.wireScannerRunListener();
1028
+ });
1029
+ // Started OUTSIDE the lock, matching boot (`start()` calls this bare). `ScannerEngine.start()`
1030
+ // bootstraps context dependencies with real scanner runs — network time — and the DSL monitor's
1031
+ // tick awaits this same "reconcile" key before its price fetch, so holding the lock across the
1032
+ // start would stall breach evaluation and the trade loop on a live book for the duration of
1033
+ // those fetches. By here the components are committed and the listener wired, so a scan that
1034
+ // completes immediately lands in the queue like any other.
1035
+ await this.scannerModule.start();
1036
+ }
1037
+ /**
1038
+ * Drop the scan listener, stop the scanner module, stop and release the DSL plugin.
1039
+ *
1040
+ * Shared by the rebuild and its rollback so the two cannot drift — the rollback runs it a second
1041
+ * time to clear whatever the failed commit had already installed, and every step is idempotent
1042
+ * for exactly that reason.
1043
+ *
1044
+ * Order matters: the listener comes off before the module it is registered on;
1045
+ * `scannerModule.stop()` clears its timers and awaits in-flight runs; `dslPlugin.stop()` is
1046
+ * AWAITED, because it releases its bus subscriptions and drains the tick that may still be
1047
+ * mid-flight at the venue.
1048
+ */
1049
+ async teardownComponentsForRebuild() {
1050
+ this.unwireScannerRunListeners();
1051
+ await this.scannerModule.stop();
1052
+ if (this.dslPlugin) {
1053
+ await this.dslPlugin.stop();
1054
+ this.dslPlugin = null;
1055
+ }
1056
+ }
723
1057
  async removeStrategy(address) {
724
1058
  let abandoned = 0;
725
1059
  if (this.dslPlugin) {
@@ -968,6 +1302,17 @@ export class Runtime {
968
1302
  }
969
1303
  }
970
1304
  async stop() {
1305
+ // Serialized on the same "rebuild" key as `rebuildComponents`. An unserialized stop landing in
1306
+ // the rebuild's teardown-to-swap window completes against the torn-down components and is then
1307
+ // undone by the swap: re-armed scanner timers and a fresh DSL monitor on a runtime that already
1308
+ // reported stopped — and since `started` is false by then, every later stop() returns at the
1309
+ // guard below, leaving the resurrected components running unreachably for the life of the
1310
+ // process. Waiting the rebuild out and tearing down whatever it installed is the only order
1311
+ // that ends with nothing running. ("rebuild", not "reconcile": the teardown below drains the
1312
+ // DSL tick, which takes "reconcile" — sharing that key would deadlock shutdown.)
1313
+ return this.reconcileMutex.withKeyLock("rebuild", () => this.stopLocked());
1314
+ }
1315
+ async stopLocked() {
971
1316
  if (!this.started)
972
1317
  return;
973
1318
  this.started = false;