@senpi-ai/runtime 3.0.98 → 3.0.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/guide-commands.d.ts.map +1 -1
- package/dist/cli/guide-commands.js +38 -0
- package/dist/cli/guide-commands.js.map +1 -1
- package/dist/cli/senpi-commands.d.ts.map +1 -1
- package/dist/cli/senpi-commands.js +29 -1
- package/dist/cli/senpi-commands.js.map +1 -1
- package/dist/cli/update-command.d.ts +69 -0
- package/dist/cli/update-command.d.ts.map +1 -0
- package/dist/cli/update-command.js +153 -0
- package/dist/cli/update-command.js.map +1 -0
- package/dist/dsl/plugin/index.d.ts +35 -1
- package/dist/dsl/plugin/index.d.ts.map +1 -1
- package/dist/dsl/plugin/index.js +62 -7
- package/dist/dsl/plugin/index.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +205 -27
- package/dist/index.js.map +1 -1
- package/dist/runtime/effective-config.d.ts +67 -0
- package/dist/runtime/effective-config.d.ts.map +1 -0
- package/dist/runtime/effective-config.js +324 -0
- package/dist/runtime/effective-config.js.map +1 -0
- package/dist/runtime/lifecycle-events.d.ts +19 -0
- package/dist/runtime/lifecycle-events.d.ts.map +1 -1
- package/dist/runtime/lifecycle-events.js +27 -0
- package/dist/runtime/lifecycle-events.js.map +1 -1
- package/dist/runtime/rebuild-errors.d.ts +26 -0
- package/dist/runtime/rebuild-errors.d.ts.map +1 -0
- package/dist/runtime/rebuild-errors.js +36 -0
- package/dist/runtime/rebuild-errors.js.map +1 -0
- package/dist/runtime/run.d.ts +10 -0
- package/dist/runtime/run.d.ts.map +1 -1
- package/dist/runtime/run.js +13 -1
- package/dist/runtime/run.js.map +1 -1
- package/dist/runtime/runtime.d.ts +146 -1
- package/dist/runtime/runtime.d.ts.map +1 -1
- package/dist/runtime/runtime.js +511 -148
- package/dist/runtime/runtime.js.map +1 -1
- package/dist/runtime/strategy-registry.d.ts +1 -0
- package/dist/runtime/strategy-registry.d.ts.map +1 -1
- package/dist/runtime/strategy-registry.js +19 -0
- package/dist/runtime/strategy-registry.js.map +1 -1
- package/dist/runtime/update-apply-deps.d.ts +101 -0
- package/dist/runtime/update-apply-deps.d.ts.map +1 -0
- package/dist/runtime/update-apply-deps.js +159 -0
- package/dist/runtime/update-apply-deps.js.map +1 -0
- package/dist/runtime/update-handler.d.ts +197 -0
- package/dist/runtime/update-handler.d.ts.map +1 -0
- package/dist/runtime/update-handler.js +381 -0
- package/dist/runtime/update-handler.js.map +1 -0
- package/dist/runtime/update-plan.d.ts +60 -0
- package/dist/runtime/update-plan.d.ts.map +1 -0
- package/dist/runtime/update-plan.js +203 -0
- package/dist/runtime/update-plan.js.map +1 -0
- package/dist/runtime/update-register.d.ts +67 -0
- package/dist/runtime/update-register.d.ts.map +1 -0
- package/dist/runtime/update-register.js +173 -0
- package/dist/runtime/update-register.js.map +1 -0
- package/dist/runtime/update-report.d.ts +125 -0
- package/dist/runtime/update-report.d.ts.map +1 -0
- package/dist/runtime/update-report.js +215 -0
- package/dist/runtime/update-report.js.map +1 -0
- package/dist/runtime-api/intake/intake-events.d.ts +4 -2
- package/dist/runtime-api/intake/intake-events.d.ts.map +1 -1
- package/dist/runtime-api/intake/intake-events.js +4 -2
- package/dist/runtime-api/intake/intake-events.js.map +1 -1
- package/dist/runtime-api/intake/intake-mount.d.ts +2 -0
- package/dist/runtime-api/intake/intake-mount.d.ts.map +1 -1
- package/dist/runtime-api/intake/intake-mount.js +8 -20
- package/dist/runtime-api/intake/intake-mount.js.map +1 -1
- package/dist/runtime-api/intake/scaffold-error-router.d.ts +43 -0
- package/dist/runtime-api/intake/scaffold-error-router.d.ts.map +1 -0
- package/dist/runtime-api/intake/scaffold-error-router.js +63 -0
- package/dist/runtime-api/intake/scaffold-error-router.js.map +1 -0
- package/dist/scanners/external/scaffold/scaffold.py +94 -1
- package/dist/scanners/external/scaffold/senpi_mcp.py +427 -71
- package/dist/utils/event-catalog.d.ts +29 -2
- package/dist/utils/event-catalog.d.ts.map +1 -1
- package/dist/utils/event-catalog.js +2 -1
- package/dist/utils/event-catalog.js.map +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/runtime/runtime.js
CHANGED
|
@@ -18,13 +18,15 @@ import { resolveStrategyConfig } from "./map-strategies.js";
|
|
|
18
18
|
import { mapRuntimeExitToDslConfig } from "./map-exit-config.js";
|
|
19
19
|
import { buildActionScanPayload } from "./build-action-scan-payload.js";
|
|
20
20
|
import { createScannerModuleFromRuntimeConfig } from "./create-scanner-module.js";
|
|
21
|
-
import { createRuntimeContext, createStrategyStatesMap } from "./create-runtime-context.js";
|
|
21
|
+
import { createRuntimeContext, createStrategyState, createStrategyStatesMap, } from "./create-runtime-context.js";
|
|
22
22
|
import { createActions } from "./create-actions.js";
|
|
23
23
|
import { ActionRuntimeModule } from "../actions/action-runtime-module.js";
|
|
24
24
|
import { validateDslPositionTracking } from "./validate-dsl-position-tracking.js";
|
|
25
25
|
import { ScannerRegistry } from "./scanner-registry.js";
|
|
26
26
|
import { ActionRegistry } from "./action-registry.js";
|
|
27
27
|
import { initRisk } from "../risk/index.js";
|
|
28
|
+
import { parseDslPluginConfig } from "../dsl/config/parse-plugin-config.js";
|
|
29
|
+
import { RuntimeRebuildRollbackFailedError } from "./rebuild-errors.js";
|
|
28
30
|
import { createHealthStateModule, } from "../health/index.js";
|
|
29
31
|
import { DslPlugin as DslPluginClass } from "../dsl/plugin/index.js";
|
|
30
32
|
import { HOOK_EVENT } from "../types/hooks.js";
|
|
@@ -58,6 +60,7 @@ const SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD = 10;
|
|
|
58
60
|
/** Default liveness-heartbeat window (~15 min) when `health.window_seconds` is unset. */
|
|
59
61
|
const DEFAULT_LIVENESS_WINDOW_SECONDS = 900;
|
|
60
62
|
export class Runtime {
|
|
63
|
+
/** Not readonly: {@link Runtime.rebuildComponents} swaps it, which is the whole point. */
|
|
61
64
|
config;
|
|
62
65
|
options;
|
|
63
66
|
bus;
|
|
@@ -91,9 +94,45 @@ export class Runtime {
|
|
|
91
94
|
livenessHeartbeatTimer = null;
|
|
92
95
|
runActionsForSignal;
|
|
93
96
|
strategyConfig;
|
|
97
|
+
/**
|
|
98
|
+
* The supervision fact source this runtime was handed, kept so a rebuilt scanner module inherits
|
|
99
|
+
* it.
|
|
100
|
+
*
|
|
101
|
+
* Every caller of {@link attachScannerSupervisionSource} sits in the plugin's boot and
|
|
102
|
+
* hot-install paths, and an in-place update runs neither. A module built by a rebuild therefore
|
|
103
|
+
* starts with no source and no way to be given one, which is not a cosmetic gap: liveness for
|
|
104
|
+
* push-driven scanners is derived entirely from it.
|
|
105
|
+
*/
|
|
106
|
+
supervisionSource = null;
|
|
107
|
+
/**
|
|
108
|
+
* The context the CURRENT recipe's actions hold as `this.context`.
|
|
109
|
+
*
|
|
110
|
+
* Retained so the configuration the trading path is actually using can be read back. An update's
|
|
111
|
+
* report is a claim about what should be live; this is the thing that is, and the two are worth
|
|
112
|
+
* being able to compare — the failure this guards against is a report that says `slots: 1` over a
|
|
113
|
+
* strategy still entering three.
|
|
114
|
+
*/
|
|
115
|
+
currentRuntimeContext;
|
|
94
116
|
actionRuntimeModule;
|
|
95
117
|
/** Stored so we can call off() in stop() and avoid retaining Runtime via scanner EventEmitter. */
|
|
96
118
|
onScannerRunComplete = null;
|
|
119
|
+
onScannerRunError = null;
|
|
120
|
+
/**
|
|
121
|
+
* Release both run listeners from the CURRENT scanner module. Idempotent; called on every
|
|
122
|
+
* teardown (boot failure, rebuild, stop) so a stale module never retains the Runtime.
|
|
123
|
+
*/
|
|
124
|
+
unwireScannerRunListeners() {
|
|
125
|
+
if (this.onScannerRunComplete != null) {
|
|
126
|
+
this.scannerModule.off("scanner:run:complete", this.onScannerRunComplete);
|
|
127
|
+
this.onScannerRunComplete = null;
|
|
128
|
+
}
|
|
129
|
+
if (this.onScannerRunError != null) {
|
|
130
|
+
this.scannerModule.off("scanner:run:error", this.onScannerRunError);
|
|
131
|
+
this.onScannerRunError = null;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/** Set at the end of start()'s dependency phase; read by every build and rebuild. */
|
|
135
|
+
componentDeps;
|
|
97
136
|
/** Cached strategy id resolved from the runtime wallet on first {@link auditQuery} call. */
|
|
98
137
|
cachedStrategyId = null;
|
|
99
138
|
/** In-flight resolution promise; ensures concurrent first-burst callers reuse one `listStrategies`. */
|
|
@@ -179,6 +218,9 @@ export class Runtime {
|
|
|
179
218
|
* which joins liveness/supervisor verdicts onto its health and state rows.
|
|
180
219
|
*/
|
|
181
220
|
attachScannerSupervisionSource(source) {
|
|
221
|
+
// Remembered as well as forwarded: a rebuild replaces the module this was attached to, and the
|
|
222
|
+
// runtime is the only thing that outlives both sides of the swap.
|
|
223
|
+
this.supervisionSource = source;
|
|
182
224
|
this.scannerModule.attachSupervisionSource(source);
|
|
183
225
|
}
|
|
184
226
|
/**
|
|
@@ -402,9 +444,7 @@ export class Runtime {
|
|
|
402
444
|
senpi_auth_token: this.options.apiKey,
|
|
403
445
|
...this.config.notifications,
|
|
404
446
|
});
|
|
405
|
-
const riskBlock = this.config.risk;
|
|
406
447
|
const strategyAddress = this.strategyConfig.address;
|
|
407
|
-
const auditLog = riskBlock ? createRiskAuditLog(stateDir) : undefined;
|
|
408
448
|
// Local domain-event ring. Always on (independent of the OTLP collector) so the agent can query
|
|
409
449
|
// the trade narrative locally; the singleton logger taps it by wallet via this registration.
|
|
410
450
|
registerEventSink(strategyAddress, createEventStore(stateDir, strategyAddress));
|
|
@@ -417,63 +457,7 @@ export class Runtime {
|
|
|
417
457
|
notifications,
|
|
418
458
|
runtimeName: this.config.name,
|
|
419
459
|
});
|
|
420
|
-
const riskGuard = riskBlock && senpi
|
|
421
|
-
? createRiskGuard({
|
|
422
|
-
senpi,
|
|
423
|
-
notifications,
|
|
424
|
-
strategies: new Map([[strategyAddress, this.strategyConfig]]),
|
|
425
|
-
// Human strategy label for alerts; risk-guard falls back to the
|
|
426
|
-
// truncated address when an address has no name here.
|
|
427
|
-
strategyNames: { [strategyAddress]: notificationLabel },
|
|
428
|
-
auditLog,
|
|
429
|
-
lifecycleTracker: lifecycle,
|
|
430
|
-
})
|
|
431
|
-
: createRiskGuardStub();
|
|
432
460
|
logger.info("Runtime.start() creating scanner module");
|
|
433
|
-
const layout = buildStateLayout(stateDir);
|
|
434
|
-
// Augment caller-supplied providers with StateManager-backed DSL state access
|
|
435
|
-
const augmentedProviders = {
|
|
436
|
-
...scannerProviders,
|
|
437
|
-
getDslStatesForStrategy: scannerProviders?.getDslStatesForStrategy
|
|
438
|
-
?? (async (address) => this.stateManager.listDslStates(address)),
|
|
439
|
-
};
|
|
440
|
-
this.scannerModule = createScannerModuleFromRuntimeConfig(this.config, layout.scannerRoot, augmentedProviders, undefined, buildIdentityEnvelope);
|
|
441
|
-
logger.info("Runtime.start() scanner module created, calling start()");
|
|
442
|
-
// Pre-load max leverage cache (non-blocking — log warning on failure)
|
|
443
|
-
const maxLeverageCache = new MaxLeverageCache(senpi);
|
|
444
|
-
try {
|
|
445
|
-
await maxLeverageCache.load();
|
|
446
|
-
}
|
|
447
|
-
catch (err) {
|
|
448
|
-
logger.warn("failed to pre-load max leverage cache, will retry on first use", { err });
|
|
449
|
-
}
|
|
450
|
-
// Build RuntimeContext once per trading recipe and reuse for all actions.
|
|
451
|
-
const runtimeContext = createRuntimeContext(this.strategyConfig, {
|
|
452
|
-
senpi,
|
|
453
|
-
hookSystem: this.hookSystem,
|
|
454
|
-
llmDecision,
|
|
455
|
-
contextBuilder,
|
|
456
|
-
riskGuard,
|
|
457
|
-
notifications,
|
|
458
|
-
scannerModule: this.scannerModule,
|
|
459
|
-
maxLeverageCache,
|
|
460
|
-
positionIdRegistry: this.positionIdRegistry,
|
|
461
|
-
notificationLabel,
|
|
462
|
-
});
|
|
463
|
-
this.actionRuntimeModule = new ActionRuntimeModule({ stateDir: layout.actionRoot });
|
|
464
|
-
const { runActionsForSignal } = await createActions(this.config, {
|
|
465
|
-
bus: this.bus,
|
|
466
|
-
senpi,
|
|
467
|
-
stateManager: this.stateManager,
|
|
468
|
-
hookSystem: this.hookSystem,
|
|
469
|
-
llmDecision,
|
|
470
|
-
contextBuilder,
|
|
471
|
-
riskGuard,
|
|
472
|
-
notifications,
|
|
473
|
-
runtimeContext,
|
|
474
|
-
actionModule: this.actionRuntimeModule,
|
|
475
|
-
});
|
|
476
|
-
this.runActionsForSignal = runActionsForSignal;
|
|
477
461
|
// Create the reconcile gate BEFORE DslPlugin.create() so the monitor's
|
|
478
462
|
// first synchronous tick (fired inside create()) sees a gate that already exists.
|
|
479
463
|
this.reconcileGate = (() => {
|
|
@@ -490,25 +474,22 @@ export class Runtime {
|
|
|
490
474
|
})();
|
|
491
475
|
// Build strategyStates map once — shared by DslPlugin and the reconcile loop.
|
|
492
476
|
const strategyStatesMap = createStrategyStatesMap(this.strategyConfig, senpi);
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
else {
|
|
510
|
-
this.dslPlugin = null;
|
|
511
|
-
}
|
|
477
|
+
// Everything a rebuild reuses. Config-derived values are deliberately absent: those are
|
|
478
|
+
// recomputed from the config each build, which is what makes a rebuild a rebuild.
|
|
479
|
+
this.componentDeps = {
|
|
480
|
+
senpi,
|
|
481
|
+
llmDecision,
|
|
482
|
+
contextBuilder,
|
|
483
|
+
lifecycle,
|
|
484
|
+
stateDir,
|
|
485
|
+
scannerProviders,
|
|
486
|
+
buildIdentityEnvelope,
|
|
487
|
+
strategyStates: strategyStatesMap,
|
|
488
|
+
};
|
|
489
|
+
// Once per runtime, not once per build: a rebuild must not re-run it. (It is a no-op today,
|
|
490
|
+
// kept for backward compatibility, but the placement is the contract.)
|
|
491
|
+
initRisk(this.bus);
|
|
492
|
+
await this.buildComponents(this.config);
|
|
512
493
|
// Launch detached boot-reconcile loop (does NOT block start()).
|
|
513
494
|
// Last-resort guard: an unexpected throw here would otherwise become an
|
|
514
495
|
// unhandled rejection and silently kill the loop. Do NOT resolve the gate
|
|
@@ -517,68 +498,8 @@ export class Runtime {
|
|
|
517
498
|
void this.runBootReconcileLoop(strategyStatesMap).catch((err) => {
|
|
518
499
|
logger.error("[runtime][reconcile] boot reconcile loop crashed", { err });
|
|
519
500
|
});
|
|
520
|
-
|
|
521
|
-
this.
|
|
522
|
-
runtimeName: this.config.name,
|
|
523
|
-
stateDir,
|
|
524
|
-
startedAt: this.startedAt,
|
|
525
|
-
providers: {
|
|
526
|
-
scanners: this.scannerModule,
|
|
527
|
-
dsl: this.dslPlugin ?? createDisabledDslProvider(),
|
|
528
|
-
actions: this.actionRuntimeModule,
|
|
529
|
-
risk: riskBlock
|
|
530
|
-
? createRiskHealthProvider(riskGuard, strategyAddress)
|
|
531
|
-
: undefined,
|
|
532
|
-
},
|
|
533
|
-
});
|
|
534
|
-
this.onScannerRunComplete = (event) => {
|
|
535
|
-
// Per-scan-run id: stamped on the scanner_run emit and echoed through the pass so scan +
|
|
536
|
-
// pass land in one trace. Prefer the id minted at the scanner.run span so scan + pass
|
|
537
|
-
// share one senpi.run.id.
|
|
538
|
-
const runId = event.result.runId ?? randomUUID();
|
|
539
|
-
this.signalQueue.push({ result: event.result, enqueuedAt: Date.now(), runId });
|
|
540
|
-
// Edge-triggered backpressure alarm: depth rises one enqueue at a time, so `=== threshold`
|
|
541
|
-
// fires exactly once as it crosses up and re-arms only after the consumer drains back below.
|
|
542
|
-
if (this.signalQueue.length === SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD) {
|
|
543
|
-
emitEvent(logger, () => buildQueueBackpressureEvent({
|
|
544
|
-
depth: this.signalQueue.length,
|
|
545
|
-
threshold: SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD,
|
|
546
|
-
scannerId: event.result.scannerId,
|
|
547
|
-
address: this.strategyConfig.address,
|
|
548
|
-
}));
|
|
549
|
-
}
|
|
550
|
-
if (this.signalQueueWait) {
|
|
551
|
-
const w = this.signalQueueWait;
|
|
552
|
-
this.signalQueueWait = null;
|
|
553
|
-
w();
|
|
554
|
-
}
|
|
555
|
-
// Scanner-run summary payload (startedAt is data stamping for telemetry durations).
|
|
556
|
-
const scannerRunPayload = {
|
|
557
|
-
address: event.result.address,
|
|
558
|
-
data: {
|
|
559
|
-
...event.result,
|
|
560
|
-
startedAt: event.startedAt,
|
|
561
|
-
runId,
|
|
562
|
-
},
|
|
563
|
-
timestamp: new Date().toISOString(),
|
|
564
|
-
};
|
|
565
|
-
// First-class event log. The build runs inside emitEvent's guard — telemetry never
|
|
566
|
-
// disturbs the scan path.
|
|
567
|
-
emitEvent(logger, () => buildScannerSummaryEvent(scannerRunPayload, Date.now()));
|
|
568
|
-
recordScannerRun({ scannerId: event.result.scannerId, outcome: event.result.status });
|
|
569
|
-
// TODO: Market-scanner alerts are default-OFF and MUST NOT be delivered
|
|
570
|
-
// until per-user default-mute-list support exists (today we have mute-list-only
|
|
571
|
-
// settings, which can't honour a default-OFF notification). When that lands,
|
|
572
|
-
// forward `event.result.summary?.alerts` as a MARKET_SCANNER notification with:
|
|
573
|
-
// in-app: `🔔 <b>${n} market alert(s)</b><br>${message}<br>Signals only — no trade placed yet.`
|
|
574
|
-
// push: no title (the notification service resolves it), body `🔔 ${message}`
|
|
575
|
-
// Address-less by design (the scanner watches the market, not one strategy), so
|
|
576
|
-
// it opens a chat by category alone and the strategy wallet must not leak into
|
|
577
|
-
// the seed. Re-add per-(scannerId, message) dedupe at that time.
|
|
578
|
-
// Resolve this TODO when default mute list functionality is available.
|
|
579
|
-
};
|
|
580
|
-
this.scannerModule.on("scanner:run:complete", this.onScannerRunComplete);
|
|
581
|
-
this.signalConsumerDone = this.runSignalConsumer();
|
|
501
|
+
this.wireScannerRunListener();
|
|
502
|
+
this.startSignalConsumer();
|
|
582
503
|
logger.info("Runtime.start() invoking scannerModule.start()");
|
|
583
504
|
await this.scannerModule.start();
|
|
584
505
|
logger.info("Runtime.start() scannerModule.start() returned");
|
|
@@ -617,10 +538,7 @@ export class Runtime {
|
|
|
617
538
|
// listeners, the DSL monitor interval, and the MCP connection. Best-effort — a
|
|
618
539
|
// teardown error must not mask the original boot error.
|
|
619
540
|
try {
|
|
620
|
-
|
|
621
|
-
this.scannerModule.off("scanner:run:complete", this.onScannerRunComplete);
|
|
622
|
-
this.onScannerRunComplete = null;
|
|
623
|
-
}
|
|
541
|
+
this.unwireScannerRunListeners();
|
|
624
542
|
if (this.dslPlugin) {
|
|
625
543
|
this.dslPlugin.stop();
|
|
626
544
|
this.dslPlugin = null;
|
|
@@ -699,6 +617,443 @@ export class Runtime {
|
|
|
699
617
|
* Cancel exchange SLs and archive DSL state for the strategy, then log removal.
|
|
700
618
|
*/
|
|
701
619
|
/** @returns Count of active DSL positions archived during teardown. */
|
|
620
|
+
/**
|
|
621
|
+
* Build every component derived from the recipe: scanners, actions, the DSL plugin, and the
|
|
622
|
+
* health surface that holds references to all three.
|
|
623
|
+
*
|
|
624
|
+
* Called by `start()` and again by {@link rebuildComponents}. Deliberately does NOT create the
|
|
625
|
+
* reconcile gate, launch the boot-reconcile loop, or call `initRisk` — those are once-per-runtime
|
|
626
|
+
* and re-running them would re-block trading on an already-converged runtime, duplicate a
|
|
627
|
+
* detached loop, and double-subscribe the bus respectively.
|
|
628
|
+
*/
|
|
629
|
+
async buildComponents(config) {
|
|
630
|
+
const constructed = await this.constructComponents(config);
|
|
631
|
+
await this.commitComponents(config, constructed);
|
|
632
|
+
}
|
|
633
|
+
/**
|
|
634
|
+
* Phase one of a build: everything the recipe determines, constructed but NOT installed on `this`
|
|
635
|
+
* and NOT started.
|
|
636
|
+
*
|
|
637
|
+
* The split exists so a rebuild can do all of its fallible work while the OLD components are
|
|
638
|
+
* still running. A recipe that names a scanner which will not construct, or an action class that
|
|
639
|
+
* does not exist, throws here — with the live runtime untouched, still scanning and still holding
|
|
640
|
+
* its stops. The alternative (the previous shape) tore everything down first and then discovered
|
|
641
|
+
* the new config was unbuildable, leaving a funded wallet with an open book, no scanners and no
|
|
642
|
+
* exit engine.
|
|
643
|
+
*
|
|
644
|
+
* Nothing in here may subscribe to the bus, start a timer, or touch disk. That is what makes it
|
|
645
|
+
* safe to run twice over — once for the attempt, once more for a rollback. `DslPlugin.create` is
|
|
646
|
+
* deliberately NOT called here for exactly that reason: it subscribes position handlers and
|
|
647
|
+
* starts the monitor loop, so pre-building it would put two plugins on one bus, both ticking the
|
|
648
|
+
* same positions. Its config is parsed here instead, which is where a bad preset actually throws.
|
|
649
|
+
*/
|
|
650
|
+
async constructComponents(config) {
|
|
651
|
+
const { senpi, llmDecision, contextBuilder, lifecycle, stateDir, scannerProviders, buildIdentityEnvelope, } = this.componentDeps;
|
|
652
|
+
const exitConfig = mapRuntimeExitToDslConfig(config.exit);
|
|
653
|
+
const hasDsl = exitConfig.dsl_preset != null &&
|
|
654
|
+
typeof exitConfig.dsl_preset === "object" &&
|
|
655
|
+
!Array.isArray(exitConfig.dsl_preset);
|
|
656
|
+
// Boot enforces this and a rebuild must too. Without it an update could keep `exit.dsl_preset`
|
|
657
|
+
// while dropping the position-tracker scanner (a pure removal, which the plan does not refuse)
|
|
658
|
+
// or its POSITION_TRACKER action: the DSL plugin is created, but nothing confirms position
|
|
659
|
+
// openings from clearing-house state, so every position opened afterwards gets no DSL state —
|
|
660
|
+
// no floor, no trailing stop, no exit — while the apply reports success. It also leaves the
|
|
661
|
+
// runtime in a config `start()` would throw on, so the next restart fails to boot at all.
|
|
662
|
+
//
|
|
663
|
+
// In the fallible phase deliberately: this throws while the OLD components are still running
|
|
664
|
+
// and still holding their stops.
|
|
665
|
+
validateDslPositionTracking(config, hasDsl);
|
|
666
|
+
// Parse the preset now rather than inside `DslPlugin.create`, so a malformed exit block is
|
|
667
|
+
// rejected while the old plugin is still holding its stops. The result is discarded — this is
|
|
668
|
+
// a validation, and `create` re-parses from the same input.
|
|
669
|
+
if (hasDsl)
|
|
670
|
+
parseDslPluginConfig(exitConfig);
|
|
671
|
+
const layout = buildStateLayout(stateDir);
|
|
672
|
+
// Augment caller-supplied providers with StateManager-backed DSL state access
|
|
673
|
+
const augmentedProviders = {
|
|
674
|
+
...scannerProviders,
|
|
675
|
+
getDslStatesForStrategy: scannerProviders?.getDslStatesForStrategy
|
|
676
|
+
?? (async (address) => this.stateManager.listDslStates(address)),
|
|
677
|
+
};
|
|
678
|
+
const scannerModule = createScannerModuleFromRuntimeConfig(config, layout.scannerRoot, augmentedProviders, undefined, buildIdentityEnvelope);
|
|
679
|
+
logger.info("Runtime.start() scanner module created, calling start()");
|
|
680
|
+
// Pre-load max leverage cache (non-blocking — log warning on failure)
|
|
681
|
+
const maxLeverageCache = new MaxLeverageCache(senpi);
|
|
682
|
+
try {
|
|
683
|
+
await maxLeverageCache.load();
|
|
684
|
+
}
|
|
685
|
+
catch (err) {
|
|
686
|
+
logger.warn("failed to pre-load max leverage cache, will retry on first use", { err });
|
|
687
|
+
}
|
|
688
|
+
// Resolved from the incoming recipe. On boot this reproduces what `start()` already assigned;
|
|
689
|
+
// on a rebuild it is the only thing that moves the new sizing into the actions.
|
|
690
|
+
const strategyConfig = resolveStrategyConfig(config);
|
|
691
|
+
// Both are recipe-derived, so both are rebuilt with it. Kept in the boot bag they were a third
|
|
692
|
+
// instance of the same silent no-op: `diffRecipes` reports `notifications.*`, `group` and
|
|
693
|
+
// `name` as changed and the apply returns success, while the running strategy went on notifying
|
|
694
|
+
// through the boot-time service under the boot-time label until the process restarted.
|
|
695
|
+
//
|
|
696
|
+
// Constructed here rather than at commit because `createNotificationService` THROWS when its
|
|
697
|
+
// token cannot be resolved — that belongs in the fallible phase, while the old components are
|
|
698
|
+
// still running, not in the window after teardown.
|
|
699
|
+
const notificationLabel = config.group ?? config.name;
|
|
700
|
+
const notifications = createNotificationService({
|
|
701
|
+
senpi_auth_token: this.options.apiKey,
|
|
702
|
+
...config.notifications,
|
|
703
|
+
});
|
|
704
|
+
// The guard is derived from `risk:`, so it is rebuilt with the recipe rather than kept in the
|
|
705
|
+
// boot bag. Kept there, three things were wrong at once and all of them silently: adding a
|
|
706
|
+
// `risk:` block to a running strategy registered health reporting around the boot-time STUB,
|
|
707
|
+
// which answers "enabled, healthy, OPEN" while enforcing nothing; changing a threshold moved
|
|
708
|
+
// the report and not the gate; and removing the block hid the health row while the real guard
|
|
709
|
+
// carried on refusing entries.
|
|
710
|
+
//
|
|
711
|
+
// Safe to rebuild because the guard is stateless by construction — every gate is derived from
|
|
712
|
+
// live venue history per call. The self-pause state lives in `lifecycle`, which is deliberately
|
|
713
|
+
// NOT rebuilt. The one thing lost is the in-memory gate-transition map, so a gate that is
|
|
714
|
+
// already closed may announce itself once more after an update.
|
|
715
|
+
const riskBlock = config.risk;
|
|
716
|
+
const auditLog = riskBlock ? createRiskAuditLog(stateDir) : undefined;
|
|
717
|
+
const riskGuard = riskBlock && senpi
|
|
718
|
+
? createRiskGuard({
|
|
719
|
+
senpi,
|
|
720
|
+
notifications,
|
|
721
|
+
strategies: new Map([[strategyConfig.address, strategyConfig]]),
|
|
722
|
+
// Human strategy label for alerts; risk-guard falls back to the
|
|
723
|
+
// truncated address when an address has no name here.
|
|
724
|
+
strategyNames: { [strategyConfig.address]: notificationLabel },
|
|
725
|
+
auditLog,
|
|
726
|
+
lifecycleTracker: lifecycle,
|
|
727
|
+
})
|
|
728
|
+
: createRiskGuardStub();
|
|
729
|
+
// Build RuntimeContext once per trading recipe and reuse for all actions.
|
|
730
|
+
const runtimeContext = createRuntimeContext(strategyConfig, {
|
|
731
|
+
senpi,
|
|
732
|
+
hookSystem: this.hookSystem,
|
|
733
|
+
llmDecision,
|
|
734
|
+
contextBuilder,
|
|
735
|
+
riskGuard,
|
|
736
|
+
notifications,
|
|
737
|
+
scannerModule,
|
|
738
|
+
maxLeverageCache,
|
|
739
|
+
positionIdRegistry: this.positionIdRegistry,
|
|
740
|
+
notificationLabel,
|
|
741
|
+
});
|
|
742
|
+
const actionRuntimeModule = new ActionRuntimeModule({ stateDir: layout.actionRoot });
|
|
743
|
+
const { runActionsForSignal } = await createActions(config, {
|
|
744
|
+
bus: this.bus,
|
|
745
|
+
senpi,
|
|
746
|
+
stateManager: this.stateManager,
|
|
747
|
+
hookSystem: this.hookSystem,
|
|
748
|
+
llmDecision,
|
|
749
|
+
contextBuilder,
|
|
750
|
+
riskGuard,
|
|
751
|
+
notifications,
|
|
752
|
+
runtimeContext,
|
|
753
|
+
actionModule: actionRuntimeModule,
|
|
754
|
+
});
|
|
755
|
+
return {
|
|
756
|
+
scannerModule,
|
|
757
|
+
actionRuntimeModule,
|
|
758
|
+
runActionsForSignal,
|
|
759
|
+
exitConfig,
|
|
760
|
+
hasDsl,
|
|
761
|
+
strategyConfig,
|
|
762
|
+
runtimeContext,
|
|
763
|
+
riskGuard,
|
|
764
|
+
notifications,
|
|
765
|
+
notificationLabel,
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
/**
|
|
769
|
+
* Phase two of a build: install the constructed components on `this`, then create the one
|
|
770
|
+
* component that cannot be constructed without side effects.
|
|
771
|
+
*
|
|
772
|
+
* Everything fallible that COULD be done earlier already has been, in
|
|
773
|
+
* {@link constructComponents}. What remains is assignment — which cannot throw — plus
|
|
774
|
+
* `DslPlugin.create`, whose config this build already parsed and whose state manager, senpi
|
|
775
|
+
* client and strategy-state map are all passed in (each of those resolvers returns immediately
|
|
776
|
+
* when supplied). That is why the teardown window in {@link rebuildComponents} is narrow enough
|
|
777
|
+
* to be worth taking, and why its rollback is a backstop rather than the plan.
|
|
778
|
+
*/
|
|
779
|
+
async commitComponents(config, built) {
|
|
780
|
+
const { senpi, stateDir, buildIdentityEnvelope, strategyStates: strategyStatesMap, } = this.componentDeps;
|
|
781
|
+
const riskBlock = config.risk;
|
|
782
|
+
// Install the recipe-derived strategy config before anything below reads it.
|
|
783
|
+
//
|
|
784
|
+
// The map is refreshed by replacing its entry rather than by minting a new Map: the boot
|
|
785
|
+
// reconcile loop and the DSL plugin were both handed this exact instance, and swapping the
|
|
786
|
+
// Map would leave them reading a container nobody writes to any more. Every consumer does
|
|
787
|
+
// `strategyStates.get(address)` at call time, so replacing the value is enough. The dropped
|
|
788
|
+
// StrategyState carried only venue caches (clearinghouse, open orders, past trades), which
|
|
789
|
+
// refetch on demand — and the swap runs under the reconcile lock, so no read is in flight.
|
|
790
|
+
this.strategyConfig = built.strategyConfig;
|
|
791
|
+
this.currentRuntimeContext = built.runtimeContext;
|
|
792
|
+
const strategyAddress = built.strategyConfig.address;
|
|
793
|
+
strategyStatesMap.set(strategyAddress, createStrategyState(built.strategyConfig, senpi));
|
|
794
|
+
this.scannerModule = built.scannerModule;
|
|
795
|
+
// Carried across the swap. Without it `derivePushDrivenScannerHealth` sees `lastAliveAt: null`
|
|
796
|
+
// for every external scanner, so each row reads `unknown` and the runtime's overall status
|
|
797
|
+
// follows — permanently, until the process restarts, rather than for the "next minute or so"
|
|
798
|
+
// an applied update's report promises while the scanners report back in.
|
|
799
|
+
if (this.supervisionSource) {
|
|
800
|
+
this.scannerModule.attachSupervisionSource(this.supervisionSource);
|
|
801
|
+
}
|
|
802
|
+
this.actionRuntimeModule = built.actionRuntimeModule;
|
|
803
|
+
this.runActionsForSignal = built.runActionsForSignal;
|
|
804
|
+
if (built.hasDsl) {
|
|
805
|
+
this.dslPlugin = await DslPluginClass.create({
|
|
806
|
+
config: built.exitConfig,
|
|
807
|
+
bus: this.bus,
|
|
808
|
+
stateDir,
|
|
809
|
+
stateManager: this.stateManager,
|
|
810
|
+
senpiClient: senpi,
|
|
811
|
+
strategyStates: strategyStatesMap,
|
|
812
|
+
notifications: built.notifications,
|
|
813
|
+
// Monitor runs the unified periodic reconcile (adopt/resize/gone), serialized
|
|
814
|
+
// against the trade loop on the same reconcileMutex key, and can resolve the boot gate.
|
|
815
|
+
reconcileSerialize: (fn) => this.reconcileMutex.withKeyLock("reconcile", fn),
|
|
816
|
+
reconcileGateResolve: () => this.resolveReconcileGate(),
|
|
817
|
+
identity: buildIdentityEnvelope,
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
else {
|
|
821
|
+
this.dslPlugin = null;
|
|
822
|
+
}
|
|
823
|
+
this.healthStateModule = createHealthStateModule({
|
|
824
|
+
runtimeName: config.name,
|
|
825
|
+
stateDir,
|
|
826
|
+
startedAt: this.startedAt ?? undefined,
|
|
827
|
+
providers: {
|
|
828
|
+
scanners: this.scannerModule,
|
|
829
|
+
dsl: this.dslPlugin ?? createDisabledDslProvider(),
|
|
830
|
+
actions: this.actionRuntimeModule,
|
|
831
|
+
risk: riskBlock
|
|
832
|
+
? createRiskHealthProvider(built.riskGuard, strategyAddress)
|
|
833
|
+
: undefined,
|
|
834
|
+
},
|
|
835
|
+
});
|
|
836
|
+
}
|
|
837
|
+
/**
|
|
838
|
+
* Register the scan-complete listener on the CURRENT scanner module.
|
|
839
|
+
*
|
|
840
|
+
* Separate from {@link buildComponents} because the listener is held against a specific module
|
|
841
|
+
* instance: a rebuild replaces the module, so the old registration goes with it and a fresh one
|
|
842
|
+
* has to be made against the new one.
|
|
843
|
+
*/
|
|
844
|
+
wireScannerRunListener() {
|
|
845
|
+
const { senpi } = this.componentDeps;
|
|
846
|
+
this.onScannerRunComplete = (event) => {
|
|
847
|
+
// Per-scan-run id: stamped on the scanner_run emit and echoed through the pass so scan +
|
|
848
|
+
// pass land in one trace. Prefer the id minted at the scanner.run span so scan + pass
|
|
849
|
+
// share one senpi.run.id.
|
|
850
|
+
const runId = event.result.runId ?? randomUUID();
|
|
851
|
+
this.signalQueue.push({ result: event.result, enqueuedAt: Date.now(), runId });
|
|
852
|
+
// Edge-triggered backpressure alarm: depth rises one enqueue at a time, so `=== threshold`
|
|
853
|
+
// fires exactly once as it crosses up and re-arms only after the consumer drains back below.
|
|
854
|
+
if (this.signalQueue.length === SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD) {
|
|
855
|
+
emitEvent(logger, () => buildQueueBackpressureEvent({
|
|
856
|
+
depth: this.signalQueue.length,
|
|
857
|
+
threshold: SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD,
|
|
858
|
+
scannerId: event.result.scannerId,
|
|
859
|
+
address: this.strategyConfig.address,
|
|
860
|
+
}));
|
|
861
|
+
}
|
|
862
|
+
if (this.signalQueueWait) {
|
|
863
|
+
const w = this.signalQueueWait;
|
|
864
|
+
this.signalQueueWait = null;
|
|
865
|
+
w();
|
|
866
|
+
}
|
|
867
|
+
// Scanner-run summary payload (startedAt is data stamping for telemetry durations).
|
|
868
|
+
const scannerRunPayload = {
|
|
869
|
+
address: event.result.address,
|
|
870
|
+
data: {
|
|
871
|
+
...event.result,
|
|
872
|
+
startedAt: event.startedAt,
|
|
873
|
+
runId,
|
|
874
|
+
},
|
|
875
|
+
timestamp: new Date().toISOString(),
|
|
876
|
+
};
|
|
877
|
+
// First-class event log. The build runs inside emitEvent's guard — telemetry never
|
|
878
|
+
// disturbs the scan path.
|
|
879
|
+
emitEvent(logger, () => buildScannerSummaryEvent(scannerRunPayload, Date.now()));
|
|
880
|
+
recordScannerRun({ scannerId: event.result.scannerId, outcome: event.result.status });
|
|
881
|
+
// TODO: Market-scanner alerts are default-OFF and MUST NOT be delivered
|
|
882
|
+
// until per-user default-mute-list support exists (today we have mute-list-only
|
|
883
|
+
// settings, which can't honour a default-OFF notification). When that lands,
|
|
884
|
+
// forward `event.result.summary?.alerts` as a MARKET_SCANNER notification with:
|
|
885
|
+
// in-app: `🔔 <b>${n} market alert(s)</b><br>${message}<br>Signals only — no trade placed yet.`
|
|
886
|
+
// push: no title (the notification service resolves it), body `🔔 ${message}`
|
|
887
|
+
// Address-less by design (the scanner watches the market, not one strategy), so
|
|
888
|
+
// it opens a chat by category alone and the strategy wallet must not leak into
|
|
889
|
+
// the seed. Re-add per-(scannerId, message) dedupe at that time.
|
|
890
|
+
// Resolve this TODO when default mute list functionality is available.
|
|
891
|
+
};
|
|
892
|
+
this.scannerModule.on("scanner:run:complete", this.onScannerRunComplete);
|
|
893
|
+
// An ERROR run-result is routed to `scanner:run:error`, never to `run:complete`, so the
|
|
894
|
+
// counter above never saw a failed run of any scanner — including a scaffold-reported
|
|
895
|
+
// tick failure from `/errors`. Count it here so `scanner.runs{outcome=error}` is a real
|
|
896
|
+
// series (the wedged-scanner alert is bridged from it); the health/telemetry side is
|
|
897
|
+
// already handled by the scanner module's own listener.
|
|
898
|
+
//
|
|
899
|
+
// Registered here rather than in `start()` so a rebuilt scanner module gets it too: the
|
|
900
|
+
// listener is bound to a module instance, and a rebuild replaces the module.
|
|
901
|
+
this.onScannerRunError = (event) => {
|
|
902
|
+
recordScannerRun({ scannerId: event.scannerId, outcome: "error" });
|
|
903
|
+
};
|
|
904
|
+
this.scannerModule.on("scanner:run:error", this.onScannerRunError);
|
|
905
|
+
}
|
|
906
|
+
/**
|
|
907
|
+
* Launch the single serial signal consumer. Called ONCE, from `start()`.
|
|
908
|
+
*
|
|
909
|
+
* Deliberately not part of {@link wireScannerRunListener}: that has to run again on every rebuild,
|
|
910
|
+
* because the listener is held against a specific scanner-module instance and the rebuild replaces
|
|
911
|
+
* it. Launching the consumer alongside it started a SECOND infinite `runSignalConsumer()` loop on
|
|
912
|
+
* every apply — nothing stopped the first, `signalConsumerStopped` is only set by `stop()`, and a
|
|
913
|
+
* rollback made it three.
|
|
914
|
+
*
|
|
915
|
+
* Two things broke. `signalQueueWait` holds exactly one resolver, so whichever consumer parked
|
|
916
|
+
* last owned the wake-up and `signalConsumerDone` tracked only the newest — `stop()` woke one
|
|
917
|
+
* consumer and then awaited a different one's promise, hanging `runtime delete`, gateway shutdown
|
|
918
|
+
* and SIGTERM forever on any runtime that had been updated once. And with a non-empty queue at
|
|
919
|
+
* swap time both consumers ran concurrently: the slot check is read BEFORE the reconcile lock, so
|
|
920
|
+
* two passes could each read `used = 0` and each open a position on a one-slot strategy.
|
|
921
|
+
*
|
|
922
|
+
* Serial signal processing is the invariant the whole trade path is built on; there must be
|
|
923
|
+
* exactly one of these per runtime.
|
|
924
|
+
*/
|
|
925
|
+
startSignalConsumer() {
|
|
926
|
+
if (this.signalConsumerDone)
|
|
927
|
+
return;
|
|
928
|
+
this.signalConsumerDone = this.runSignalConsumer();
|
|
929
|
+
}
|
|
930
|
+
/**
|
|
931
|
+
* Swap this runtime onto a new recipe without restarting it or touching its state.
|
|
932
|
+
*
|
|
933
|
+
* Teardown is the reverse of the build, and both halves matter. The scan listener is released
|
|
934
|
+
* before the module it is registered on; `scannerModule.stop()` clears its timers and awaits
|
|
935
|
+
* in-flight runs; `dslPlugin.stop()` is AWAITED, because it releases its bus subscriptions and
|
|
936
|
+
* drains the tick that may still be mid-flight at the venue — without that the outgoing monitor
|
|
937
|
+
* keeps writing state while the incoming one is already ticking the same positions, and the two
|
|
938
|
+
* hold independent StateManagers so nothing else serialises them.
|
|
939
|
+
*
|
|
940
|
+
* State on disk is untouched throughout: DSL position files, scanner stores and action history
|
|
941
|
+
* all survive, which is the entire reason an update is worth having over a delete-and-recreate.
|
|
942
|
+
*/
|
|
943
|
+
async rebuildComponents(config) {
|
|
944
|
+
// One rebuild at a time, for this runtime, whoever asks.
|
|
945
|
+
//
|
|
946
|
+
// Without it two concurrent applies interleave catastrophically: A tears down (setting
|
|
947
|
+
// `dslPlugin = null`), B's teardown then finds nothing to stop, and both commit. A's DSL plugin
|
|
948
|
+
// is never stopped — it keeps its bus subscriptions and its monitor loop, ticking the same
|
|
949
|
+
// positions through an independent StateManager and racing B over venue stops and closes. A's
|
|
950
|
+
// scanner module leaks the same way. `stop()` only ever tears down whatever `this.dslPlugin`
|
|
951
|
+
// and `this.scannerModule` point at, so the loser of the race is unreachable for the life of
|
|
952
|
+
// the process.
|
|
953
|
+
//
|
|
954
|
+
// A DIFFERENT key from "reconcile": `swapInComponents` takes that one inside this, and
|
|
955
|
+
// `KeyedMutex` is non-reentrant, so sharing the key would deadlock every rebuild.
|
|
956
|
+
return this.reconcileMutex.withKeyLock("rebuild", () => this.rebuildComponentsLocked(config));
|
|
957
|
+
}
|
|
958
|
+
async rebuildComponentsLocked(config) {
|
|
959
|
+
// `stop()` holds this same "rebuild" key, so a stop can never land inside the window below —
|
|
960
|
+
// but it can have COMPLETED while this call sat in the lock queue. Rebuilding then would
|
|
961
|
+
// resurrect components on a runtime that already reported stopped (over a disconnected senpi
|
|
962
|
+
// client), and `stop()`'s own `started` guard would make them unreachable forever.
|
|
963
|
+
if (!this.started) {
|
|
964
|
+
throw new Error("Runtime is stopped; there is nothing to rebuild in place.");
|
|
965
|
+
}
|
|
966
|
+
// Everything fallible happens FIRST, while the old components are still running. A recipe that
|
|
967
|
+
// will not build throws here, and the live runtime never notices: it is still scanning, still
|
|
968
|
+
// ticking, still holding the stops it has already promised.
|
|
969
|
+
const next = await this.constructComponents(config);
|
|
970
|
+
const previousConfig = this.config;
|
|
971
|
+
await this.teardownComponentsForRebuild();
|
|
972
|
+
this.config = config;
|
|
973
|
+
try {
|
|
974
|
+
await this.swapInComponents(config, next);
|
|
975
|
+
}
|
|
976
|
+
catch (err) {
|
|
977
|
+
// Backstop, not the plan. Reaching here means something threw in the narrow window that
|
|
978
|
+
// construction could not cover — realistically `DslPlugin.create` failing to resolve state
|
|
979
|
+
// it has already been handed. The runtime is currently torn down with an open book, so the
|
|
980
|
+
// only acceptable outcome is to get the PREVIOUS recipe running again.
|
|
981
|
+
logger.error("[runtime] rebuild failed after teardown — restoring the previous recipe", { err });
|
|
982
|
+
try {
|
|
983
|
+
await this.teardownComponentsForRebuild();
|
|
984
|
+
const restored = await this.constructComponents(previousConfig);
|
|
985
|
+
this.config = previousConfig;
|
|
986
|
+
await this.swapInComponents(previousConfig, restored);
|
|
987
|
+
}
|
|
988
|
+
catch (rollbackErr) {
|
|
989
|
+
// Both the new recipe and the old one failed to build. Nothing is managing the book, and
|
|
990
|
+
// no further automatic recovery is available — say so in the loudest terms the process has,
|
|
991
|
+
// naming the wallet, because a human has to act.
|
|
992
|
+
logger.error("[runtime] ROLLBACK FAILED — runtime is down with an open book", {
|
|
993
|
+
address: this.strategyConfig.address,
|
|
994
|
+
rebuildError: err,
|
|
995
|
+
rollbackError: rollbackErr,
|
|
996
|
+
});
|
|
997
|
+
throw new RuntimeRebuildRollbackFailedError(this.strategyConfig.address, err, rollbackErr);
|
|
998
|
+
}
|
|
999
|
+
// The runtime is healthy again on its previous recipe; the update itself did not apply.
|
|
1000
|
+
throw err;
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
/**
|
|
1004
|
+
* Install the freshly built components, serialized against the trade loop.
|
|
1005
|
+
*
|
|
1006
|
+
* The signal consumer runs every action pass inside
|
|
1007
|
+
* `reconcileMutex.withKeyLock("reconcile", () => this.runActionsForSignal(...))`, reading
|
|
1008
|
+
* `runActionsForSignal` at call time from inside the lock. Taking the same lock here means a
|
|
1009
|
+
* signal already in flight finishes against the OLD actions before the swap happens, and the next
|
|
1010
|
+
* one picks up the new ones — no pass can straddle the change.
|
|
1011
|
+
*
|
|
1012
|
+
* The lock is scoped to the SWAP and deliberately NOT to the teardown that precedes it.
|
|
1013
|
+
* `teardownComponentsForRebuild` awaits the DSL plugin's in-flight tick, and that tick takes this
|
|
1014
|
+
* very lock for its periodic reconcile (`runReconcile` is handed `reconcileSerialize`). Holding
|
|
1015
|
+
* the lock across the drain would deadlock the runtime — permanently, with an open book and
|
|
1016
|
+
* nothing managing it — because `KeyedMutex` is a plain FIFO and grants no reentrancy.
|
|
1017
|
+
*
|
|
1018
|
+
* Creating the DSL plugin inside the lock is safe for the mirror-image reason: `create()` starts
|
|
1019
|
+
* the monitor loop but does not await its first tick, so that tick queues behind this lock
|
|
1020
|
+
* instead of blocking the call that holds it.
|
|
1021
|
+
*/
|
|
1022
|
+
async swapInComponents(config, built) {
|
|
1023
|
+
await this.reconcileMutex.withKeyLock("reconcile", async () => {
|
|
1024
|
+
await this.commitComponents(config, built);
|
|
1025
|
+
// Listener only. The consumer already running is the one that must keep running — see
|
|
1026
|
+
// `startSignalConsumer`.
|
|
1027
|
+
this.wireScannerRunListener();
|
|
1028
|
+
});
|
|
1029
|
+
// Started OUTSIDE the lock, matching boot (`start()` calls this bare). `ScannerEngine.start()`
|
|
1030
|
+
// bootstraps context dependencies with real scanner runs — network time — and the DSL monitor's
|
|
1031
|
+
// tick awaits this same "reconcile" key before its price fetch, so holding the lock across the
|
|
1032
|
+
// start would stall breach evaluation and the trade loop on a live book for the duration of
|
|
1033
|
+
// those fetches. By here the components are committed and the listener wired, so a scan that
|
|
1034
|
+
// completes immediately lands in the queue like any other.
|
|
1035
|
+
await this.scannerModule.start();
|
|
1036
|
+
}
|
|
1037
|
+
/**
|
|
1038
|
+
* Drop the scan listener, stop the scanner module, stop and release the DSL plugin.
|
|
1039
|
+
*
|
|
1040
|
+
* Shared by the rebuild and its rollback so the two cannot drift — the rollback runs it a second
|
|
1041
|
+
* time to clear whatever the failed commit had already installed, and every step is idempotent
|
|
1042
|
+
* for exactly that reason.
|
|
1043
|
+
*
|
|
1044
|
+
* Order matters: the listener comes off before the module it is registered on;
|
|
1045
|
+
* `scannerModule.stop()` clears its timers and awaits in-flight runs; `dslPlugin.stop()` is
|
|
1046
|
+
* AWAITED, because it releases its bus subscriptions and drains the tick that may still be
|
|
1047
|
+
* mid-flight at the venue.
|
|
1048
|
+
*/
|
|
1049
|
+
async teardownComponentsForRebuild() {
|
|
1050
|
+
this.unwireScannerRunListeners();
|
|
1051
|
+
await this.scannerModule.stop();
|
|
1052
|
+
if (this.dslPlugin) {
|
|
1053
|
+
await this.dslPlugin.stop();
|
|
1054
|
+
this.dslPlugin = null;
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
702
1057
|
async removeStrategy(address) {
|
|
703
1058
|
let abandoned = 0;
|
|
704
1059
|
if (this.dslPlugin) {
|
|
@@ -947,6 +1302,17 @@ export class Runtime {
|
|
|
947
1302
|
}
|
|
948
1303
|
}
|
|
949
1304
|
async stop() {
|
|
1305
|
+
// Serialized on the same "rebuild" key as `rebuildComponents`. An unserialized stop landing in
|
|
1306
|
+
// the rebuild's teardown-to-swap window completes against the torn-down components and is then
|
|
1307
|
+
// undone by the swap: re-armed scanner timers and a fresh DSL monitor on a runtime that already
|
|
1308
|
+
// reported stopped — and since `started` is false by then, every later stop() returns at the
|
|
1309
|
+
// guard below, leaving the resurrected components running unreachably for the life of the
|
|
1310
|
+
// process. Waiting the rebuild out and tearing down whatever it installed is the only order
|
|
1311
|
+
// that ends with nothing running. ("rebuild", not "reconcile": the teardown below drains the
|
|
1312
|
+
// DSL tick, which takes "reconcile" — sharing that key would deadlock shutdown.)
|
|
1313
|
+
return this.reconcileMutex.withKeyLock("rebuild", () => this.stopLocked());
|
|
1314
|
+
}
|
|
1315
|
+
async stopLocked() {
|
|
950
1316
|
if (!this.started)
|
|
951
1317
|
return;
|
|
952
1318
|
this.started = false;
|
|
@@ -984,10 +1350,7 @@ export class Runtime {
|
|
|
984
1350
|
this.signalConsumerDone = null;
|
|
985
1351
|
}
|
|
986
1352
|
// riskGuardian cleanup removed — no background timer to stop
|
|
987
|
-
|
|
988
|
-
this.scannerModule.off("scanner:run:complete", this.onScannerRunComplete);
|
|
989
|
-
this.onScannerRunComplete = null;
|
|
990
|
-
}
|
|
1353
|
+
this.unwireScannerRunListeners();
|
|
991
1354
|
await this.scannerModule.stop();
|
|
992
1355
|
if (this.dslPlugin) {
|
|
993
1356
|
this.dslPlugin.stop();
|