@senpi/runtime 3.0.187 → 3.0.191

package/dist/index.js

@@ -17,7 +17,7 @@ import { join, dirname } from "node:path";
 import { isMoneyCallTool, SenpiClient, senpiOrderCallsInFlight } from "./senpi/client.js";
 import { MoneyCallTracker, registerMoneyCallHooks } from "./runtime/auto-update/money-call-tracker.js";
 import { buildWalletStatusHint } from "./senpi/strategy-status.js";
-import { bootSkippedStatusLabel, partitionByWalletStatus, readBootWalletStatuses, trackableWalletKeys, } from "./runtime/boot-skip.js";
+import { bootSkippedStatusLabel, partitionByWalletStatus, readBootWalletStatuses, readWalletStatusesByWorld, trackableWalletKeys, } from "./runtime/boot-skip.js";
 import { DeployJobController } from "./deploy/job.js";
 import { DEPLOY_CAPABILITIES, registerDeployGatewayMethods } from "./deploy/register.js";
 import { createQuantDesk } from "./quant/register.js";
@@ -60,6 +60,9 @@ import { registerSkillRouterHook } from "./prompt/skill-router-hook.js";
 import { registerCribSheet } from "./tool-surface/crib-hook.js";
 import { getDefaultPiiRedactor, initDefaultPiiRedactor } from "./telemetry/spans/redact-pii.js";
 import { flushOtelLogs, initOtelSdk, shutdownOtelSdk, waitForOtelInit } from "./telemetry/otel-sdk.js";
+import { createBoxMemoryReader } from "./telemetry/box-memory.js";
+import { registerBoxMemoryGauges } from "./telemetry/metrics.js";
+import { walletMode } from "./senpi/paper.js";
 import { getLogLevel, getLogLevelSource, parseLogLevel, setLogLevel, } from "./utils/logger.js";
 import { ExternalScannerIngestError, } from "./scanners/external-scanner-receiver.js";
 import { createLogger } from "./utils/logger.js";
@@ -227,10 +230,19 @@ function registerSenpiCliMetadata(api) {
         registerSenpiCli(ctx.program);
     }, { commands: ["senpi"] });
 }
+/** Running runtimes by world, read from each handle's wallet: a paper wallet is a simulation. */
+function countRuntimesByMode(handles) {
+    const counts = { live: 0, paper: 0 };
+    for (const handle of handles)
+        counts[walletMode(handle.wallet)] += 1;
+    return counts;
+}
 /**
  * OpenClaw plugin entry.
  * Boots from internal registry (installed_runtimes.json) in stateDir; does not read runtime list from config.
  */
+/** How long service start waits for the OTel SDK before registering the box memory gauges anyway. */
+const BOX_MEMORY_OTEL_READY_MS = 120_000;
 const senpiRuntimePlugin = {
     // Must match openclaw.plugin.json `id` — openclaw loader rejects the plugin
     // when `installed_plugins.json` record id (sourced from the manifest) and
@@ -656,6 +668,7 @@ const senpiRuntimePlugin = {
          */
         let serviceState = "idle";
         let serviceStops = 0;
+        let disposeBoxMemoryGauges;
         // Dedicated logger for the runtime-api subtree — required because the
         // PluginLogger surface (info/warn/error) lacks `debug`, while the
         // Logger contract used by `runtime-api/` includes it.
@@ -865,7 +878,7 @@ const senpiRuntimePlugin = {
                 if (!mcpTransport)
                     await client.connect();
                 try {
-                    return await client.listStrategies({ strategyAddresses: wallets }, { signal });
+                    return await client.listStrategiesForWallets(wallets, { signal });
                 }
                 finally {
                     if (!mcpTransport)
@@ -997,26 +1010,26 @@ const senpiRuntimePlugin = {
                 // lower-case storage, and rejects the whole batch on one malformed address.
                 const registeredWallets = trackableWalletKeys(registry.runtimes);
                 if (registeredWallets.length > 0) {
-                    const read = await readBootWalletStatuses(registeredWallets, async (wallets, signal) => {
+                    const read = await readWalletStatusesByWorld(registeredWallets, async (wallets, signal) => {
                         const client = mcpTransport ? new SenpiClient(mcpTransport) : new SenpiClient({ apiKey });
                         if (!mcpTransport)
                             await client.connect();
                         try {
-                            return await client.listStrategies({ strategyAddresses: wallets }, { signal });
+                            return await client.listStrategiesForWallets(wallets, { signal });
                         }
                         finally {
                             if (!mcpTransport)
                                 await client.disconnect().catch(() => { });
                         }
                     });
-                    if (read.ok) {
+                    if (read.answered.size > 0) {
                         walletStatuses = read.statuses;
                         bootStrategyIds = read.strategyIds;
-                        reaper.recordStatuses(registeredWallets, read.statuses);
+                        reaper.recordStatuses([...read.answered], read.statuses);
                     }
-                    else {
-                        pluginLogger.warn("[senpi-runtime] Could not read strategy statuses at boot; booting every registered runtime", { reason: read.reason, err: read.message });
-                        emitEvent(apiLogger, () => buildRuntimeBootStatusUnreadEvent({ reason: read.reason, registeredCount: registeredWallets.length }));
+                    for (const failure of read.failures) {
+                        pluginLogger.warn("[senpi-runtime] Could not read strategy statuses at boot; booting every registered runtime of that world", { reason: failure.reason, err: failure.message, paper: failure.paper });
+                        emitEvent(apiLogger, () => buildRuntimeBootStatusUnreadEvent({ reason: failure.reason, registeredCount: failure.walletCount }));
                     }
                 }
                 const { boot: entriesToBoot, skipped } = partitionByWalletStatus(registry.runtimes, walletStatuses);
@@ -1244,6 +1257,17 @@ const senpiRuntimePlugin = {
                 strategiesPublisher.start();
                 // Sends the events added since the previous send, every 1 s.
                 eventsPublisher.start();
+                // The metrics API has no proxy meter: gauges made before the SDK sets its provider never export.
+                const stopsAtStart = serviceStops;
+                void waitForOtelInit(BOX_MEMORY_OTEL_READY_MS).then(() => {
+                    if (serviceStops !== stopsAtStart)
+                        return;
+                    disposeBoxMemoryGauges?.();
+                    disposeBoxMemoryGauges = registerBoxMemoryGauges({
+                        read: createBoxMemoryReader({ warn: (message, meta) => pluginLogger.warn(message, meta) }),
+                        runtimes: () => countRuntimesByMode(runtimeHandles.values()),
+                    });
+                });
                 // Last, once everything above is up: the reaper's post-boot pass and hourly sweep. Its
                 // not-active escalation is also off until here, so no delete can land while the boot loop is
                 // still wiring runtimes and launchers. Nothing here may fail start() — a throw from inside the boot loop takes every runtime down, which
@@ -1273,6 +1297,8 @@ const senpiRuntimePlugin = {
                 await reaper.stop();
                 strategiesPublisher.stop();
                 eventsPublisher.stop();
+                disposeBoxMemoryGauges?.();
+                disposeBoxMemoryGauges = undefined;
                 // Drain the HTTP server FIRST so in-flight POST /signals requests
                 // finish before the underlying runtime they target is torn down.
                 // `apiHandle.close()` never throws.
@@ -1833,7 +1859,7 @@ const senpiRuntimePlugin = {
                         if (!mcpTransport)
                             await client.connect();
                         try {
-                            const strategies = await client.listStrategies({ strategyAddresses: [wallet] });
+                            const strategies = await client.listStrategiesForWallets([wallet]);
                             const match = strategies.find((s) => s.strategyWalletAddress.toLowerCase() === wallet.toLowerCase());
                             if (!match) {
                                 pluginLogger.info("[senpi-runtime] Wallet validation failed: not owned", { wallet });
@@ -2021,7 +2047,8 @@ const senpiRuntimePlugin = {
             const targetAddress = typeof params?.address === "string" ? params.address : undefined;
             // Set only by the lifecycle reaper: why the strategy counts as gone (stamped on runtime.deleted)
             // and which wallet that verdict was about (the concurrency guard below).
-            const deleteReason = params?.deleteReason === "closed_status" ? params.deleteReason
+            const deleteReason = params?.deleteReason === "closed_status" || params?.deleteReason === "paper_failed_status"
+                ? params.deleteReason
                 : undefined;
             const expectWallet = typeof params?.expectWallet === "string" && params.expectWallet.trim().length > 0
                 ? params.expectWallet.trim().toLowerCase()
@@ -2265,7 +2292,7 @@ const senpiRuntimePlugin = {
                     return;
                 let ids = strategyIds ?? null;
                 if (strategyIds === undefined && !plan.unchanged) {
-                    const read = await readBootWalletStatuses([plan.wallet.trim().toLowerCase()], (wallets, signal) => withSenpiClient((client) => client.listStrategies({ strategyAddresses: wallets }, { signal })));
+                    const read = await readBootWalletStatuses([plan.wallet.trim().toLowerCase()], (wallets, signal) => withSenpiClient((client) => client.listStrategiesForWallets(wallets, { signal })));
                     if (!read.ok) {
                         factsLogger.warn("strategy facts failed: could not read the strategy id; retrying next boot", undefined, {
                             runtimeId: plan.runtimeId,
@@ -2513,6 +2540,10 @@ const senpiRuntimePlugin = {
             },
             registryFindByWallet: findRegistryEntryByWallet,
             registryFindById: findRegistryEntryById,
+            countRegistryRuntimes: async () => {
+                const { readRegistry } = await import("./runtime/strategy-registry.js");
+                return (await readRegistry(baseStateDir)).runtimes.length;
+            },
             // A deploy job outlives a reload; it reads the runtime where it runs (see `liveHostIfStopped`).
             getScannerRows: (id) => liveHostIfStopped()?.getScannerRows(id) ?? readScannerRows(id),
             // H4: `runtimeHandles` holds the live `run()` handles. A registry row without one is a