@senpi/runtime 3.0.118 → 3.0.123

package/dist/index.js

@@ -16,10 +16,11 @@ import { parse as parseYaml } from "yaml";
 import { join, dirname } from "node:path";
 import { SenpiClient } from "./senpi/client.js";
 import { buildWalletStatusHint } from "./senpi/strategy-status.js";
+import { bootSkippedStatusLabel, partitionByWalletStatus, readBootWalletStatuses, trackableWalletKeys, } from "./runtime/boot-skip.js";
 import { DeployJobController } from "./deploy/job.js";
 import { registerDeployGatewayMethods } from "./deploy/register.js";
 import { registerUpdateGatewayMethod } from "./runtime/update-register.js";
-import { checkProofCoversRecipe, makeApplyToRuntime, makeWriteRegistry, } from "./runtime/update-apply-deps.js";
+import { checkProofCoversRecipe, makeApplyToRuntime, makeWriteRegistry, runtimeChangeLock, } from "./runtime/update-apply-deps.js";
 import { INSTALL_DEADLINE_MS, WALLET_INSTALL_IN_FLIGHT_CODE } from "./deploy/deadlines.js";
 import { AuditQueryError } from "./runtime/audit-query.types.js";
 import { buildRuntimeStateDir, archiveStateDir } from "./runtime/state-layout.js";
@@ -52,7 +53,8 @@ import { registerToolApprovalGate, resolveToolApprovalsMode, } from "./approvals
 import { registerShowWidgetTool } from "./widgets/index.js";
 import { EVENT_NAME, SENPI_ERROR_CODE, eventAttrs, scannerUnwiredErrorCode } from "./utils/event-catalog.js";
 import { emitEvent } from "./actions/event-derivation.js";
-import { buildRuntimeCreatedEvent, buildRuntimeDeletedEvent, buildRuntimeUpdatedEvent } from "./runtime/lifecycle-events.js";
+import { createLifecycleReaper, resolveReaperBootDelayMs, resolveReaperEnforcement, } from "./runtime/lifecycle-reaper.js";
+import { buildRuntimeBootSkippedEvent, buildRuntimeBootStatusUnreadEvent, buildRuntimeCreatedEvent, buildRuntimeDeletedEvent, buildRuntimeUpdatedEvent, } from "./runtime/lifecycle-events.js";
 import { isWalletAddress } from "./utils/wallet-address.js";
 import { fileURLToPath } from "node:url";
 import { loadApiConfig } from "./runtime-api/config.js";
@@ -374,6 +376,10 @@ const senpiRuntimePlugin = {
         // cleared on a successful wire/install. senpi.listRuntimes reads it to render
         // "running — NO ENTRY SCANNERS" instead of a bare "running" for a blind runtime.
         const unwiredRuntimes = new Map();
+        // Registered runtimes boot left down because the platform reports their strategy wallet dead,
+        // keyed runtimeId → that status. senpi.listRuntimes reads it so the row says why it is stopped;
+        // cleared when the row is deleted.
+        const bootSkippedRuntimes = new Map();
         /**
          * Wallets with an install IN FLIGHT in this process, keyed by lowercased address.
          *
@@ -638,6 +644,39 @@ const senpiRuntimePlugin = {
             now: () => Date.now(),
             log: (line) => api.logger.info(`[senpi-runtime] ${line}`),
         });
+        // ── lifecycle reaper ─────────────────────────────────────────────────────────────────────────
+        // A runtime exists iff its strategy is open. Every close path ends at the backend and none of
+        // them tells this box, so the reaper asks: once after boot, hourly, and whenever the backend
+        // rejects an order with STRATEGY_NOT_ACTIVE. Observe-only unless enforcement is on (see
+        // runtime/lifecycle-reaper.ts for the safety rule). Constructed here so both `run()` call sites
+        // (boot and install) can hand it their not-active rejections; it does no I/O until started.
+        const reaperBootDelayOverride = resolveReaperBootDelayMs(process.env);
+        const reaper = createLifecycleReaper({
+            readRegistry: async () => {
+                const { readRegistry } = await import("./runtime/strategy-registry.js");
+                return readRegistry(baseStateDir);
+            },
+            readStatuses: async (wallets, signal) => {
+                const client = mcpTransport ? new SenpiClient(mcpTransport) : new SenpiClient({ apiKey });
+                if (!mcpTransport)
+                    await client.connect();
+                try {
+                    return await client.listStrategies({ strategyAddresses: wallets }, { signal });
+                }
+                finally {
+                    if (!mcpTransport)
+                        await client.disconnect().catch(() => { });
+                }
+            },
+            deleteRuntime: async (id, reason, wallet) => {
+                const result = await deleteRuntimeCore({ id, deleteReason: reason, expectWallet: wallet });
+                return result.ok ? { ok: true } : { ok: false, code: result.error.code, message: result.error.message };
+            },
+            emit: (build) => emitEvent(apiLogger, build),
+            enforce: resolveReaperEnforcement(process.env),
+            logger: pluginLogger,
+            ...(reaperBootDelayOverride !== undefined ? { bootDelayMs: reaperBootDelayOverride } : {}),
+        });
         api.registerService({
             id: "senpi-runtime",
             async start() {
@@ -714,9 +753,48 @@ const senpiRuntimePlugin = {
                     // publishes a level rather than recording a run.
                     routeScaffoldTickToTelemetry = buildScaffoldTickRouter(ctx);
                 }
+                // One bounded read: which registered wallets the platform says are dead, so a restart does
+                // not re-scaffold runtimes on closed wallets (fail-open — see runtime/boot-skip.ts).
+                let walletStatuses = null;
+                // Lower-cased and trackable-only: the backend matches addresses case-sensitively against
+                // lower-case storage, and rejects the whole batch on one malformed address.
+                const registeredWallets = trackableWalletKeys(registry.runtimes);
+                if (registeredWallets.length > 0) {
+                    const read = await readBootWalletStatuses(registeredWallets, async (wallets, signal) => {
+                        const client = mcpTransport ? new SenpiClient(mcpTransport) : new SenpiClient({ apiKey });
+                        if (!mcpTransport)
+                            await client.connect();
+                        try {
+                            return await client.listStrategies({ strategyAddresses: wallets }, { signal });
+                        }
+                        finally {
+                            if (!mcpTransport)
+                                await client.disconnect().catch(() => { });
+                        }
+                    });
+                    if (read.ok) {
+                        walletStatuses = read.statuses;
+                        reaper.recordStatuses(registeredWallets, read.statuses);
+                    }
+                    else {
+                        pluginLogger.warn("[senpi-runtime] Could not read strategy statuses at boot; booting every registered runtime", { reason: read.reason, err: read.message });
+                        emitEvent(apiLogger, () => buildRuntimeBootStatusUnreadEvent({ reason: read.reason, registeredCount: registeredWallets.length }));
+                    }
+                }
+                const { boot: entriesToBoot, skipped } = partitionByWalletStatus(registry.runtimes, walletStatuses);
+                bootSkippedRuntimes.clear();
+                for (const { entry, status } of skipped) {
+                    bootSkippedRuntimes.set(entry.id, status);
+                    pluginLogger.info("[senpi-runtime] Not booting runtime: its strategy wallet is closed", {
+                        runtimeId: entry.id,
+                        wallet: entry.wallet,
+                        status,
+                    });
+                    emitEvent(apiLogger, () => buildRuntimeBootSkippedEvent({ runtimeId: entry.id, address: entry.wallet ?? "", status }));
+                }
                 const started = [];
                 try {
-                    for (const entry of registry.runtimes) {
+                    for (const entry of entriesToBoot) {
                         const hasPath = typeof entry.runtimeYamlPath === "string" && entry.runtimeYamlPath.length > 0;
                         const hasContent = typeof entry.runtimeYamlContent === "string";
                         if (hasPath && hasContent)
@@ -735,6 +813,7 @@ const senpiRuntimePlugin = {
                             apiKey,
                             scannerProviders,
                             runtimeId: entry.id,
+                            onStrategyNotActive: (address) => reaper.checkWallet(address),
                         });
                         runtimeHandles.set(entry.id, handle);
                         started.push(entry.id);
@@ -911,6 +990,21 @@ const senpiRuntimePlugin = {
                 if (process.env.SENPI_RUNTIME_API_DISABLED !== "true") {
                     liveTelemetry.start();
                 }
+                // Last, once everything above is up: the reaper's post-boot pass and hourly sweep. Its
+                // not-active escalation is also off until here, so no delete can land while the boot loop is
+                // still wiring runtimes and launchers. Nothing here may fail start() — a throw from inside the boot loop takes every runtime down, which
+                // is exactly the failure a lifecycle check must never cause. The unit-test seam
+                // SENPI_RUNTIME_API_DISABLED keeps its timers off unless a boot delay is set explicitly.
+                try {
+                    if (process.env.SENPI_RUNTIME_API_DISABLED !== "true" || reaperBootDelayOverride !== undefined) {
+                        reaper.start();
+                    }
+                }
+                catch (err) {
+                    pluginLogger.warn("[senpi-runtime] lifecycle reaper did not start", {
+                        err: err instanceof Error ? err.message : String(err),
+                    });
+                }
             },
             async stop() {
                 // Live telemetry goes first: it owns a timer and an in-memory buffer, nothing downstream
@@ -918,6 +1012,10 @@ const senpiRuntimePlugin = {
                 // runtime handle. Records emitted during the teardown below land in a buffer nobody drains,
                 // which is the right trade for a process on its way out.
                 liveTelemetry.stop();
+                // Then the reaper, before the runtimes are torn down below, so no pass starts deleting
+                // while they go away. The wait is bounded, so a pass already under way cannot stop a handle
+                // the teardown below is stopping too.
+                await reaper.stop();
                 // Drain the HTTP server FIRST so in-flight POST /signals requests
                 // finish before the underlying runtime they target is torn down.
                 // `apiHandle.close()` never throws.
@@ -1250,6 +1348,7 @@ const senpiRuntimePlugin = {
                         apiKey,
                         scannerProviders,
                         runtimeId: resolvedId,
+                        onStrategyNotActive: (address) => reaper.checkWallet(address),
                     });
                     runtimeHandles.set(resolvedId, handle);
                     if (parsedConfig)
@@ -1385,8 +1484,11 @@ const senpiRuntimePlugin = {
                 // say so rather than reporting a bare "running" the operator would read as healthy.
                 // unwiredRuntimes is populated at the four wire/launch failure sites (and is
                 // exactly the set of "declared external scanners but has no launcher").
+                const bootSkipStatus = bootSkippedRuntimes.get(e.id);
                 const status = !runtimeHandles.has(e.id)
-                    ? "stopped"
+                    ? bootSkipStatus !== undefined
+                        ? bootSkippedStatusLabel(bootSkipStatus)
+                        : "stopped"
                     : unwiredRuntimes.has(e.id)
                         ? "running — NO ENTRY SCANNERS"
                         : "running";
@@ -1397,6 +1499,10 @@ const senpiRuntimePlugin = {
                         ? e.runtimeYamlPath
                         : "(inline)",
                     status,
+                    // What the strategy behind this runtime last read as (open / closing / closed / absent /
+                    // unchecked / unknown) — separate from `status`, whose running…/stopped… prefixes other
+                    // readers key on.
+                    ...reaper.lifecycleOf(e.wallet),
                     // The deploy path installs by CONTENT, not path (orchestrator.ts installRuntime), so
                     // `source` is "(inline)" for every deployed strategy and the YAML is unreachable through
                     // it. The engine renders the facts instead of shipping a blob to re-interpret.
@@ -1411,13 +1517,19 @@ const senpiRuntimePlugin = {
          * unwired-verdict cleanup all happen exactly once, here.
          */
         async function deleteRuntimeCore(params) {
-            const { readRegistry, writeRegistry, withRegistryLock } = await import("./runtime/strategy-registry.js");
-            const registry = await readRegistry(baseStateDir);
             let targetId = typeof params?.id === "string" ? params.id : undefined;
             const targetAddress = typeof params?.address === "string" ? params.address : undefined;
+            // Set only by the lifecycle reaper: why the strategy counts as gone (stamped on runtime.deleted)
+            // and which wallet that verdict was about (the concurrency guard below).
+            const deleteReason = params?.deleteReason === "closed_status" ? params.deleteReason
+                : undefined;
+            const expectWallet = typeof params?.expectWallet === "string" && params.expectWallet.trim().length > 0
+                ? params.expectWallet.trim().toLowerCase()
+                : undefined;
             // Resolve target by address if id not provided
             if (!targetId && targetAddress) {
-                const entry = registry.runtimes.find((e) => e.wallet === targetAddress);
+                const { readRegistry } = await import("./runtime/strategy-registry.js");
+                const entry = (await readRegistry(baseStateDir)).runtimes.find((e) => e.wallet === targetAddress);
                 if (!entry) {
                     return { ok: false, error: {
                             code: "NOT_FOUND",
@@ -1432,6 +1544,24 @@ const senpiRuntimePlugin = {
                         message: "Please provide the runtime id or wallet address. Example: openclaw senpi runtime delete --id my-runtime",
                     } };
             }
+            // One delete or update apply per runtime at a time (see `runtimeChangeLock`). Operator, agent
+            // and deploy deletes wait their turn; the lifecycle reaper never queues behind someone else's
+            // change to the runtime it judged — it backs off and the next lifecycle check re-reads it.
+            // The check and the acquire below run in one synchronous step.
+            const id = targetId;
+            if (expectWallet !== undefined && runtimeChangeLock.hasKey(id)) {
+                return { ok: false, error: {
+                        code: "CONFLICT",
+                        message: `Runtime "${id}" was not removed: another delete or an update is already running on it. Nothing was changed; the next lifecycle check re-reads it.`,
+                    } };
+            }
+            return runtimeChangeLock.withKeyLock(id, () => deleteRuntimeLocked(id, deleteReason, expectWallet));
+        }
+        /** {@link deleteRuntimeCore} past target resolution, under the runtime's change lock. */
+        async function deleteRuntimeLocked(targetId, deleteReason, expectWallet) {
+            const { readRegistry, writeRegistry, withRegistryLock } = await import("./runtime/strategy-registry.js");
+            // Read under the lock: a delete that waited behind another change sees what that change left.
+            const registry = await readRegistry(baseStateDir);
             const deletedEntry = registry.runtimes.find((e) => e.id === targetId);
             const handle = runtimeHandles.get(targetId);
             if (!deletedEntry && !handle) {
@@ -1440,6 +1570,35 @@ const senpiRuntimePlugin = {
                         message: `No runtime found with id "${targetId}".`,
                     } };
             }
+            /**
+             * The lifecycle reaper's guard. Its verdict was about one wallet at one moment; the delete
+             * below must not land on anything else. Evaluated synchronously between the registry read
+             * above and the first teardown step, so nothing can interleave between check and act:
+             * - an install on that wallet is running (it could land a live runtime under this id);
+             * - a deploy job holds the single-flight slot (it retires and reinstalls runtimes);
+             * - the row under this id no longer carries that wallet (ids derive from name + wallet and
+             *   are reused, so a reinstall onto a new wallet can take the old id);
+             * - a live handle under this id trades a different wallet.
+             * Operator, agent and deploy deletes pass no `expectWallet` and are unaffected.
+             */
+            if (expectWallet !== undefined) {
+                const fold = (w) => (w ?? "").trim().toLowerCase();
+                const conflict = installsInFlight.has(expectWallet)
+                    ? "an install is running on that wallet"
+                    : deployController.running() !== null
+                        ? "a deploy is in progress"
+                        : !deletedEntry || fold(deletedEntry.wallet) !== expectWallet
+                            ? "the registry row under that id no longer carries that wallet"
+                            : handle && fold(handle.wallet) !== expectWallet
+                                ? "the running runtime under that id trades a different wallet"
+                                : undefined;
+                if (conflict !== undefined) {
+                    return { ok: false, error: {
+                            code: "CONFLICT",
+                            message: `Runtime "${targetId}" was not removed: ${conflict}. Nothing was changed; the next lifecycle check re-reads it.`,
+                        } };
+                }
+            }
             if (handle) {
                 await handle.teardownDslForRuntimeDeletion();
                 await handle.stop();
@@ -1462,6 +1621,7 @@ const senpiRuntimePlugin = {
             // Same reuse hazard: a deleted id left in the index would keep publishing a strategy that no
             // longer exists, and would hand a reinstall the old name and scanner list.
             telemetryStrategies.delete(targetId);
+            bootSkippedRuntimes.delete(targetId);
             // Archive state directory before removing from registry
             if (deletedEntry) {
                 const runtimeStateDir = buildRuntimeStateDir(baseStateDir, targetId, deletedEntry.wallet ?? "");
@@ -1487,6 +1647,7 @@ const senpiRuntimePlugin = {
                 runtimeId: targetId,
                 address: deletedEntry?.wallet,
                 hadLiveHandle: Boolean(handle),
+                deleteReason,
             }));
             return { ok: true, payload: { id: targetId } };
         }