@senpi-ai/runtime 3.0.70 → 3.0.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -2
- package/dist/cli/deploy-render.d.ts +93 -0
- package/dist/cli/deploy-render.d.ts.map +1 -0
- package/dist/cli/deploy-render.js +438 -0
- package/dist/cli/deploy-render.js.map +1 -0
- package/dist/cli/guide-commands.d.ts.map +1 -1
- package/dist/cli/guide-commands.js +94 -9
- package/dist/cli/guide-commands.js.map +1 -1
- package/dist/cli/senpi-commands.d.ts +6 -1
- package/dist/cli/senpi-commands.d.ts.map +1 -1
- package/dist/cli/senpi-commands.js +290 -41
- package/dist/cli/senpi-commands.js.map +1 -1
- package/dist/cli/validate-command.d.ts.map +1 -1
- package/dist/cli/validate-command.js +10 -1
- package/dist/cli/validate-command.js.map +1 -1
- package/dist/deploy/deadlines.d.ts +107 -0
- package/dist/deploy/deadlines.d.ts.map +1 -0
- package/dist/deploy/deadlines.js +139 -0
- package/dist/deploy/deadlines.js.map +1 -0
- package/dist/deploy/error-text.d.ts +25 -0
- package/dist/deploy/error-text.d.ts.map +1 -0
- package/dist/deploy/error-text.js +42 -0
- package/dist/deploy/error-text.js.map +1 -0
- package/dist/deploy/funding.d.ts +125 -0
- package/dist/deploy/funding.d.ts.map +1 -0
- package/dist/deploy/funding.js +268 -0
- package/dist/deploy/funding.js.map +1 -0
- package/dist/deploy/install-gate-runner.d.ts +12 -0
- package/dist/deploy/install-gate-runner.d.ts.map +1 -0
- package/dist/deploy/install-gate-runner.js +17 -0
- package/dist/deploy/install-gate-runner.js.map +1 -0
- package/dist/deploy/job.d.ts +215 -0
- package/dist/deploy/job.d.ts.map +1 -0
- package/dist/deploy/job.js +484 -0
- package/dist/deploy/job.js.map +1 -0
- package/dist/deploy/messages.d.ts +603 -0
- package/dist/deploy/messages.d.ts.map +1 -0
- package/dist/deploy/messages.js +1388 -0
- package/dist/deploy/messages.js.map +1 -0
- package/dist/deploy/min-budget.d.ts +68 -0
- package/dist/deploy/min-budget.d.ts.map +1 -0
- package/dist/deploy/min-budget.js +297 -0
- package/dist/deploy/min-budget.js.map +1 -0
- package/dist/deploy/orchestrator.d.ts +314 -0
- package/dist/deploy/orchestrator.d.ts.map +1 -0
- package/dist/deploy/orchestrator.js +2171 -0
- package/dist/deploy/orchestrator.js.map +1 -0
- package/dist/deploy/package.d.ts +228 -0
- package/dist/deploy/package.d.ts.map +1 -0
- package/dist/deploy/package.js +542 -0
- package/dist/deploy/package.js.map +1 -0
- package/dist/deploy/proof-gate.d.ts +86 -0
- package/dist/deploy/proof-gate.d.ts.map +1 -0
- package/dist/deploy/proof-gate.js +278 -0
- package/dist/deploy/proof-gate.js.map +1 -0
- package/dist/deploy/python-float.d.ts +34 -0
- package/dist/deploy/python-float.d.ts.map +1 -0
- package/dist/deploy/python-float.js +46 -0
- package/dist/deploy/python-float.js.map +1 -0
- package/dist/deploy/recheck-commands.d.ts +50 -0
- package/dist/deploy/recheck-commands.d.ts.map +1 -0
- package/dist/deploy/recheck-commands.js +77 -0
- package/dist/deploy/recheck-commands.js.map +1 -0
- package/dist/deploy/register.d.ts +101 -0
- package/dist/deploy/register.d.ts.map +1 -0
- package/dist/deploy/register.js +714 -0
- package/dist/deploy/register.js.map +1 -0
- package/dist/deploy/taxonomy.d.ts +43 -0
- package/dist/deploy/taxonomy.d.ts.map +1 -0
- package/dist/deploy/taxonomy.js +59 -0
- package/dist/deploy/taxonomy.js.map +1 -0
- package/dist/dsl/events/handlers.d.ts +6 -4
- package/dist/dsl/events/handlers.d.ts.map +1 -1
- package/dist/dsl/events/handlers.js +29 -17
- package/dist/dsl/events/handlers.js.map +1 -1
- package/dist/dsl/events/initial-state.d.ts +21 -11
- package/dist/dsl/events/initial-state.d.ts.map +1 -1
- package/dist/dsl/events/initial-state.js +6 -0
- package/dist/dsl/events/initial-state.js.map +1 -1
- package/dist/dsl/notifications/floor-store.d.ts.map +1 -1
- package/dist/dsl/notifications/floor-store.js +34 -5
- package/dist/dsl/notifications/floor-store.js.map +1 -1
- package/dist/dsl/notifications/formatter.d.ts.map +1 -1
- package/dist/dsl/notifications/formatter.js +13 -5
- package/dist/dsl/notifications/formatter.js.map +1 -1
- package/dist/dsl/notifications/index.d.ts +18 -1
- package/dist/dsl/notifications/index.d.ts.map +1 -1
- package/dist/dsl/notifications/index.js +78 -11
- package/dist/dsl/notifications/index.js.map +1 -1
- package/dist/dsl/notifications/stop-raise-debounce.d.ts +113 -0
- package/dist/dsl/notifications/stop-raise-debounce.d.ts.map +1 -0
- package/dist/dsl/notifications/stop-raise-debounce.js +223 -0
- package/dist/dsl/notifications/stop-raise-debounce.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +467 -222
- package/dist/index.js.map +1 -1
- package/dist/runtime/env-resolve.d.ts +13 -0
- package/dist/runtime/env-resolve.d.ts.map +1 -1
- package/dist/runtime/env-resolve.js +14 -1
- package/dist/runtime/env-resolve.js.map +1 -1
- package/dist/runtime/runtime-descriptor.d.ts +37 -0
- package/dist/runtime/runtime-descriptor.d.ts.map +1 -0
- package/dist/runtime/runtime-descriptor.js +163 -0
- package/dist/runtime/runtime-descriptor.js.map +1 -0
- package/dist/runtime/runtime-schema.d.ts +100 -100
- package/dist/runtime/strategy-registry.d.ts +8 -0
- package/dist/runtime/strategy-registry.d.ts.map +1 -1
- package/dist/runtime/strategy-registry.js +14 -1
- package/dist/runtime/strategy-registry.js.map +1 -1
- package/dist/runtime-api/intake/intake-mount.d.ts.map +1 -1
- package/dist/runtime-api/intake/intake-mount.js +7 -4
- package/dist/runtime-api/intake/intake-mount.js.map +1 -1
- package/dist/scanners/external/scaffold/preflight.py +39 -0
- package/dist/scanners/implementations/market-regime.d.ts +1 -1
- package/dist/senpi/client.d.ts +58 -4
- package/dist/senpi/client.d.ts.map +1 -1
- package/dist/senpi/client.js +262 -18
- package/dist/senpi/client.js.map +1 -1
- package/dist/senpi/strategy-status.d.ts +32 -0
- package/dist/senpi/strategy-status.d.ts.map +1 -0
- package/dist/senpi/strategy-status.js +70 -0
- package/dist/senpi/strategy-status.js.map +1 -0
- package/dist/senpi/types.d.ts +65 -3
- package/dist/senpi/types.d.ts.map +1 -1
- package/dist/skills-manager/manifest.js +3 -3
- package/dist/skills-manager/skills-manifest.json +3 -3
- package/dist/telemetry/metrics.d.ts +30 -0
- package/dist/telemetry/metrics.d.ts.map +1 -1
- package/dist/telemetry/metrics.js +72 -0
- package/dist/telemetry/metrics.js.map +1 -1
- package/dist/types/dsl/index.d.ts +14 -0
- package/dist/types/dsl/index.d.ts.map +1 -1
- package/dist/types/strategy.d.ts +9 -1
- package/dist/types/strategy.d.ts.map +1 -1
- package/dist/universe/package-universe.d.ts +97 -0
- package/dist/universe/package-universe.d.ts.map +1 -0
- package/dist/universe/package-universe.js +304 -0
- package/dist/universe/package-universe.js.map +1 -0
- package/dist/validate/codes.d.ts +30 -0
- package/dist/validate/codes.d.ts.map +1 -1
- package/dist/validate/codes.js +74 -1
- package/dist/validate/codes.js.map +1 -1
- package/dist/validate/index.d.ts +1 -1
- package/dist/validate/index.d.ts.map +1 -1
- package/dist/validate/index.js.map +1 -1
- package/dist/validate/install-gate.d.ts.map +1 -1
- package/dist/validate/install-gate.js +10 -2
- package/dist/validate/install-gate.js.map +1 -1
- package/dist/validate/live-instruments.d.ts +44 -0
- package/dist/validate/live-instruments.d.ts.map +1 -0
- package/dist/validate/live-instruments.js +71 -0
- package/dist/validate/live-instruments.js.map +1 -0
- package/dist/validate/live-stage.d.ts +8 -0
- package/dist/validate/live-stage.d.ts.map +1 -1
- package/dist/validate/live-stage.js +87 -18
- package/dist/validate/live-stage.js.map +1 -1
- package/dist/validate/package-gates-stage.d.ts +61 -0
- package/dist/validate/package-gates-stage.d.ts.map +1 -0
- package/dist/validate/package-gates-stage.js +215 -0
- package/dist/validate/package-gates-stage.js.map +1 -0
- package/dist/validate/proof.d.ts +11 -1
- package/dist/validate/proof.d.ts.map +1 -1
- package/dist/validate/proof.js +10 -6
- package/dist/validate/proof.js.map +1 -1
- package/dist/validate/recipe-checks.d.ts.map +1 -1
- package/dist/validate/recipe-checks.js +6 -4
- package/dist/validate/recipe-checks.js.map +1 -1
- package/dist/validate/run.d.ts +29 -1
- package/dist/validate/run.d.ts.map +1 -1
- package/dist/validate/run.js +65 -6
- package/dist/validate/run.js.map +1 -1
- package/dist/validate/target.d.ts +41 -0
- package/dist/validate/target.d.ts.map +1 -1
- package/dist/validate/target.js +192 -16
- package/dist/validate/target.js.map +1 -1
- package/dist/validate/taxonomy-finding.d.ts +22 -0
- package/dist/validate/taxonomy-finding.d.ts.map +1 -0
- package/dist/validate/taxonomy-finding.js +25 -0
- package/dist/validate/taxonomy-finding.js.map +1 -0
- package/dist/validate/universe-stage.d.ts +71 -0
- package/dist/validate/universe-stage.d.ts.map +1 -0
- package/dist/validate/universe-stage.js +165 -0
- package/dist/validate/universe-stage.js.map +1 -0
- package/dist/validate/wallet-binding-stage.d.ts +44 -0
- package/dist/validate/wallet-binding-stage.d.ts.map +1 -0
- package/dist/validate/wallet-binding-stage.js +92 -0
- package/dist/validate/wallet-binding-stage.js.map +1 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,2171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `senpi deploy` state machine.
|
|
3
|
+
*
|
|
4
|
+
* `runDeploy` is a pure, sequential orchestration over injected collaborators:
|
|
5
|
+
* it performs no I/O of its own beyond reading the package's own YAML, holds no
|
|
6
|
+
* timers, and never touches the job slot, the journal, or the gateway. Every
|
|
7
|
+
* async concern lives in `job.ts`, so this whole flow tests on fakes and a
|
|
8
|
+
* virtual clock.
|
|
9
|
+
*
|
|
10
|
+
* Two rules shape the design and must survive every edit:
|
|
11
|
+
* - **Decisions read live surfaces only** — backend strategies and the runtime
|
|
12
|
+
* registry. Emitted events are narration; a throwing `emit` cannot alter the
|
|
13
|
+
* flow, and nothing ever reads a journal back to decide anything. Exactly one
|
|
14
|
+
* carve-out exists: the journal may supply a *lookup key* (a strategyId this
|
|
15
|
+
* box journaled at create time, `params.priorStrategyIds`) that is always
|
|
16
|
+
* verified against the live backend before use — never a decision.
|
|
17
|
+
* - **Abandonment is honored at step boundaries and between poll iterations
|
|
18
|
+
* only** — never mid-MCP-call, so money-moving calls are never left in a
|
|
19
|
+
* half-known state. The deadline watchdog in `job.ts` is the only thing that
|
|
20
|
+
* raises it (there is no cancel verb: undeploying is closing the strategy).
|
|
21
|
+
* Nothing is rolled back **except the one D-6 case: a wallet this job created
|
|
22
|
+
* whose install then failed**; the report says what completed.
|
|
23
|
+
*/
|
|
24
|
+
import { join } from "node:path";
|
|
25
|
+
import { buildUniverseNotLiveRefusal, buildUniverseUnavailableDetail, checkPackageUniverse, describeUniverseVerdict, } from "../universe/package-universe.js";
|
|
26
|
+
import { instancesWithoutDslExit, nonLowercasePackageId, renderRuntimeYaml, runtimeUsesLlmDecisions, readRuntimeYaml, sanitizeStrategyName, scannersDeclaringEnabled, unboundWalletInstances, walletName, } from "./package.js";
|
|
27
|
+
import { DEAD_STATUSES, INITIALIZING_STATUSES, isDead, isInitializing, } from "../senpi/strategy-status.js";
|
|
28
|
+
import { MIN_WALLET, accessibleUsd, planFunding, resumeCommand, sharesOverAuthorization, underfundedNote, usd, } from "./funding.js";
|
|
29
|
+
import { DEFAULT_MAX_WAIT_SEC, DEFAULT_TICK_WAIT_SEC, INSTALL_DEADLINE_MS, INSTALL_INDETERMINATE_CODE, isInstallDeadlineExceeded, } from "./deadlines.js";
|
|
30
|
+
import { packageMinBudget } from "./min-budget.js";
|
|
31
|
+
import { errorText } from "./error-text.js";
|
|
32
|
+
import { gateInstance } from "./install-gate-runner.js";
|
|
33
|
+
import { buildInstanceDirCollisionRefusal, buildProofRefusal, checkPackageProof, collidingInstanceDirs, proofRefusalParts, } from "./proof-gate.js";
|
|
34
|
+
import { buildAmbiguousRefusal, buildBindingUnknownRefusal, buildBudgetEscape, buildBudgetNotPositiveRefusal, buildBudgetRequiredRefusal, buildCreateAbandonedDetail, buildCreateDeadDetail, buildCreateNameRetryNotIssuedDetail, buildCreateNotDeployableDetail, buildCreatePendingDetail, buildDecisionModelEnvRefusal, buildDecisionModelRefusal, buildForeignPackageRefusal, buildFundedUnreadableNote, buildInstallGateRefusal, buildInstallGateUnavailableDetail, buildInstallIndeterminateDetail, buildInstallInFlightDetail, buildNoDslExitRefusal, buildNonLowercaseIdRefusal, buildNotDeployableRefusal, buildPartialFundNote, buildScannerEnabledUnsupportedRefusal, buildShareOverrunRefusal, buildSoftBudgetNote, buildUnboundWalletRefusal, buildUnobservedDetail, buildUnreadableWalletRefusal, buildUnwiredInstallDetail, foreignCollisionClause, forwardTense, fundedClause, itsFunded, matchedByClause, reconcileCanMatchByName, rollbackFundedClause, } from "./messages.js";
|
|
35
|
+
const CREATE_POLL_MS = 10_000;
|
|
36
|
+
const OBSERVE_POLL_MS = 5_000;
|
|
37
|
+
const STEP_ORDER = ["reconcile", "preflight", "create", "install", "observe"];
|
|
38
|
+
/**
|
|
39
|
+
* Backend codes that mean "the name is unusable AND nothing was created", so retrying unnamed is
|
|
40
|
+
* free — see {@link isNameRejection}. Spelled out one per entry rather than packed into a character
|
|
41
|
+
* class (`SERR05[568]`) because the reader who arrives here arrives holding a code: a grep for the
|
|
42
|
+
* exact string they were handed has to land on this list.
|
|
43
|
+
*
|
|
44
|
+
* All three are `INVALID_ARGUMENT` in the MCP's registry (`senpi-error-registry.ts:155-167`):
|
|
45
|
+
* deterministic input refusals. 055 is the active-name collision; 056 and 058 are the length and
|
|
46
|
+
* whitespace rules, enforced by zod before the request ever reaches the backend. The backend
|
|
47
|
+
* rejected the input and created nothing, so a retry cannot double-fund.
|
|
48
|
+
*
|
|
49
|
+
* SERR083 was here and is now DELIBERATELY EXCLUDED (decision, 2026-08-11). Its text is generic
|
|
50
|
+
* ("Custom strategy creation failed unexpectedly. Retry with the same payload.") and the registry
|
|
51
|
+
* classes it UNAVAILABLE/`retryable` — meaning the outcome is unknown and a wallet may exist behind
|
|
52
|
+
* it. The case it was added for is real: a strategy still holding its name made every named create
|
|
53
|
+
* fail, and two calls 3.6s apart at the same $693 budget went named → SERR083, unnamed → success.
|
|
54
|
+
* But that unblock is bought with a retry that funds a SECOND wallet whenever the first create did
|
|
55
|
+
* land, and the runtime has a cheaper route to the same outcome: reconcile reads the live
|
|
56
|
+
* strategies and adopts an initializing record BY NAME before the create call is ever made
|
|
57
|
+
* (`awaitingActive`), which is exactly the state that produces SERR083. deploy.py, where this guard
|
|
58
|
+
* came from, had no such step — that is why the code list travelled here with 083 in it. So the
|
|
59
|
+
* defensive arm was chosen: no retry, and the operator gets the backend's real words (item 1) in
|
|
60
|
+
* place of "backend returned no strategyId".
|
|
61
|
+
*/
|
|
62
|
+
const NAME_REJECTION_CODES = ["SERR055", "SERR056", "SERR058"];
|
|
63
|
+
/** The `SERRnnn` token in a code or a message, upper-cased; `undefined` when there is none. */
|
|
64
|
+
function readSerrCode(text) {
|
|
65
|
+
return text?.toUpperCase().match(/SERR\d{3}/)?.[0];
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* True when a failed create is the backend refusing the wallet NAME **and nothing was created**, so
|
|
69
|
+
* retrying unnamed works and cannot cost a second funded wallet.
|
|
70
|
+
*
|
|
71
|
+
* Keyed on the CODE, not on prose. Reading the code was impossible until the client learned to
|
|
72
|
+
* parse the error envelope: a backend refusal has no `data` key, so the old flat `data.message` read
|
|
73
|
+
* returned `undefined` for every one of them and this predicate was called with `""`. That made the
|
|
74
|
+
* safe arm — the enumerated backend codes — dead, while the dangerous arm stayed live: `callCreate`
|
|
75
|
+
* shapes every THROW into `{ success: false, message: errorText(err) }`, and a transport error's
|
|
76
|
+
* text is authored outside this repo. The predicate the codes were written for could never fire;
|
|
77
|
+
* the only failures that could reach the retry were the ones whose outcome nobody knows.
|
|
78
|
+
*
|
|
79
|
+
* Three arms, in this order:
|
|
80
|
+
*
|
|
81
|
+
* 1. **A throw is never a name rejection.** `callTool` throws only on an RPC fault
|
|
82
|
+
* (`senpi/client.ts` `callToolInner`); every backend refusal RESOLVES as an error envelope. So
|
|
83
|
+
* a throw here means transport, and a transport failure raised after the request was written —
|
|
84
|
+
* socket reset, 502, a gateway-injected transport failing mid-call — may have already created
|
|
85
|
+
* and funded a wallet. Retrying funds a second one beside it. Refusing on `threw` is stricter
|
|
86
|
+
* than telling pre-send from post-send (a connection refused before the request left is safe to
|
|
87
|
+
* retry), and it is deliberately the strict version: the give-up costs one legible name, the
|
|
88
|
+
* false positive costs a funded wallet. A deadline abort is one member of this class, not the
|
|
89
|
+
* whole of it — `aborted` flags only the member `callCreate` recognises.
|
|
90
|
+
* 2. **A SERR code decides, wherever it is written.** The structured `errorCode` first, then the
|
|
91
|
+
* message — because the zod-enforced rules (056 length, 058 whitespace) arrive as
|
|
92
|
+
* `INVALID_ARGUMENT` with the SERR code in the TEXT (`classifyError` → `InvalidArgumentError`),
|
|
93
|
+
* while a backend-mapped refusal carries it structurally. If a code is found, it decides
|
|
94
|
+
* alone: an out-of-list code (SERR083, SERR045, SERR037) is a NO, and the text is not consulted
|
|
95
|
+
* afterwards, so a non-name refusal can never sneak in on a stray word.
|
|
96
|
+
* 3. **Text, only when there is no code at all.** So an uncoded future refusal is not silently
|
|
97
|
+
* unmatchable — the reason the bare-word arm existed. It matches the WORD `name`
|
|
98
|
+
* (`\bname\b`), not the substring: `HOSTNAME`, `USERNAME`, `UNNAMED` and an echoed
|
|
99
|
+
* `strategyName` key all fail the word boundary, which is the false-positive class the
|
|
100
|
+
* follow-up ticket named. And because arm 1 already returned, this arm can only ever see a
|
|
101
|
+
* RETURNED failure — the backend answered, so nothing was created — which is why widening it
|
|
102
|
+
* cannot re-open the outcome-unknown hole.
|
|
103
|
+
*
|
|
104
|
+
* Code matching is case-INSENSITIVE, as the regex this replaced was: the backend spells them
|
|
105
|
+
* upper-case today, and nothing here is worth losing the unblock over if some relay downcases a
|
|
106
|
+
* message. (deploy.py compared the codes case-sensitively; that is the one deliberate difference,
|
|
107
|
+
* and it is in the direction of firing more often, never less.)
|
|
108
|
+
*/
|
|
109
|
+
function isNameRejection(result) {
|
|
110
|
+
if (result.threw)
|
|
111
|
+
return false;
|
|
112
|
+
const code = readSerrCode(result.errorCode) ?? readSerrCode(result.message);
|
|
113
|
+
if (code)
|
|
114
|
+
return NAME_REJECTION_CODES.includes(code);
|
|
115
|
+
return /\bname\b/i.test(result.message ?? "");
|
|
116
|
+
}
|
|
117
|
+
const ABANDONED_DETAIL = "abandoned at the deploy deadline — the job exceeded its wall-clock cap";
|
|
118
|
+
/** Ticks that prove the scanner executed; `ok` literally means "found a signal", so heartbeat counts. */
|
|
119
|
+
const OBSERVED_RUN_STATUSES = ["ok", "heartbeat"];
|
|
120
|
+
function isFreshTick(row, sinceMs) {
|
|
121
|
+
if (row.scheduleMode === "external") {
|
|
122
|
+
// C1: intake short-circuits empty POSTs before run telemetry, so a watch-only external
|
|
123
|
+
// scanner never records a lastRunStatus. lastAliveAt is id-keyed and advances every POST.
|
|
124
|
+
return row.lastAliveAt !== null && row.lastAliveAt >= sinceMs;
|
|
125
|
+
}
|
|
126
|
+
return (row.lastRunStatus !== null &&
|
|
127
|
+
OBSERVED_RUN_STATUSES.includes(row.lastRunStatus) &&
|
|
128
|
+
row.lastRunFinishedAt !== null &&
|
|
129
|
+
row.lastRunFinishedAt >= sinceMs // H3: pre-install telemetry survives a re-mint
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Did THIS row record a run of its own since the install — regardless of how it is scheduled?
|
|
134
|
+
*
|
|
135
|
+
* Kept separate from {@link isFreshTick}, which is the GATE and must not change: its external
|
|
136
|
+
* branch short-circuits on `lastAliveAt` before reading these fields, deliberately, because a
|
|
137
|
+
* watch-only external scanner can never record a run status and demanding one would make its
|
|
138
|
+
* deploy unable to ever reach `live`. This is the RENDER classifier — it decides which of the two
|
|
139
|
+
* claims the observe line is allowed to make, and it must answer for the row in front of it rather
|
|
140
|
+
* than for its schedule mode, since an external scanner that posts non-empty signals does commit
|
|
141
|
+
* run telemetry.
|
|
142
|
+
*
|
|
143
|
+
* Merging the two would be the bug: the gate would start demanding what this asks for.
|
|
144
|
+
*/
|
|
145
|
+
function recordedFreshRun(row, sinceMs) {
|
|
146
|
+
return (row.lastRunStatus !== null &&
|
|
147
|
+
OBSERVED_RUN_STATUSES.includes(row.lastRunStatus) &&
|
|
148
|
+
row.lastRunFinishedAt !== null &&
|
|
149
|
+
row.lastRunFinishedAt >= sinceMs);
|
|
150
|
+
}
|
|
151
|
+
function isFreshError(row, sinceMs) {
|
|
152
|
+
return (row.lastRunStatus === "error" &&
|
|
153
|
+
row.lastRunFinishedAt !== null &&
|
|
154
|
+
row.lastRunFinishedAt >= sinceMs);
|
|
155
|
+
}
|
|
156
|
+
/** deploy.py's `_budget_verdict` tolerance: below this fraction of the ask, say so out loud. */
|
|
157
|
+
const BUDGET_TOLERANCE = 0.9;
|
|
158
|
+
/**
|
|
159
|
+
* Did the D-6 rollback's `strategy_close` actually run? Returns the reason it did not, or null when
|
|
160
|
+
* the close is PROVEN — only an explicit `success: true` proves it.
|
|
161
|
+
*
|
|
162
|
+
* The client resolves a tool-level `strategy_close` failure as `{ success: false }` and maps an
|
|
163
|
+
* unrecognizable payload onto that same shape (senpi/client.ts); it throws only on an RPC fault. So
|
|
164
|
+
* "the promise resolved" is not evidence, and reading it as one reported funds returned from a
|
|
165
|
+
* wallet that is still live and still funded.
|
|
166
|
+
*/
|
|
167
|
+
function describeCloseFailure(result) {
|
|
168
|
+
const record = typeof result === "object" && result !== null
|
|
169
|
+
? result
|
|
170
|
+
: undefined;
|
|
171
|
+
if (record?.success === true)
|
|
172
|
+
return null;
|
|
173
|
+
const message = typeof record?.message === "string" && record.message.length > 0
|
|
174
|
+
? record.message
|
|
175
|
+
: "";
|
|
176
|
+
return message
|
|
177
|
+
? `\`strategy_close\` reported failure: ${message}`
|
|
178
|
+
: `\`strategy_close\` did not report success (it returned ${JSON.stringify(result) ?? String(result)})`;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* A call this job stopped waiting on at its own deadline, as opposed to one the backend failed.
|
|
182
|
+
* `boundAwait` (senpi/client.ts) names those `AbortError`; the self-managed MCP transport surfaces
|
|
183
|
+
* the same shape as a `TimeoutError`. The distinction is load-bearing on the money path: a failure
|
|
184
|
+
* means nothing happened, an abandonment means nobody knows — and the two must not read alike.
|
|
185
|
+
*/
|
|
186
|
+
function isDeadlineAbort(err) {
|
|
187
|
+
return err instanceof Error && (err.name === "AbortError" || err.name === "TimeoutError");
|
|
188
|
+
}
|
|
189
|
+
// ─── refusal / message builders (one producer per taxonomy string) ───────────
|
|
190
|
+
function buildAdoptedWalletDetail(pkg, inst, matched, wallet, budget, matchedBy = "name") {
|
|
191
|
+
const how = matchedBy === "created-key" ? "matched by journaled create key, verified against the backend" : "strategyName match";
|
|
192
|
+
const base = `reusing ${matched.status} wallet ${wallet} (${how}), ${fundedClause(matched.totalFunded)}`;
|
|
193
|
+
if (budget === undefined)
|
|
194
|
+
return base;
|
|
195
|
+
const share = inst.fundingShare ?? 1 / pkg.instances.length;
|
|
196
|
+
const intended = Math.max(MIN_WALLET, Math.round(budget * share * 100) / 100);
|
|
197
|
+
// An unread figure is not a comparison input. Collapsing it to 0 made this line say "the
|
|
198
|
+
// requested $500 was NOT added — top it up, or close this strategy first" about a wallet the
|
|
199
|
+
// clause above calls unreadable, and it put a second money authority on a report whose
|
|
200
|
+
// [W_BUDGET_FUNDED_UNREADABLE] already says "Move no money on this". Nothing is steered from a
|
|
201
|
+
// number nobody read: state the gap as UNKNOWN and defer to the balance being established.
|
|
202
|
+
if (matched.totalFunded === undefined) {
|
|
203
|
+
return (`${base} — deploy never funds an existing wallet, and whether the ${usd(intended)} requested ` +
|
|
204
|
+
`for this instance is already on it is UNKNOWN here (unreadable is never zero). No money move ` +
|
|
205
|
+
`follows from this read: establish the wallet's actual balance with the USER first, and only ` +
|
|
206
|
+
`then decide whether anything needs topping up or resizing.`);
|
|
207
|
+
}
|
|
208
|
+
if (matched.totalFunded >= intended * BUDGET_TOLERANCE)
|
|
209
|
+
return base;
|
|
210
|
+
return (`${base} — the requested ${usd(intended)} for this instance was NOT added; deploy never funds an ` +
|
|
211
|
+
`existing wallet. To add funds, top it up separately; to deploy at the requested size, close this ` +
|
|
212
|
+
`strategy first and re-run.`);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* How `[E_ROLLBACK_INCOMPLETE]` should finish its sentence, given what else is live on this package.
|
|
216
|
+
*
|
|
217
|
+
* The stranded wallet has no runtime, so `close.py --instance` cannot resolve it — the only
|
|
218
|
+
* per-wallet route is a direct `strategy_close` on the address, which the caller states first. What
|
|
219
|
+
* this adds is whether the package-wide `close.py <id>` is a safe convenience or a trap: it is fine
|
|
220
|
+
* when this stranded wallet is all that exists, and destructive the moment another sleeve is live,
|
|
221
|
+
* because it would take that one down too while "reclaiming" this one.
|
|
222
|
+
*/
|
|
223
|
+
function buildStrandedCloseCaveat(pkg, runs, stranded) {
|
|
224
|
+
const otherLive = runs.filter((run) => run !== stranded && run.report.wallet && !run.report.rolledBack);
|
|
225
|
+
// `pkg.id` is quoted the way `--instance` is: these strings are copied into a shell verbatim.
|
|
226
|
+
if (otherLive.length === 0) {
|
|
227
|
+
return (`. Nothing else in this package is live, so \`python3 senpi-strategy-ops/scripts/close.py ` +
|
|
228
|
+
`'${pkg.id}'\` is equivalent and safe here.`);
|
|
229
|
+
}
|
|
230
|
+
return (`. Do NOT reach for \`close.py '${pkg.id}'\` to do it: ${otherLive.length} other sleeve(s) ` +
|
|
231
|
+
`(${otherLive.map((run) => run.inst.name).join(", ")}) hold live funded wallets and that ` +
|
|
232
|
+
`command would close them too. Read them first if you need context: ` +
|
|
233
|
+
`\`python3 senpi-strategy-ops/scripts/status.py '${pkg.id}'\`.`);
|
|
234
|
+
}
|
|
235
|
+
/** Instance names a re-run of this package would have to fund: everything without a live wallet. */
|
|
236
|
+
function walletlessAfter(pkg, runs) {
|
|
237
|
+
const held = new Set(runs.filter((run) => run.report.wallet && !run.report.rolledBack).map((run) => run.inst.name));
|
|
238
|
+
return pkg.instances.map((inst) => inst.name).filter((name) => !held.has(name));
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* The start-over clause for a PAUSED poll, scoped to this job's whole INTENT, never to a mid-loop
|
|
242
|
+
* snapshot.
|
|
243
|
+
*
|
|
244
|
+
* The string is frozen the moment the poll settles, but creates run one instance at a time and a
|
|
245
|
+
* `create: failed` does not halt the loop — a sibling still queued behind this one can be funded
|
|
246
|
+
* and installed AFTER the text is written. So "live" here means live-or-intended: a sibling that
|
|
247
|
+
* already holds a wallet, one this job is still going to create (`needsWallet`) or wait on
|
|
248
|
+
* (`awaitingActive`) — or one whose reconcile FAILED, whose live state is therefore unknown
|
|
249
|
+
* (the fail-closed `strategy_list` throw means "unknown, not empty", and unknown must withdraw
|
|
250
|
+
* like live does, never read as "not live"). Judging `report.wallet` alone let this same job
|
|
251
|
+
* falsify the "nothing else in this package is live" claim on the very report that carries it,
|
|
252
|
+
* two lines below the offer.
|
|
253
|
+
* ({@link buildStrandedCloseCaveat} may judge present state — it runs at rollback time, after the
|
|
254
|
+
* whole create phase.)
|
|
255
|
+
*
|
|
256
|
+
* The remedy split mirrors that caveat with opposite emphasis: there the command being scoped is a
|
|
257
|
+
* per-wallet `strategy_close` and the package-wide one is merely warned off; here the package-wide
|
|
258
|
+
* teardown IS the offer, so when any sibling is live or intended the command is WITHDRAWN and
|
|
259
|
+
* replaced by the per-strategy route. Reconcile can hand out `close.py <pkg>` unscoped because it
|
|
260
|
+
* finishes for every instance before any wallet exists; this seam cannot.
|
|
261
|
+
*
|
|
262
|
+
* Rendered mid-sentence after "If you meant to start over instead, " and ends with its own period
|
|
263
|
+
* — the caller appends nothing, so the seam cannot splice ungrammatically again.
|
|
264
|
+
*/
|
|
265
|
+
function buildPackageCloseOffer(pkg, runs, self, live) {
|
|
266
|
+
const otherLive = runs.filter((run) => run !== self &&
|
|
267
|
+
(run.report.wallet ||
|
|
268
|
+
run.needsWallet ||
|
|
269
|
+
run.awaitingActive ||
|
|
270
|
+
run.report.steps.reconcile.status === "failed") &&
|
|
271
|
+
!run.report.rolledBack);
|
|
272
|
+
const funds = `(that returns ${itsFunded(live.totalFunded)})`;
|
|
273
|
+
const closePy = `\`python3 senpi-strategy-ops/scripts/close.py '${pkg.id}'\``;
|
|
274
|
+
if (otherLive.length === 0) {
|
|
275
|
+
return (`tear this one down first with ${closePy} ${funds}, then re-run — nothing else in this ` +
|
|
276
|
+
`package is live or queued for funding by this deploy, so that command closes exactly this ` +
|
|
277
|
+
`strategy.`);
|
|
278
|
+
}
|
|
279
|
+
const perStrategy = live.strategyWalletAddress
|
|
280
|
+
? `MCP \`strategy_close\` on wallet ${live.strategyWalletAddress}`
|
|
281
|
+
: `MCP \`strategy_close\` on its wallet (read the address with \`strategy_get\`, strategy_id ` +
|
|
282
|
+
`"${live.id}" — its address is unreadable here)`;
|
|
283
|
+
return (`tear THIS one down first with ${perStrategy} ${funds}, then re-run. Do NOT reach for ` +
|
|
284
|
+
`${closePy} to do it: it closes every live strategy in this package, and ${otherLive.length} ` +
|
|
285
|
+
`other sleeve(s) (${otherLive.map((run) => run.inst.name).join(", ")}) belong to this same ` +
|
|
286
|
+
`deploy — any of them live by the time you read this would be closed too.`);
|
|
287
|
+
}
|
|
288
|
+
function buildObserveFailedDetail(row, runtimeId) {
|
|
289
|
+
return (`scanner "${row.scannerId}" ran and failed: lastRunStatus=error errorCount=${row.errorCount} ` +
|
|
290
|
+
`lastError=${JSON.stringify(row.lastError)}. Read \`openclaw senpi scanner -r ${runtimeId}\`, fix the ` +
|
|
291
|
+
`scanner code — do NOT close/recreate the strategy.`);
|
|
292
|
+
}
|
|
293
|
+
function buildNext(overall, pkg, budget, runs) {
|
|
294
|
+
switch (overall) {
|
|
295
|
+
case "live":
|
|
296
|
+
return undefined;
|
|
297
|
+
case "refused":
|
|
298
|
+
return `Nothing was created past the refusal above. Resolve it, then re-run: \`${resumeCommand(pkg.dir, budget)}\``;
|
|
299
|
+
case "pending":
|
|
300
|
+
// Not "reconciliation adopts what exists; nothing is lost" — that promised an outcome this
|
|
301
|
+
// read cannot prove. Name-rejection retries land unnamed (the next name lookup misses them),
|
|
302
|
+
// and an empty `strategy_list` read defeats both match routes, so a re-run over that payload
|
|
303
|
+
// creates and funds a second wallet — executed, not theorised (384 `runDeploy`s over that
|
|
304
|
+
// matrix). The true, provable clause is the control-flow fact: reconcile re-reads the live
|
|
305
|
+
// list before it creates anything, and what it finds there decides the rest. This function
|
|
306
|
+
// renders one line below a per-instance detail that already carries the ordering (triage
|
|
307
|
+
// via `strategy_list` first) where that detail ran; this footer only names the command.
|
|
308
|
+
return `re-run \`${resumeCommand(pkg.dir, budget)}\` to resume — reconcile re-reads the live list before it creates anything, and what it finds there decides the rest`;
|
|
309
|
+
case "failed":
|
|
310
|
+
// "Fix the cause, then re-run" is false for exactly one failure on this verb: an install this
|
|
311
|
+
// deploy stopped WAITING on has no cause to fix — it was never cancelled, and its outcome is
|
|
312
|
+
// unknown — so the footer that outranks every step line (an agent reads the report's `next`
|
|
313
|
+
// as the instruction) would send a reader hunting for a fault that does not exist, and then
|
|
314
|
+
// re-running into a wallet whose fence may still be held. The step's own detail owns this
|
|
315
|
+
// case; the footer's whole job here is to hand the reader back to it.
|
|
316
|
+
return runs.some((run) => run.report.steps.install.evidence?.indeterminate === true)
|
|
317
|
+
? `Read the [${INSTALL_INDETERMINATE_CODE}] step above and follow it — there is nothing to ` +
|
|
318
|
+
`fix from this report: this deploy stopped waiting on an install it did not cancel, so ` +
|
|
319
|
+
`what that install left behind is what decides the next step. Re-run ` +
|
|
320
|
+
`\`${resumeCommand(pkg.dir, budget)}\` only once that read says it is safe to.`
|
|
321
|
+
: `Read the failed step above, fix the cause, then re-run: \`${resumeCommand(pkg.dir, budget)}\``;
|
|
322
|
+
case "installed-unobserved": {
|
|
323
|
+
const runtimeId = runs.find((run) => run.report.runtimeId)?.report.runtimeId;
|
|
324
|
+
// A `<runtimeId>` placeholder inside a backticked command is a command an agent pastes
|
|
325
|
+
// literally — and `<` is a shell redirect, so it fails in a way that says nothing about the
|
|
326
|
+
// deploy. With no id to name, name the command that needs none.
|
|
327
|
+
// Names BOTH things the observe step looks for, because it accepts either: a recorded run
|
|
328
|
+
// (scheduled) or a fresh liveness post (external). `no verified tick yet` named only the
|
|
329
|
+
// first, which reads as though a tick were the sole way out of this state — and it is the
|
|
330
|
+
// mirror of the `verified tick:` prefix the ok side no longer prints for an external row.
|
|
331
|
+
return runtimeId
|
|
332
|
+
? `no tick and no liveness post observed yet — check in a few minutes: \`openclaw senpi scanner -r ${runtimeId}\``
|
|
333
|
+
: `no tick and no liveness post observed yet — this report carries no runtime id: list them with ` +
|
|
334
|
+
`\`openclaw senpi runtime list\` and check this package's runtime in a few minutes`;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
function notReached() {
|
|
339
|
+
return { status: "pending", detail: "not reached" };
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Record that an instance is bound to `strategyId` — the WRITE half of the reconcile claim ledger
|
|
343
|
+
* (see `claimed` in {@link runDeploy}).
|
|
344
|
+
*
|
|
345
|
+
* An unreadable id is not claimable. `listStrategies` coerces a missing or unparseable id to `""`
|
|
346
|
+
* (senpi/client.ts), and a `""` in the ledger answers "already accounted for" for every OTHER
|
|
347
|
+
* unreadable-id wallet in the same read — which is exactly how a second live funded wallet became
|
|
348
|
+
* invisible to the gate. Not recording it is the fail-closed direction: the gate keeps presenting
|
|
349
|
+
* it and the run refuses rather than funding beside it.
|
|
350
|
+
*
|
|
351
|
+
* **Either this guard or {@link isAccountedFor}'s alone closes that hole; both are kept
|
|
352
|
+
* deliberately.** Delete this one and the suite still passes on the deploy paths — the read guard
|
|
353
|
+
* covers it. That is defence in depth, not dead code, and `orchestrator.test.ts` pins each half
|
|
354
|
+
* separately so neither can be removed as redundant.
|
|
355
|
+
*/
|
|
356
|
+
export function claimStrategyId(claimed, strategyId) {
|
|
357
|
+
if (strategyId)
|
|
358
|
+
claimed.add(strategyId);
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* Is this live strategy PROVEN to be accounted for by an instance of this run? — the READ half of
|
|
362
|
+
* the same ledger.
|
|
363
|
+
*
|
|
364
|
+
* Fail-closed on an unreadable id: it can never have been claimed and can never be matched, so it
|
|
365
|
+
* can never be proven accounted for. Resting that on `claimed.has("")` happening to answer false is
|
|
366
|
+
* not a property to give a money gate — it is only true while the write half holds. Same rule this
|
|
367
|
+
* file already applies to an unreadable `totalFunded` and an unreadable `skillName`: unknown is
|
|
368
|
+
* never zero, and unknown is never "not ours".
|
|
369
|
+
*
|
|
370
|
+
* **Either this guard or {@link claimStrategyId}'s alone closes the hole; both are kept
|
|
371
|
+
* deliberately** — see there.
|
|
372
|
+
*/
|
|
373
|
+
export function isAccountedFor(claimed, strategyId) {
|
|
374
|
+
return Boolean(strategyId) && claimed.has(strategyId);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Run one phase inside an injected span, surviving a tracer that throws.
|
|
378
|
+
*
|
|
379
|
+
* Same rule as narration's `emit` and `log` — telemetry may not decide what a deploy does — but
|
|
380
|
+
* with the largest blast radius of the three: the span wraps EVERY phase, including create and
|
|
381
|
+
* install, so a throwing tracer escaped `runDeploy` after a wallet had been funded and left the job
|
|
382
|
+
* layer synthesizing an empty `instances: []` report over a live funded wallet.
|
|
383
|
+
*
|
|
384
|
+
* It is deliberately NOT "on failure, run the body": **the body is only safe to run once.** A
|
|
385
|
+
* tracer that fails while ENDING a span has already run it, and re-running `deploy.create` is a
|
|
386
|
+
* second funded wallet — telemetry causing the exact loss this file is built to prevent. So the
|
|
387
|
+
* body is invoked through a one-shot handle and the two cases are told apart by whether it fired:
|
|
388
|
+
*
|
|
389
|
+
* - the tracer failed BEFORE reaching the body — run it here, un-spanned, and return its value;
|
|
390
|
+
* - the tracer failed AFTER the body ran — the body's own outcome is the truth, including its
|
|
391
|
+
* rejection if it had one. A tracer that masks a failing body with an error of its own therefore
|
|
392
|
+
* cannot hide the real cause: what surfaces is the body's error, not the tracer's.
|
|
393
|
+
*
|
|
394
|
+
* Module-scope and exported for ONE reason: the synchronous-throw edge below is unreachable through
|
|
395
|
+
* `runDeploy` (every call site passes an async arrow), so it can only be pinned here.
|
|
396
|
+
*
|
|
397
|
+
* @param spanFn the injected tracer — assumed to call `handle` at most once; calling it more often
|
|
398
|
+
* is harmless, since the handle collapses to the first invocation either way
|
|
399
|
+
* @param log narration for the degraded path; must not throw (callers pass the guarded wrapper)
|
|
400
|
+
* @returns whatever the body resolves with — never the tracer's own value on a failure
|
|
401
|
+
*/
|
|
402
|
+
export async function spanOnce(spanFn, log, name, attrs, body) {
|
|
403
|
+
let started;
|
|
404
|
+
/**
|
|
405
|
+
* `async () => body()` rather than `body()`, and it is load-bearing: a body that throws
|
|
406
|
+
* SYNCHRONOUSLY never completes the assignment, so `started` stays undefined, the catch reads
|
|
407
|
+
* that as "the tracer never reached it", and the body runs a SECOND time. On `deploy.create` that
|
|
408
|
+
* is the double-fund this helper exists to prevent, delivered by the helper itself. Wrapping
|
|
409
|
+
* turns every failure into a rejection, so "did it start" is decided by the call and never by how
|
|
410
|
+
* the body failed.
|
|
411
|
+
*/
|
|
412
|
+
const once = () => (started ??= (async () => body())());
|
|
413
|
+
try {
|
|
414
|
+
return await spanFn(name, attrs, once);
|
|
415
|
+
}
|
|
416
|
+
catch (err) {
|
|
417
|
+
if (started !== undefined)
|
|
418
|
+
return started;
|
|
419
|
+
log(`deploy: span "${name}" failed before the phase ran (${errorText(err)}) — running it un-spanned`);
|
|
420
|
+
return once();
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
/** Worst-first, so a phase's one-word summary is always its most serious outcome. */
|
|
424
|
+
const STATUS_SEVERITY = ["failed", "refused", "pending", "unobserved", "skipped", "ok"];
|
|
425
|
+
/** The phase's worst recorded step status — the `status` dimension of its span and metric. */
|
|
426
|
+
function phaseStatus(runs, step) {
|
|
427
|
+
const seen = runs.filter((run) => run.recorded.has(step)).map((run) => run.report.steps[step].status);
|
|
428
|
+
if (seen.length === 0)
|
|
429
|
+
return "not-reached";
|
|
430
|
+
return STATUS_SEVERITY.find((status) => seen.includes(status)) ?? "ok";
|
|
431
|
+
}
|
|
432
|
+
function newInstanceRun(pkg, inst) {
|
|
433
|
+
return {
|
|
434
|
+
inst,
|
|
435
|
+
strategyName: walletName(pkg, inst),
|
|
436
|
+
needsWallet: false,
|
|
437
|
+
recorded: new Set(),
|
|
438
|
+
report: {
|
|
439
|
+
instance: inst.name,
|
|
440
|
+
steps: {
|
|
441
|
+
reconcile: notReached(),
|
|
442
|
+
preflight: notReached(),
|
|
443
|
+
create: notReached(),
|
|
444
|
+
install: notReached(),
|
|
445
|
+
observe: notReached(),
|
|
446
|
+
},
|
|
447
|
+
},
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Sequential orchestration of one deploy. Resolves with a report in every case —
|
|
452
|
+
* refusals and failures are reported, never thrown, so the job layer always has
|
|
453
|
+
* a truthful terminal state to journal.
|
|
454
|
+
*/
|
|
455
|
+
export async function runDeploy(params, deps) {
|
|
456
|
+
const { pkg, budget, decisionModel } = params;
|
|
457
|
+
const maxWaitSec = params.maxWaitSec ?? DEFAULT_MAX_WAIT_SEC;
|
|
458
|
+
const tickWaitSec = params.tickWaitSec ?? DEFAULT_TICK_WAIT_SEC;
|
|
459
|
+
const runs = pkg.instances.map((inst) => newInstanceRun(pkg, inst));
|
|
460
|
+
let refused = false;
|
|
461
|
+
let abandoned = false;
|
|
462
|
+
let summariesOnce = null;
|
|
463
|
+
/**
|
|
464
|
+
* Strategy ids an instance of this package is BOUND to — adopted, or waited on for ACTIVE.
|
|
465
|
+
*
|
|
466
|
+
* The gate's question is "which stamped wallets did nothing account for" — a wallet a sibling
|
|
467
|
+
* sleeve just matched is accounted for, and presenting it to the next sleeve as unexplained
|
|
468
|
+
* refuses a legitimate partial deploy and points the reader at a teardown of the live sleeve.
|
|
469
|
+
*
|
|
470
|
+
* Membership is written by the binding branches of `applyCandidates` themselves, at the same
|
|
471
|
+
* statement that binds. That placement is the invariant, not a convention layered over one: a
|
|
472
|
+
* claim that does not bind is unrepresentable, so no editor has to remember the rule and no
|
|
473
|
+
* checker has to enforce it. The set was previously filled ahead of every branch, which made
|
|
474
|
+
* "answered for" and "bound to" different things and left the gate one forgetful branch away from
|
|
475
|
+
* hiding a live funded wallet and letting the deploy fund a second one beside it.
|
|
476
|
+
*
|
|
477
|
+
* What is left rests on two properties, in this order of importance:
|
|
478
|
+
*
|
|
479
|
+
* - **Forgetting to claim is SAFE.** A future branch that binds but does not claim under-claims,
|
|
480
|
+
* so the gate sees an accounted-for wallet as unaccounted and REFUSES. That is a false refusal,
|
|
481
|
+
* which is annoying; the opposite error funds a second wallet beside a live one. The residual
|
|
482
|
+
* failure mode points at the fail-closed side, which is the whole reason the claim moved.
|
|
483
|
+
* - It is COMPLETE before it is read. `boundaryAbandoned()` returns true on `halted()` before it
|
|
484
|
+
* consults `deps.shouldAbandon()`, and pass 2 opens with it — so pass 2 runs only when pass 1
|
|
485
|
+
* ran to the end over every instance. That, not iteration order, is why the gate's answer does
|
|
486
|
+
* not depend on the order the sleeves are declared in.
|
|
487
|
+
*
|
|
488
|
+
* The branches that answer for a candidate and REFUSE claim nothing, and do not need to: each
|
|
489
|
+
* sets `refused`, `halted()` makes pass 2 a no-op, and a refused sibling can never coexist with a
|
|
490
|
+
* gated peer in one job. Their claims were always moot — only unsafe to reason about.
|
|
491
|
+
*/
|
|
492
|
+
const claimed = new Set();
|
|
493
|
+
/**
|
|
494
|
+
* The SOFT tier's findings, filled in by the preflight and carried into the final report.
|
|
495
|
+
*
|
|
496
|
+
* Only FACTS are stored, never the finished note: the escape depends on what the deploy ends up
|
|
497
|
+
* having created, which is not known until the report is assembled.
|
|
498
|
+
*/
|
|
499
|
+
const soft = { min: null, under: [] };
|
|
500
|
+
/** Narration is fire-and-forget: a broken journal must never change what a deploy does. */
|
|
501
|
+
const emit = (event) => {
|
|
502
|
+
try {
|
|
503
|
+
deps.emit(event);
|
|
504
|
+
}
|
|
505
|
+
catch {
|
|
506
|
+
/* narration only */
|
|
507
|
+
}
|
|
508
|
+
};
|
|
509
|
+
/**
|
|
510
|
+
* The same rule as {@link emit}, for the same reason: `deps.log` is an injected collaborator, and
|
|
511
|
+
* it is called from the very catch blocks that exist to keep the report alive. A throwing logger
|
|
512
|
+
* took the original error with it, escaped `runDeploy`, and left the job layer synthesizing an
|
|
513
|
+
* empty `instances: []` report — narration deciding the outcome of a deploy that had already
|
|
514
|
+
* created and funded a wallet.
|
|
515
|
+
*/
|
|
516
|
+
const log = (line) => {
|
|
517
|
+
try {
|
|
518
|
+
deps.log(line);
|
|
519
|
+
}
|
|
520
|
+
catch {
|
|
521
|
+
/* narration only */
|
|
522
|
+
}
|
|
523
|
+
};
|
|
524
|
+
const emitPhase = (phase, detail) => emit({ type: "job", phase, detail });
|
|
525
|
+
const record = (run, step, outcome) => {
|
|
526
|
+
run.report.steps[step] = outcome;
|
|
527
|
+
run.recorded.add(step);
|
|
528
|
+
emit({
|
|
529
|
+
type: "step",
|
|
530
|
+
instance: run.inst.name,
|
|
531
|
+
step,
|
|
532
|
+
status: outcome.status,
|
|
533
|
+
detail: outcome.detail,
|
|
534
|
+
...(outcome.evidence ? { evidence: outcome.evidence } : {}),
|
|
535
|
+
});
|
|
536
|
+
};
|
|
537
|
+
const halted = () => refused || abandoned;
|
|
538
|
+
/** Marks every step not yet recorded as abandoned — for every instance, not just this one. */
|
|
539
|
+
const honorDeadline = () => {
|
|
540
|
+
abandoned = true;
|
|
541
|
+
for (const run of runs) {
|
|
542
|
+
for (const step of STEP_ORDER) {
|
|
543
|
+
if (!run.recorded.has(step)) {
|
|
544
|
+
record(run, step, { status: "skipped", detail: ABANDONED_DETAIL });
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
};
|
|
549
|
+
/**
|
|
550
|
+
* Run one step's body, converting a thrown dependency into a recorded `failed` outcome.
|
|
551
|
+
*
|
|
552
|
+
* `runDeploy` promises a report in EVERY case: a throw from any injected collaborator (an MCP
|
|
553
|
+
* timeout on the create poll, a registry read error) must not escape and leave the job layer
|
|
554
|
+
* synthesizing an empty `instances: []` report — that erases the wallet, the strategyId and every
|
|
555
|
+
* completed step from the only surface that reports them.
|
|
556
|
+
*/
|
|
557
|
+
const guarded = async (run, step, body) => {
|
|
558
|
+
try {
|
|
559
|
+
await body();
|
|
560
|
+
}
|
|
561
|
+
catch (err) {
|
|
562
|
+
const message = errorText(err);
|
|
563
|
+
// An install we stopped WAITING on is not an install that failed: `installRuntime` is
|
|
564
|
+
// in-process and takes no signal, so the work continues after this throw. Both the detail and
|
|
565
|
+
// the D-6 decision below branch on it.
|
|
566
|
+
const indeterminate = step === "install" && isInstallDeadlineExceeded(err);
|
|
567
|
+
/**
|
|
568
|
+
* Is that install STILL running on this wallet right now? The detail's instruction depends
|
|
569
|
+
* on it: while the fence is held, the install can still write its registry row and start a
|
|
570
|
+
* runtime, so nothing may be closed and nothing read now is final. `installRuntimeCore`
|
|
571
|
+
* holds the fence across its whole body — the wallet-ownership MCP call included, which
|
|
572
|
+
* takes no signal — so this is the only fact that separates "the install is over, this is
|
|
573
|
+
* what it left" from "it is mid-flight, do not touch it".
|
|
574
|
+
*
|
|
575
|
+
* `undefined` = unread, and the arm it selects is the conservative one. The read is guarded
|
|
576
|
+
* because it runs INSIDE the catch that exists to keep the report alive: a throwing
|
|
577
|
+
* collaborator here would escape `runDeploy` and erase the whole report, wallet included.
|
|
578
|
+
*/
|
|
579
|
+
let stillInFlight;
|
|
580
|
+
if (indeterminate && run.report.wallet) {
|
|
581
|
+
try {
|
|
582
|
+
stillInFlight = await deps.installPendingOnWallet(run.report.wallet);
|
|
583
|
+
}
|
|
584
|
+
catch (fenceErr) {
|
|
585
|
+
log(`deploy: could not read the install fence for wallet ${run.report.wallet}: ` +
|
|
586
|
+
`${errorText(fenceErr)} — reporting the install as possibly still in flight`);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
// Record FIRST, narrate second. `log` cannot throw and `record` narrates through `emit`,
|
|
590
|
+
// which cannot either — but the ordering is what the promise rests on, so it is written the
|
|
591
|
+
// way the promise reads: the report is made whole before anything is told about it.
|
|
592
|
+
if (!run.recorded.has(step)) {
|
|
593
|
+
record(run, step, indeterminate
|
|
594
|
+
? {
|
|
595
|
+
status: "failed",
|
|
596
|
+
detail: buildInstallIndeterminateDetail({
|
|
597
|
+
wallet: run.report.wallet,
|
|
598
|
+
funded: run.report.funded,
|
|
599
|
+
runtimeId: sanitizeStrategyName(`${pkg.id}-${run.inst.name}`),
|
|
600
|
+
deadlineMs: INSTALL_DEADLINE_MS,
|
|
601
|
+
createdByThisJob: run.needsWallet,
|
|
602
|
+
stillInFlight,
|
|
603
|
+
resume: resumeCommand(pkg.dir, budget),
|
|
604
|
+
caveat: buildStrandedCloseCaveat(pkg, runs, run),
|
|
605
|
+
}),
|
|
606
|
+
evidence: {
|
|
607
|
+
wallet: run.report.wallet,
|
|
608
|
+
funded: run.report.funded,
|
|
609
|
+
indeterminate: true,
|
|
610
|
+
// The fence read the detail forked on, kept beside it: the report's prose is a
|
|
611
|
+
// rendering, and a triage reader asking "was it still running?" should not have
|
|
612
|
+
// to parse a sentence to find out. `undefined` (unread) is omitted rather than
|
|
613
|
+
// rendered false — an unread fence is not a cleared one.
|
|
614
|
+
...(stillInFlight !== undefined ? { installPending: stillInFlight } : {}),
|
|
615
|
+
error: message,
|
|
616
|
+
},
|
|
617
|
+
}
|
|
618
|
+
: {
|
|
619
|
+
status: "failed",
|
|
620
|
+
detail: `${step} step failed: ${message}`,
|
|
621
|
+
evidence: { error: message },
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
log(`deploy: ${step} step threw for instance "${run.inst.name}": ${message}`);
|
|
625
|
+
// D-6 backstop. `installInstance` unwinds its RESOLVED failure branches itself, but any of
|
|
626
|
+
// its collaborators can throw — the registry reads, the dead-row/old-wallet deletes, the
|
|
627
|
+
// install call (ENOSPC, corrupt registry, module resolution, permissions) — and a throw
|
|
628
|
+
// lands here instead: same stranded shape (a wallet THIS job created and funded, with no
|
|
629
|
+
// runtime and nothing watching it), so same unwind. Guarded at this chokepoint rather than
|
|
630
|
+
// around each call so every throw site is covered, not only the ones wrapped by hand. The
|
|
631
|
+
// status check keeps the zombie rule: a throw after install was recorded ok/skipped must
|
|
632
|
+
// never close a wallet with a live runtime on it.
|
|
633
|
+
// The deadline brand is carved out: the install may still bind a runtime to this wallet, and
|
|
634
|
+
// an unwind that races it produces the one state worse than a stranded wallet — a live
|
|
635
|
+
// runtime on a closed strategy. The report names the wallet instead.
|
|
636
|
+
if (step === "install" &&
|
|
637
|
+
!indeterminate &&
|
|
638
|
+
run.needsWallet &&
|
|
639
|
+
!run.report.rolledBack &&
|
|
640
|
+
run.report.steps.install.status === "failed") {
|
|
641
|
+
await rollbackWallet(run);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
};
|
|
645
|
+
const span = (name, attrs, body) => spanOnce(deps.span, log, name, attrs, body);
|
|
646
|
+
/**
|
|
647
|
+
* The step boundary: the only place a pending deadline stop is honored — plus one deliberate
|
|
648
|
+
* mid-step boundary, ahead of the create phase's unnamed retry, which is the single money-moving
|
|
649
|
+
* call with no step boundary in front of it (see `createWallet`). Both go through here.
|
|
650
|
+
*/
|
|
651
|
+
const boundaryAbandoned = () => {
|
|
652
|
+
if (halted())
|
|
653
|
+
return true;
|
|
654
|
+
if (!deps.shouldAbandon())
|
|
655
|
+
return false;
|
|
656
|
+
honorDeadline();
|
|
657
|
+
return true;
|
|
658
|
+
};
|
|
659
|
+
/**
|
|
660
|
+
* L4: `strategy_list` runs unfiltered ONCE per job and is matched in memory (O(n) per instance).
|
|
661
|
+
*
|
|
662
|
+
* Backend behaviour verified 2026-08-05 against `senpi-hyperliquid-mcp`: the `strategy_list` tool
|
|
663
|
+
* exposes no pagination at all — its input schema accepts only filters plus `orderBy`
|
|
664
|
+
* (`src/tools/strategy.tools.ts`), the `FetchStrategies` GraphQL query selects no `pageInfo`/
|
|
665
|
+
* cursor fields (`src/clients/senpi.client.ts`), and its own resource doc says "No pagination;
|
|
666
|
+
* use filters to reduce response size". The full set returns in one call. (The `FetchStrategies`
|
|
667
|
+
* resolver itself lives in strategy-manager, outside that repo, so a SERVER-side cap cannot be
|
|
668
|
+
* ruled out from the client — but the tool offers no way to request a second page and no way to
|
|
669
|
+
* detect truncation, which is exactly why this must be re-verified if it ever changes.)
|
|
670
|
+
*
|
|
671
|
+
* Re-verify if that changes: a paginated backend would silently hide name matches and cause
|
|
672
|
+
* double-creates.
|
|
673
|
+
*/
|
|
674
|
+
const loadSummaries = async () => {
|
|
675
|
+
if (summariesOnce === null)
|
|
676
|
+
summariesOnce = await deps.listStrategies();
|
|
677
|
+
return summariesOnce;
|
|
678
|
+
};
|
|
679
|
+
/**
|
|
680
|
+
* The gate's own legibility: say so when the read carried strategies but NOT ONE readable
|
|
681
|
+
* attribution stamp.
|
|
682
|
+
*
|
|
683
|
+
* Either way it makes every candidate invisible, so the gate cannot fire and the deploy proceeds
|
|
684
|
+
* exactly as if the user owned nothing. Silent is the one thing that must not be: a stampless
|
|
685
|
+
* list of 12 strategies and a genuine "none of these are mine" are indistinguishable without it.
|
|
686
|
+
*
|
|
687
|
+
* It reports the observation and BOTH known causes rather than diagnosing one. A producer that
|
|
688
|
+
* stopped returning `strategyMetadata` is the same failure class as the missing `strategyName`
|
|
689
|
+
* this gate was built beside — but the spec's own risk list names a second: wallets created
|
|
690
|
+
* outside the sanctioned paths carry no stamp at all, and a user whose strategies are all of that
|
|
691
|
+
* kind would be told an MCP fault they do not have.
|
|
692
|
+
*/
|
|
693
|
+
const noteStamplessList = (summaries) => {
|
|
694
|
+
if (summaries.length === 0)
|
|
695
|
+
return;
|
|
696
|
+
if (summaries.some((s) => s.skillName !== undefined))
|
|
697
|
+
return;
|
|
698
|
+
log(`deploy: reconcile read ${summaries.length} live strategy(ies) and NONE carried a readable ` +
|
|
699
|
+
`attribution stamp (strategyMetadata.skillName) — the unaccounted-wallet gate can see no ` +
|
|
700
|
+
`candidate on this read and will not fire. Two things produce that, and this read cannot ` +
|
|
701
|
+
`tell them apart: a producer that stopped returning strategyMetadata, or strategies ` +
|
|
702
|
+
`created outside the sanctioned paths, which carry no stamp at all. Neither is proof that ` +
|
|
703
|
+
`none of these wallets belong to package "${pkg.id}".`);
|
|
704
|
+
};
|
|
705
|
+
// ── reconcile ─────────────────────────────────────────────────────────────
|
|
706
|
+
// Protection gate — before ANY backend read or write. A package that cannot stop its own
|
|
707
|
+
// positions out is never funded, on this path or the skills path.
|
|
708
|
+
const unprotected = instancesWithoutDslExit(pkg);
|
|
709
|
+
if (unprotected.length > 0) {
|
|
710
|
+
refused = true;
|
|
711
|
+
const detail = buildNoDslExitRefusal(unprotected, pkg);
|
|
712
|
+
const offenders = new Set(unprotected.map((inst) => inst.name));
|
|
713
|
+
for (const run of runs) {
|
|
714
|
+
if (offenders.has(run.inst.name))
|
|
715
|
+
record(run, "reconcile", { status: "refused", detail });
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
// Binding gate, beside the protection gate and for the same reason: a package that would fund one
|
|
719
|
+
// wallet and trade another must never reach a create. Package-derivable, so it is decided here,
|
|
720
|
+
// before the money exists to go to the wrong place. The render-time assertion in
|
|
721
|
+
// `renderRuntimeYaml` still runs — it re-reads the file and is the only check that sees the exact
|
|
722
|
+
// bytes being installed — but by then a wallet is funded and enforcing costs a rollback.
|
|
723
|
+
const unbound = unboundWalletInstances(pkg);
|
|
724
|
+
if (unbound.length > 0) {
|
|
725
|
+
refused = true;
|
|
726
|
+
const detail = buildUnboundWalletRefusal(unbound, pkg);
|
|
727
|
+
const offenders = new Set(unbound.map((fault) => fault.instance.name));
|
|
728
|
+
for (const run of runs) {
|
|
729
|
+
if (offenders.has(run.inst.name) && !run.recorded.has("reconcile")) {
|
|
730
|
+
record(run, "reconcile", { status: "refused", detail });
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
// D-6 premise repair: a refusal must mean "nothing was created". Both decision-model gates are
|
|
735
|
+
// package+params-derivable, so they belong HERE, before any money moves — recorded at the install
|
|
736
|
+
// step (their old home) they fired AFTER create+fund and stranded a funded wallet with no
|
|
737
|
+
// rollback and no [E_ROLLBACK_INCOMPLETE]. register.ts's `localStartRefusal` carries the same
|
|
738
|
+
// pair so the gateway never even starts a job; this copy keeps every other caller gated.
|
|
739
|
+
for (const run of runs) {
|
|
740
|
+
if (halted() || run.recorded.has("reconcile"))
|
|
741
|
+
continue;
|
|
742
|
+
if (!runtimeUsesLlmDecisions(readRuntimeYaml(run.inst)))
|
|
743
|
+
continue;
|
|
744
|
+
if (!decisionModel) {
|
|
745
|
+
refused = true;
|
|
746
|
+
record(run, "reconcile", {
|
|
747
|
+
status: "refused",
|
|
748
|
+
detail: buildDecisionModelRefusal(run.inst.name, pkg, budget),
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
else if (!pkg.decisionModelEnv) {
|
|
752
|
+
refused = true;
|
|
753
|
+
record(run, "reconcile", {
|
|
754
|
+
status: "refused",
|
|
755
|
+
detail: buildDecisionModelEnvRefusal(run.inst.name, pkg),
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
// D-2 (revised): an unsupported scanner-level `enabled` key. Package-derivable, so it is decided
|
|
760
|
+
// here rather than discovered after install — and the whole package's offenses go into the one
|
|
761
|
+
// refusal every affected instance carries, so one read fixes them all. Instances already refused
|
|
762
|
+
// above keep that refusal — an unprotected package is the more severe finding and leads.
|
|
763
|
+
const enabledKeys = scannersDeclaringEnabled(pkg);
|
|
764
|
+
if (enabledKeys.length > 0) {
|
|
765
|
+
refused = true;
|
|
766
|
+
const detail = buildScannerEnabledUnsupportedRefusal(enabledKeys, pkg);
|
|
767
|
+
const affected = new Set(enabledKeys.map((entry) => entry.instance));
|
|
768
|
+
for (const run of runs) {
|
|
769
|
+
if (affected.has(run.inst.name) && !run.recorded.has("reconcile")) {
|
|
770
|
+
record(run, "reconcile", { status: "refused", detail });
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
// The attribution stamp is minted from `pkg.id` VERBATIM (`createWallet`) while the backend
|
|
775
|
+
// stores what it is given case-normalized, so a mixed-case id is stamped under one spelling and
|
|
776
|
+
// read back under another — the false "no OPEN strategies to close." over a live funded wallet.
|
|
777
|
+
// Package-derivable, so it is decided here, before a wallet exists to carry the wrong stamp;
|
|
778
|
+
// register.ts's `localStartRefusal` carries the same gate so the gateway never even starts a job,
|
|
779
|
+
// and this copy keeps every other caller gated. Last of the local gates: the ones above are
|
|
780
|
+
// money-safety findings and lead when a package trips several.
|
|
781
|
+
if (nonLowercasePackageId(pkg) !== null) {
|
|
782
|
+
refused = true;
|
|
783
|
+
const detail = buildNonLowercaseIdRefusal(pkg, budget);
|
|
784
|
+
for (const run of runs) {
|
|
785
|
+
if (!run.recorded.has("reconcile"))
|
|
786
|
+
record(run, "reconcile", { status: "refused", detail });
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
// D-6 premise: a refusal must mean "nothing was created". These four conditions are
|
|
790
|
+
// package-derivable, so they belong here rather than at the install step — the same move the
|
|
791
|
+
// decision-model gates already made above (see the comment at the top of this block).
|
|
792
|
+
if (!halted()) {
|
|
793
|
+
await span("deploy.install_gate", { "senpi.deploy.step": "install_gate" }, async () => {
|
|
794
|
+
if (boundaryAbandoned())
|
|
795
|
+
return "skipped";
|
|
796
|
+
for (const run of runs) {
|
|
797
|
+
if (run.recorded.has("reconcile"))
|
|
798
|
+
continue;
|
|
799
|
+
let gate;
|
|
800
|
+
try {
|
|
801
|
+
gate = await gateInstance(run.inst);
|
|
802
|
+
}
|
|
803
|
+
catch (err) {
|
|
804
|
+
// Fail CLOSED, matching the universe check's `unavailable` branch: pre-money, a refusal
|
|
805
|
+
// costs nothing and a gate that cannot run has proven nothing. `installRuntimeCore` fails
|
|
806
|
+
// OPEN on the same fault, deliberately — refusing a good recipe AT the gateway blocks a
|
|
807
|
+
// working install, while refusing a deploy costs only a re-run. Neither side may be
|
|
808
|
+
// changed alone.
|
|
809
|
+
refused = true;
|
|
810
|
+
record(run, "reconcile", {
|
|
811
|
+
status: "failed",
|
|
812
|
+
detail: buildInstallGateUnavailableDetail(run.inst, errorText(err)),
|
|
813
|
+
evidence: { error: errorText(err) },
|
|
814
|
+
});
|
|
815
|
+
continue;
|
|
816
|
+
}
|
|
817
|
+
if (gate.ok)
|
|
818
|
+
continue;
|
|
819
|
+
refused = true;
|
|
820
|
+
record(run, "reconcile", {
|
|
821
|
+
status: "refused",
|
|
822
|
+
detail: buildInstallGateRefusal(gate, run.inst, pkg, budget),
|
|
823
|
+
evidence: { codes: gate.blocking.map((f) => f.code) },
|
|
824
|
+
});
|
|
825
|
+
}
|
|
826
|
+
return refused ? "refused" : "ok";
|
|
827
|
+
});
|
|
828
|
+
}
|
|
829
|
+
// The proof gate: deploy refuses a package no one has ever observed run. Pre-money like every
|
|
830
|
+
// gate above it, and BEFORE the universe gate deliberately — this is a local hash comparison over
|
|
831
|
+
// files already on disk, so it costs nothing and should not sit behind a network read.
|
|
832
|
+
//
|
|
833
|
+
// It re-validates nothing and runs no tick. The observe step already waits for the deployed
|
|
834
|
+
// runtime's tick; spending the deploy deadline on proving code here would report a slow
|
|
835
|
+
// validation as an unobserved deploy. Producing (or repairing) a proof is the author's loop, run
|
|
836
|
+
// through `senpi validate` — which is what every refusal below tells the reader to do.
|
|
837
|
+
if (!halted()) {
|
|
838
|
+
await span("deploy.proof", { "senpi.deploy.step": "proof" }, async () => {
|
|
839
|
+
if (boundaryAbandoned())
|
|
840
|
+
return "skipped";
|
|
841
|
+
// Before verifying: a package whose instances share a directory cannot BE verified, because
|
|
842
|
+
// one proof file would answer for two recipes and the unproven one would ride the other's
|
|
843
|
+
// pass into funding. Refused, not waved through — see `collidingInstanceDirs`.
|
|
844
|
+
const collisions = collidingInstanceDirs(pkg);
|
|
845
|
+
if (collisions.length > 0) {
|
|
846
|
+
refused = true;
|
|
847
|
+
const detail = buildInstanceDirCollisionRefusal(collisions, pkg);
|
|
848
|
+
const offenders = new Set(collisions.flatMap((c) => c.instances.map((inst) => inst.name)));
|
|
849
|
+
for (const run of runs) {
|
|
850
|
+
if (run.recorded.has("reconcile") || !offenders.has(run.inst.name))
|
|
851
|
+
continue;
|
|
852
|
+
record(run, "reconcile", {
|
|
853
|
+
status: "refused",
|
|
854
|
+
detail,
|
|
855
|
+
evidence: { shared_dirs: collisions.map((c) => c.dir) },
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
return "refused";
|
|
859
|
+
}
|
|
860
|
+
const verdict = checkPackageProof(pkg);
|
|
861
|
+
if (verdict.ok)
|
|
862
|
+
return "ok";
|
|
863
|
+
refused = true;
|
|
864
|
+
const detail = buildProofRefusal(verdict, pkg);
|
|
865
|
+
// The refusal is about the PACKAGE — `verifyProof` answers once for every instance it was
|
|
866
|
+
// given — so every instance not already refused carries it.
|
|
867
|
+
for (const run of runs) {
|
|
868
|
+
if (run.recorded.has("reconcile"))
|
|
869
|
+
continue;
|
|
870
|
+
record(run, "reconcile", {
|
|
871
|
+
status: "refused",
|
|
872
|
+
detail,
|
|
873
|
+
// A small machine-readable bag, like every sibling gate: the prose lives in `detail`.
|
|
874
|
+
// `code` comes from the same producer that rendered it, so the two cannot disagree.
|
|
875
|
+
evidence: {
|
|
876
|
+
reason: verdict.reason,
|
|
877
|
+
// `instance_dir`, NOT `instance`: the emitted event already carries a top-level
|
|
878
|
+
// `instance` holding the MANIFEST name (see `record`), and two different identifiers
|
|
879
|
+
// under one key name on one event is how a consumer joins the wrong two rows.
|
|
880
|
+
...(verdict.instance ? { instance_dir: verdict.instance } : {}),
|
|
881
|
+
...(verdict.changedFiles ? { changed_files: verdict.changedFiles } : {}),
|
|
882
|
+
code: proofRefusalParts(verdict, pkg).code,
|
|
883
|
+
},
|
|
884
|
+
});
|
|
885
|
+
}
|
|
886
|
+
return "refused";
|
|
887
|
+
});
|
|
888
|
+
}
|
|
889
|
+
// S-1 (user rulings 2026-08-07): the live-universe gate — the deploy-side consumer of the
|
|
890
|
+
// reusable `checkPackageUniverse` (universe.ts owns the decision; this block only records it).
|
|
891
|
+
// A dead hardcoded name does not error — the scan skips it and the strategy silently trades
|
|
892
|
+
// nothing — so it refuses PRE-MONEY, before reconcile even reads the backend. It needs one MCP
|
|
893
|
+
// read, which is why it lives here as a job-time gate (the E_FUNDS precedent) rather than in
|
|
894
|
+
// register.ts's synchronous localStartRefusal: every caller passes through runDeploy, so one
|
|
895
|
+
// site gates them all. It runs BEFORE reconcile deliberately (U-2): installing a blind runtime
|
|
896
|
+
// on an adopted wallet is the same zombie as funding a fresh one. A package hardcoding nothing
|
|
897
|
+
// never reads the list at all.
|
|
898
|
+
if (!halted()) {
|
|
899
|
+
await span("deploy.universe", { "senpi.deploy.step": "universe" }, async () => {
|
|
900
|
+
// The same step boundary every other MCP-touching phase honors: once the watchdog has asked
|
|
901
|
+
// for a stop, no new deadline-bounded read may be opened ahead of it.
|
|
902
|
+
if (boundaryAbandoned())
|
|
903
|
+
return "skipped";
|
|
904
|
+
const verdict = await checkPackageUniverse(pkg, deps);
|
|
905
|
+
// Narration is computed from the verdict, never a forward claim a tickerless package would
|
|
906
|
+
// falsify — so nothing is announced for a package with nothing to check. It is rendered in
|
|
907
|
+
// plain words: the union's tag names are internal, and a journal line is read by people.
|
|
908
|
+
const narration = describeUniverseVerdict(verdict);
|
|
909
|
+
if (narration)
|
|
910
|
+
emitPhase("reconcile", narration);
|
|
911
|
+
if (verdict.status === "no-hardcoded" || verdict.status === "clean")
|
|
912
|
+
return "ok";
|
|
913
|
+
if (verdict.status === "unavailable") {
|
|
914
|
+
// Fail-closed, exactly the thrown funds-preflight shape: the halt flag stops the run, the
|
|
915
|
+
// STEP status carries the truth, and no instrument is named dead on a read that failed.
|
|
916
|
+
refused = true;
|
|
917
|
+
log(`deploy: universe check failed: ${verdict.reason}`);
|
|
918
|
+
const detail = buildUniverseUnavailableDetail(verdict.reason, pkg);
|
|
919
|
+
for (const run of runs) {
|
|
920
|
+
if (!run.recorded.has("reconcile")) {
|
|
921
|
+
record(run, "reconcile", { status: "failed", detail, evidence: { error: verdict.reason } });
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
return "failed";
|
|
925
|
+
}
|
|
926
|
+
refused = true;
|
|
927
|
+
const detail = buildUniverseNotLiveRefusal(verdict.dead, pkg, verdict.liveCount, budget);
|
|
928
|
+
// The refusal lands on every instance whose own file carries a dead name; a strategy.yaml
|
|
929
|
+
// (catalog.assets) offense belongs to the whole package, so it lands on every instance.
|
|
930
|
+
const deadFiles = new Set([...verdict.dead.values()].flat().map((occ) => occ.file));
|
|
931
|
+
const packageWide = deadFiles.has(join(pkg.dir, "strategy.yaml"));
|
|
932
|
+
for (const run of runs) {
|
|
933
|
+
if (run.recorded.has("reconcile"))
|
|
934
|
+
continue;
|
|
935
|
+
if (packageWide || deadFiles.has(run.inst.runtimeYamlPath)) {
|
|
936
|
+
record(run, "reconcile", {
|
|
937
|
+
status: "refused",
|
|
938
|
+
detail,
|
|
939
|
+
evidence: { dead: [...verdict.dead.keys()], liveCount: verdict.liveCount },
|
|
940
|
+
});
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
return "refused";
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
if (!halted())
|
|
947
|
+
emitPhase("reconcile", `matching ${pkg.instances.length} instance(s) against live strategies`);
|
|
948
|
+
await span("deploy.reconcile", { "senpi.deploy.step": "reconcile" }, async () => {
|
|
949
|
+
/**
|
|
950
|
+
* TWO passes, because the gate below asks a whole-package question and cannot be answered from
|
|
951
|
+
* a half-reconciled package.
|
|
952
|
+
*
|
|
953
|
+
* Pass 1 resolves every instance's binding and records which strategies answered. Pass 2 then
|
|
954
|
+
* gates the instances nobody accounted for against the wallets nobody accounted for. Deciding
|
|
955
|
+
* both inside one loop made the answer depend on iteration order: a two-sleeve package with
|
|
956
|
+
* only sleeve 1 live (a crashed deploy, a sleeve added to a live package, a sleeve closed by
|
|
957
|
+
* hand) refused sleeve 2 over the wallet sleeve 1 had just adopted — and the only actionable
|
|
958
|
+
* route the refusal names tears down the whole package, i.e. the live funded sleeve beside it.
|
|
959
|
+
*/
|
|
960
|
+
const unresolved = [];
|
|
961
|
+
for (const run of runs) {
|
|
962
|
+
if (boundaryAbandoned())
|
|
963
|
+
break;
|
|
964
|
+
await guarded(run, "reconcile", () => resolveInstanceBinding(run, unresolved));
|
|
965
|
+
if (halted())
|
|
966
|
+
break;
|
|
967
|
+
}
|
|
968
|
+
if (unresolved.length > 0 && summariesOnce)
|
|
969
|
+
noteStamplessList(summariesOnce);
|
|
970
|
+
for (const run of unresolved) {
|
|
971
|
+
if (boundaryAbandoned())
|
|
972
|
+
break;
|
|
973
|
+
await guarded(run, "reconcile", () => gateOrCreate(run));
|
|
974
|
+
if (halted())
|
|
975
|
+
break;
|
|
976
|
+
}
|
|
977
|
+
return phaseStatus(runs, "reconcile");
|
|
978
|
+
});
|
|
979
|
+
/**
|
|
980
|
+
* Pass 1: bind this instance to a live strategy by name or by journaled create key, or defer it
|
|
981
|
+
* to the gate. A strategy an instance BINDS to is claimed (see {@link claimed}) — adopted or
|
|
982
|
+
* waited on for ACTIVE — so a sibling sleeve's gate does not read it as unexplained.
|
|
983
|
+
*/
|
|
984
|
+
async function resolveInstanceBinding(run, unresolved) {
|
|
985
|
+
const summaries = await loadSummaries();
|
|
986
|
+
// The backend LOWERCASES a name on create (observed: "WARPATH" in, "warpath" back), so a
|
|
987
|
+
// case-sensitive compare would miss exactly the strategies it stored for us and fund a second
|
|
988
|
+
// wallet beside them. An empty name on either side is not a match — two absences are not an
|
|
989
|
+
// identity, and an unnamed strategy would otherwise bind to the first instance that asked.
|
|
990
|
+
const wanted = run.strategyName.trim().toLowerCase();
|
|
991
|
+
const byName = wanted
|
|
992
|
+
? summaries.filter((s) => (s.strategyName ?? "").trim().toLowerCase() === wanted &&
|
|
993
|
+
!DEAD_STATUSES.includes(s.status.toUpperCase()))
|
|
994
|
+
: [];
|
|
995
|
+
if (byName.length > 0) {
|
|
996
|
+
/**
|
|
997
|
+
* A name is not an identity. `walletName` derives it from `<id>-<instance>` (or the bare id
|
|
998
|
+
* for a flat package), so two DIFFERENT packages can produce the same one — a multi-sleeve
|
|
999
|
+
* `alpha`/`one` and a flat package whose id IS `alpha-one` both land on "alpha-one", and both
|
|
1000
|
+
* pass `assertIdentityRoundTrips`. Matched on the name alone, this package's deploy ADOPTS
|
|
1001
|
+
* the other's funded wallet: install then puts a second runtime on it, and the positions and
|
|
1002
|
+
* the money belong to a package nobody asked about.
|
|
1003
|
+
*
|
|
1004
|
+
* The stamp is the only thing in this read that can say whose wallet it is, so a candidate is
|
|
1005
|
+
* FOREIGN only when it carries one and it names someone else. Absent is not foreign — and
|
|
1006
|
+
* that half is load-bearing, not leniency. Pass 2's doc records that its own stamp gate "does
|
|
1007
|
+
* NOT survive STAMP starvation: an MCP that stops returning `strategyMetadata`…". A strict
|
|
1008
|
+
* match here would inherit exactly that failure with the money pointing the other way: on a
|
|
1009
|
+
* starved read every re-run of a working package would stop recognising its own wallet and
|
|
1010
|
+
* fund a second one beside it, converting a benign producer fault into a duplicate-funding
|
|
1011
|
+
* bug. Wallets predating attribution, and wallets created by hand through the MCP, are
|
|
1012
|
+
* stampless for the same reason and are equally not evidence of foreign ownership.
|
|
1013
|
+
*
|
|
1014
|
+
* Case-folded on both sides, for the reason pass 2's filter states: the backend
|
|
1015
|
+
* case-normalizes what it stores and `sanitizeStrategyName` preserves capitals, so a
|
|
1016
|
+
* case-sensitive compare would read a package's OWN stamp as someone else's.
|
|
1017
|
+
*/
|
|
1018
|
+
const stamp = pkg.id.trim().toLowerCase();
|
|
1019
|
+
const claimantOf = (s) => (s.skillName ?? "").trim().toLowerCase();
|
|
1020
|
+
const ours = byName.filter((s) => claimantOf(s) === "" || claimantOf(s) === stamp);
|
|
1021
|
+
if (ours.length > 0) {
|
|
1022
|
+
/**
|
|
1023
|
+
* Dropping the foreign rows narrows the list `applyCandidates` COUNTS, and >1 there is the
|
|
1024
|
+
* `E_STATE_AMBIGUOUS_WALLETS` refusal. So the drop is allowed only where the stamp really
|
|
1025
|
+
* does say which is which: at least one survivor has to carry THIS package's stamp.
|
|
1026
|
+
*
|
|
1027
|
+
* A survivor that carries no stamp proves nothing about ownership — that is the whole point
|
|
1028
|
+
* of treating absent as not-foreign above — so a stampless row standing beside a proven
|
|
1029
|
+
* foreign one under the same name is not "ours, obviously": the foreign row is positive
|
|
1030
|
+
* evidence the name is contested, and a raw `strategy_create_custom_strategy` or a
|
|
1031
|
+
* pre-attribution create is exactly how a stampless wallet gets there. Collapsing that pair
|
|
1032
|
+
* to one adoption would install this package's runtime onto a funded wallet nothing proved
|
|
1033
|
+
* was its own. Keep the full list instead, so the count still reaches the ambiguity refusal
|
|
1034
|
+
* and the operator triages it read-only, pre-money.
|
|
1035
|
+
*
|
|
1036
|
+
* When nothing foreign matched at all, `ours` IS `byName` and this decides nothing — the
|
|
1037
|
+
* stamp-starvation path (every row stampless) is unaffected either way.
|
|
1038
|
+
*/
|
|
1039
|
+
const proven = ours.some((s) => claimantOf(s) === stamp);
|
|
1040
|
+
applyCandidates(run, proven ? ours : byName, "name",
|
|
1041
|
+
// Only the rows this call actually dropped: when the list is not narrowed, none were.
|
|
1042
|
+
proven ? byName.filter((s) => !ours.includes(s)) : []);
|
|
1043
|
+
return;
|
|
1044
|
+
}
|
|
1045
|
+
refused = true;
|
|
1046
|
+
record(run, "reconcile", {
|
|
1047
|
+
status: "refused",
|
|
1048
|
+
detail: buildForeignPackageRefusal(pkg, run.inst.name, run.strategyName, byName, budget),
|
|
1049
|
+
evidence: {
|
|
1050
|
+
instance: run.inst.name,
|
|
1051
|
+
strategyName: run.strategyName,
|
|
1052
|
+
// Same absence rules as pass 2's evidence: an id or address the backend did not report is
|
|
1053
|
+
// OMITTED, never handed to a `--json` reader as the empty string.
|
|
1054
|
+
candidates: byName.map((s) => ({
|
|
1055
|
+
...(s.id?.trim() ? { strategyId: s.id.trim() } : {}),
|
|
1056
|
+
...(s.strategyWalletAddress?.trim() ? { wallet: s.strategyWalletAddress.trim() } : {}),
|
|
1057
|
+
...(s.skillName?.trim() ? { skillName: s.skillName.trim() } : {}),
|
|
1058
|
+
status: s.status,
|
|
1059
|
+
totalFunded: s.totalFunded,
|
|
1060
|
+
})),
|
|
1061
|
+
},
|
|
1062
|
+
});
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
// C2: a create whose custom name the backend rejected lands under a backend-assigned name, so
|
|
1066
|
+
// the name match above can never find it again and a re-run would fund a SECOND wallet beside
|
|
1067
|
+
// it. The journal supplies the strategyId it recorded at create time; the BACKEND decides
|
|
1068
|
+
// whether that id still means anything — a dead key falls straight through to a fresh create.
|
|
1069
|
+
const priorIds = params.priorStrategyIds?.get(run.inst.name) ?? [];
|
|
1070
|
+
if (priorIds.length > 0) {
|
|
1071
|
+
const verified = (await deps.listStrategies({ strategyIds: priorIds })).filter((s) => !DEAD_STATUSES.includes(s.status.toUpperCase()));
|
|
1072
|
+
if (verified.length > 0) {
|
|
1073
|
+
applyCandidates(run, verified, "created-key");
|
|
1074
|
+
return;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
unresolved.push(run);
|
|
1078
|
+
}
|
|
1079
|
+
/**
|
|
1080
|
+
* Pass 2: no match route accounted for this instance. If wallets stamped with this package are
|
|
1081
|
+
* live and unclaimed, "I could not account for these" is NOT "there are none": creating here
|
|
1082
|
+
* funds a second wallet beside a live one, and the install step then moves the runtime onto the
|
|
1083
|
+
* new wallet, leaving the old one funded, in position and supervised by nothing. Refuse while no
|
|
1084
|
+
* money has moved.
|
|
1085
|
+
*
|
|
1086
|
+
* This holds when the NAME route is starved (an older MCP build, a backend that renames) —
|
|
1087
|
+
* that is the failure it was built for. It does NOT survive STAMP starvation: an MCP that stops
|
|
1088
|
+
* returning `strategyMetadata` empties this filter and the gate silently stops existing, which is
|
|
1089
|
+
* why a stampless non-empty list is logged rather than passed over in silence.
|
|
1090
|
+
*/
|
|
1091
|
+
async function gateOrCreate(run) {
|
|
1092
|
+
const summaries = await loadSummaries();
|
|
1093
|
+
// Case-folded on BOTH sides, like the name match: the backend case-normalizes what it stores,
|
|
1094
|
+
// and a case-sensitive compare here fails OPEN — a package id with a capital in it would find
|
|
1095
|
+
// no stamped wallet, and the gate that exists to stop a second funded wallet would wave it
|
|
1096
|
+
// through. `sanitizeStrategyName` deliberately preserves capitals, so this is reachable.
|
|
1097
|
+
const stamp = pkg.id.trim().toLowerCase();
|
|
1098
|
+
const stamped = summaries.filter((s) => (s.skillName ?? "").trim().toLowerCase() === stamp &&
|
|
1099
|
+
!DEAD_STATUSES.includes(s.status.toUpperCase()) &&
|
|
1100
|
+
!isAccountedFor(claimed, s.id));
|
|
1101
|
+
if (stamped.length > 0) {
|
|
1102
|
+
refused = true;
|
|
1103
|
+
record(run, "reconcile", {
|
|
1104
|
+
status: "refused",
|
|
1105
|
+
detail: buildBindingUnknownRefusal(pkg, run.inst.name, run.strategyName, stamped, budget),
|
|
1106
|
+
evidence: {
|
|
1107
|
+
instance: run.inst.name,
|
|
1108
|
+
candidates: stamped.map((s) => ({
|
|
1109
|
+
// The id gets the same treatment as the wallet below, and for a sharper reason: an
|
|
1110
|
+
// unreadable id is exactly why this candidate could not be proven accounted for, so
|
|
1111
|
+
// reporting it as `""` would hand a `--json` reader the empty string as an identifier.
|
|
1112
|
+
...(s.id?.trim() ? { strategyId: s.id.trim() } : {}),
|
|
1113
|
+
// Same rule as `totalFunded`: an address the backend did not report is ABSENT here, not
|
|
1114
|
+
// an empty string. The prose says "unreadable"; a consumer of the evidence must not read
|
|
1115
|
+
// a falsy-but-present `""` as an address, and the key is omitted rather than set to
|
|
1116
|
+
// undefined so `in`/`hasOwnProperty` agree with the JSON a `--json` reader sees.
|
|
1117
|
+
...(s.strategyWalletAddress?.trim() ? { wallet: s.strategyWalletAddress.trim() } : {}),
|
|
1118
|
+
status: s.status,
|
|
1119
|
+
totalFunded: s.totalFunded,
|
|
1120
|
+
createdAt: s.createdAt,
|
|
1121
|
+
})),
|
|
1122
|
+
},
|
|
1123
|
+
});
|
|
1124
|
+
return;
|
|
1125
|
+
}
|
|
1126
|
+
run.needsWallet = true;
|
|
1127
|
+
record(run, "reconcile", {
|
|
1128
|
+
status: "ok",
|
|
1129
|
+
detail: `no live strategy named "${run.strategyName}" — a wallet will be created`,
|
|
1130
|
+
});
|
|
1131
|
+
}
|
|
1132
|
+
/**
|
|
1133
|
+
* The single reconcile decision, shared by both ways of finding a candidate: >1 is ambiguous and
|
|
1134
|
+
* refuses; a single one branches on its live status (ACTIVE adopt / initializing wait / anything
|
|
1135
|
+
* else refuse). Keeping one copy is what stops the two match routes from drifting apart.
|
|
1136
|
+
*/
|
|
1137
|
+
function applyCandidates(run, candidates, matchedBy,
|
|
1138
|
+
/**
|
|
1139
|
+
* Live strategies that shared the matched NAME and were dropped as another package's. Reported
|
|
1140
|
+
* rather than silently discarded: the collision is a durable fact about the backend, and the
|
|
1141
|
+
* operator meets it again at the next `close.py`, rename or status read. It never changes what
|
|
1142
|
+
* this function decides — the decision is made from `candidates` alone.
|
|
1143
|
+
*/
|
|
1144
|
+
ignoredForeign = []) {
|
|
1145
|
+
const provenance = matchedByClause(matchedBy) + foreignCollisionClause(ignoredForeign);
|
|
1146
|
+
/**
|
|
1147
|
+
* Record that THIS instance is bound to `strategyId`, so a sibling sleeve's gate does not read
|
|
1148
|
+
* that wallet as one nothing accounted for.
|
|
1149
|
+
*
|
|
1150
|
+
* Called from the binding branches below ONLY, at the statement that binds — never ahead of the
|
|
1151
|
+
* branch that decides, which is what made a claim that binds nothing representable at all.
|
|
1152
|
+
* {@link claimStrategyId} owns what may be recorded.
|
|
1153
|
+
*/
|
|
1154
|
+
const claim = (strategyId) => claimStrategyId(claimed, strategyId);
|
|
1155
|
+
if (candidates.length > 1) {
|
|
1156
|
+
refused = true;
|
|
1157
|
+
record(run, "reconcile", {
|
|
1158
|
+
status: "refused",
|
|
1159
|
+
detail: buildAmbiguousRefusal(candidates.length, matchedBy === "name"
|
|
1160
|
+
? `named "${run.strategyName}"`
|
|
1161
|
+
: `journaled as created for instance "${run.inst.name}"`, run.inst.name),
|
|
1162
|
+
evidence: {
|
|
1163
|
+
strategyName: run.strategyName,
|
|
1164
|
+
matchedBy,
|
|
1165
|
+
// `null`, not `""`, for an id the backend did not report. The omit-the-key rule the other
|
|
1166
|
+
// evidence sites use cannot apply inside an array without changing its length — and the
|
|
1167
|
+
// length IS the ambiguity being reported, so it has to survive. `null` is the array's
|
|
1168
|
+
// form of absent, and unlike `""` a consumer cannot mistake it for an identifier.
|
|
1169
|
+
candidates: candidates.map((c) => c.id?.trim() || null),
|
|
1170
|
+
},
|
|
1171
|
+
});
|
|
1172
|
+
return;
|
|
1173
|
+
}
|
|
1174
|
+
const matched = candidates[0];
|
|
1175
|
+
const status = matched.status.toUpperCase();
|
|
1176
|
+
const wallet = matched.strategyWalletAddress?.trim() ?? "";
|
|
1177
|
+
const evidence = {
|
|
1178
|
+
// The rows the stamp partition dropped, on every record this bind produces. Same absence
|
|
1179
|
+
// rules as the candidate lists above: a field the backend did not report is OMITTED.
|
|
1180
|
+
...(ignoredForeign.length > 0
|
|
1181
|
+
? {
|
|
1182
|
+
alsoNamed: ignoredForeign.map((s) => ({
|
|
1183
|
+
...(s.id?.trim() ? { strategyId: s.id.trim() } : {}),
|
|
1184
|
+
...(s.strategyWalletAddress?.trim() ? { wallet: s.strategyWalletAddress.trim() } : {}),
|
|
1185
|
+
...(s.skillName?.trim() ? { skillName: s.skillName.trim() } : {}),
|
|
1186
|
+
status: s.status,
|
|
1187
|
+
})),
|
|
1188
|
+
}
|
|
1189
|
+
: {}),
|
|
1190
|
+
// Same rule as the gate's candidate rows: an unreadable id is ABSENT from the evidence rather
|
|
1191
|
+
// than reported as the empty string, so a `--json` reader never reads `""` as an identifier.
|
|
1192
|
+
...(matched.id?.trim() ? { strategyId: matched.id.trim() } : {}),
|
|
1193
|
+
strategyName: matched.strategyName,
|
|
1194
|
+
status: matched.status,
|
|
1195
|
+
strategyWalletAddress: matched.strategyWalletAddress,
|
|
1196
|
+
totalFunded: matched.totalFunded,
|
|
1197
|
+
};
|
|
1198
|
+
if (status === "ACTIVE" && !wallet) {
|
|
1199
|
+
// deploy.py's "unreadable" refusal (taxonomy: E_STATE_AMBIGUOUS_WALLETS covers a candidate
|
|
1200
|
+
// whose address cannot be read). Adopting a blank address would install onto `undefined`;
|
|
1201
|
+
// creating a second wallet would double-fund what may be a live strategy. Refuse both.
|
|
1202
|
+
refused = true;
|
|
1203
|
+
record(run, "reconcile", {
|
|
1204
|
+
status: "refused",
|
|
1205
|
+
detail: buildUnreadableWalletRefusal(matched.strategyName, run.inst.name, matched.id),
|
|
1206
|
+
evidence: { ...evidence, matchedBy },
|
|
1207
|
+
});
|
|
1208
|
+
return;
|
|
1209
|
+
}
|
|
1210
|
+
run.report.strategyId = matched.id;
|
|
1211
|
+
if (status !== "ACTIVE" && !INITIALIZING_STATUSES.includes(status)) {
|
|
1212
|
+
// PAUSED, mid-teardown, or unrecognised: waiting cannot help, so refuse with the real status
|
|
1213
|
+
// and a next step computed from it rather than burning --max-wait on "still funding".
|
|
1214
|
+
refused = true;
|
|
1215
|
+
record(run, "reconcile", {
|
|
1216
|
+
status: "refused",
|
|
1217
|
+
detail: buildNotDeployableRefusal(matched, run.inst.name, pkg, budget),
|
|
1218
|
+
evidence: { ...evidence, matchedBy },
|
|
1219
|
+
});
|
|
1220
|
+
return;
|
|
1221
|
+
}
|
|
1222
|
+
if (status !== "ACTIVE") {
|
|
1223
|
+
// Still initializing (CREATE_WALLET → FUND_WALLET → … → ACTIVE). Do NOT adopt the wallet and
|
|
1224
|
+
// do NOT skip to install: the create phase runs the same bounded ACTIVE poll a fresh wallet
|
|
1225
|
+
// gets, so a transient state reports `pending` (re-run to resume), never `failed`.
|
|
1226
|
+
run.awaitingActive = matched.id;
|
|
1227
|
+
claim(matched.id);
|
|
1228
|
+
record(run, "reconcile", {
|
|
1229
|
+
status: "ok",
|
|
1230
|
+
detail: `matched live strategy "${matched.strategyName}" (status ${matched.status})${provenance} — ` +
|
|
1231
|
+
`not ACTIVE yet, waiting for it rather than creating a second wallet`,
|
|
1232
|
+
evidence: { ...evidence, matchedBy },
|
|
1233
|
+
});
|
|
1234
|
+
record(run, "preflight", {
|
|
1235
|
+
status: "skipped",
|
|
1236
|
+
detail: "instance already has a strategy — no new wallet to fund",
|
|
1237
|
+
});
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1240
|
+
run.report.wallet = wallet;
|
|
1241
|
+
run.report.funded = matched.totalFunded;
|
|
1242
|
+
claim(matched.id);
|
|
1243
|
+
record(run, "reconcile", {
|
|
1244
|
+
status: "ok",
|
|
1245
|
+
detail: `matched live strategy "${matched.strategyName}" (status ${matched.status})${provenance}`,
|
|
1246
|
+
evidence: { ...evidence, matchedBy },
|
|
1247
|
+
});
|
|
1248
|
+
record(run, "preflight", { status: "skipped", detail: "instance already has a wallet" });
|
|
1249
|
+
record(run, "create", {
|
|
1250
|
+
status: "skipped",
|
|
1251
|
+
detail: buildAdoptedWalletDetail(pkg, run.inst, matched, wallet, budget, matchedBy),
|
|
1252
|
+
evidence: { ...evidence, matchedBy },
|
|
1253
|
+
});
|
|
1254
|
+
}
|
|
1255
|
+
// ── preflight ─────────────────────────────────────────────────────────────
|
|
1256
|
+
const needing = runs.filter((run) => run.needsWallet);
|
|
1257
|
+
if (!halted() && needing.length > 0) {
|
|
1258
|
+
emitPhase("preflight", `checking accessible balance for ${needing.length} new wallet(s)`);
|
|
1259
|
+
await span("deploy.preflight", { "senpi.deploy.step": "preflight" }, async () => {
|
|
1260
|
+
if (!boundaryAbandoned()) {
|
|
1261
|
+
try {
|
|
1262
|
+
await runPreflight();
|
|
1263
|
+
}
|
|
1264
|
+
catch (err) {
|
|
1265
|
+
// A thrown preflight is fail-closed: no balance was proven, so nothing is created.
|
|
1266
|
+
const message = errorText(err);
|
|
1267
|
+
refused = true;
|
|
1268
|
+
log(`deploy: preflight threw: ${message}`);
|
|
1269
|
+
for (const run of needing) {
|
|
1270
|
+
if (!run.recorded.has("preflight")) {
|
|
1271
|
+
record(run, "preflight", {
|
|
1272
|
+
status: "failed",
|
|
1273
|
+
detail: `preflight step failed: ${message}. Nothing was created.`,
|
|
1274
|
+
evidence: { error: message },
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
return phaseStatus(runs, "preflight");
|
|
1281
|
+
});
|
|
1282
|
+
}
|
|
1283
|
+
async function runPreflight() {
|
|
1284
|
+
if (budget === undefined) {
|
|
1285
|
+
refused = true;
|
|
1286
|
+
const detail = buildBudgetRequiredRefusal(needing.length, pkg);
|
|
1287
|
+
for (const run of needing)
|
|
1288
|
+
record(run, "preflight", { status: "refused", detail });
|
|
1289
|
+
return;
|
|
1290
|
+
}
|
|
1291
|
+
if (budget <= 0) {
|
|
1292
|
+
refused = true;
|
|
1293
|
+
const detail = buildBudgetNotPositiveRefusal(budget, needing.length, pkg);
|
|
1294
|
+
for (const run of needing)
|
|
1295
|
+
record(run, "preflight", { status: "refused", detail });
|
|
1296
|
+
return;
|
|
1297
|
+
}
|
|
1298
|
+
let available = null;
|
|
1299
|
+
try {
|
|
1300
|
+
available = accessibleUsd(await deps.getPortfolio());
|
|
1301
|
+
}
|
|
1302
|
+
catch (err) {
|
|
1303
|
+
log(`deploy: portfolio read failed (${errorText(err)}) — ` +
|
|
1304
|
+
`proceeding without a local balance check; the backend is the funding authority`);
|
|
1305
|
+
available = null;
|
|
1306
|
+
}
|
|
1307
|
+
// Before the plan is trusted: do the shares this run will fund allocate more than `--budget`
|
|
1308
|
+
// authorizes? `planFunding` compares its total against the accessible BALANCE and nothing else,
|
|
1309
|
+
// and `loadDeployPackage` deliberately does not validate `funding_share` (the skills-side
|
|
1310
|
+
// `_pkg.validate` does, and this verb is the direct path that never runs it) — so shares
|
|
1311
|
+
// summing above 1 fund past the authorization whenever the balance is large enough to cover it,
|
|
1312
|
+
// with no step, note or `overall` in the report saying so.
|
|
1313
|
+
//
|
|
1314
|
+
// Scoped to `needing`, the same set the plan is built from, and for the reason the two-pass
|
|
1315
|
+
// reconcile above exists: a package resuming with some sleeves already adopted is funding only
|
|
1316
|
+
// what is left, and judging it on the whole manifest would refuse a legitimate partial deploy
|
|
1317
|
+
// over shares this run never spends.
|
|
1318
|
+
const overrun = sharesOverAuthorization(needing.map((run) => ({ name: run.inst.name, fundingShare: run.inst.fundingShare })));
|
|
1319
|
+
if (overrun !== null) {
|
|
1320
|
+
refused = true;
|
|
1321
|
+
const detail = buildShareOverrunRefusal(overrun, pkg, budget);
|
|
1322
|
+
for (const run of needing) {
|
|
1323
|
+
record(run, "preflight", {
|
|
1324
|
+
status: "refused",
|
|
1325
|
+
detail,
|
|
1326
|
+
evidence: { shares_sum: overrun.sum, shares: overrun.shares },
|
|
1327
|
+
});
|
|
1328
|
+
}
|
|
1329
|
+
return;
|
|
1330
|
+
}
|
|
1331
|
+
const plan = planFunding(needing.map((run) => ({ name: run.inst.name, fundingShare: run.inst.fundingShare })), budget, available);
|
|
1332
|
+
if (plan.shortfall) {
|
|
1333
|
+
refused = true;
|
|
1334
|
+
const detail = underfundedNote(plan.shortfall, needing.map((run) => ({ fundingShare: run.inst.fundingShare })), pkg.dir);
|
|
1335
|
+
for (const run of needing) {
|
|
1336
|
+
record(run, "preflight", { status: "refused", detail, evidence: { ...plan.shortfall } });
|
|
1337
|
+
}
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
// The HARD floor has now passed. Everything above is the money gate — `planFunding` floors each
|
|
1341
|
+
// wallet at MIN_WALLET and `underfundedNote` halts. What follows is the SOFT tier, and the two
|
|
1342
|
+
// must never be confused: it is a local estimate over the package's own YAML, the backend does
|
|
1343
|
+
// not enforce it, and a user is entitled to size their own budget. It warns and proceeds.
|
|
1344
|
+
//
|
|
1345
|
+
// It is handed the PLAN, not the budget: the claim is about what each wallet actually receives.
|
|
1346
|
+
applyCalculatedMinimum(plan.amounts);
|
|
1347
|
+
for (const run of needing) {
|
|
1348
|
+
run.fundAmount = plan.amounts.get(run.inst.name);
|
|
1349
|
+
record(run, "preflight", {
|
|
1350
|
+
status: "ok",
|
|
1351
|
+
detail: available === null
|
|
1352
|
+
? `will fund ${usd(run.fundAmount ?? 0)} — accessible balance unreadable, proceeding (the backend is the funding authority)`
|
|
1353
|
+
: `will fund ${usd(run.fundAmount ?? 0)} of ${usd(available)} accessible`,
|
|
1354
|
+
evidence: { initialBudget: run.fundAmount, accessibleUsd: available },
|
|
1355
|
+
});
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
/**
|
|
1359
|
+
* The SOFT calculated-minimum tier. The package-level figure is computed over the WHOLE package
|
|
1360
|
+
* (every sleeve's share of a hypothetical fresh deploy) and rides the report as CONTEXT — but the
|
|
1361
|
+
* shortfall CLAIM is per-wallet, over the allocation `planFunding` just decided.
|
|
1362
|
+
*
|
|
1363
|
+
* That split is the whole point. Those two questions have different denominators on a
|
|
1364
|
+
* partially-adopted deploy: `planFunding` divides among the instances still NEEDING a wallet,
|
|
1365
|
+
* while the minimum divides among all of them. Comparing budget-to-package-minimum therefore
|
|
1366
|
+
* announced a shortfall on a re-run that was handing one $13.50 sleeve the entire $20 — a claim
|
|
1367
|
+
* about money that is not moving, which is the same rule that makes an adopted-ONLY deploy skip
|
|
1368
|
+
* this check entirely.
|
|
1369
|
+
*
|
|
1370
|
+
* Advisory by construction: it records facts, never a step status, and a throw is swallowed. A
|
|
1371
|
+
* warning that cannot be computed is a missing warning, never a failed deploy.
|
|
1372
|
+
*/
|
|
1373
|
+
function applyCalculatedMinimum(amounts) {
|
|
1374
|
+
// The guard covers the WHOLE body, not just `packageMinBudget`. It used to end before the note
|
|
1375
|
+
// was rendered, so a throw from `buildSoftBudgetNote`/`forwardTense` escaped into the preflight
|
|
1376
|
+
// catch and REFUSED the deploy — the one outcome this tier's own doc rules out. Fail-closed, so
|
|
1377
|
+
// never a money bug; a flat contradiction of the contract two lines above it, which is how the
|
|
1378
|
+
// next editor learns the wrong rule.
|
|
1379
|
+
try {
|
|
1380
|
+
const result = packageMinBudget(pkg);
|
|
1381
|
+
soft.min = result;
|
|
1382
|
+
const needsOf = new Map(result.breakdown.map((row) => [row.wallet, row.perWalletMin]));
|
|
1383
|
+
for (const [wallet, funded] of amounts) {
|
|
1384
|
+
const needs = needsOf.get(wallet);
|
|
1385
|
+
if (needs !== undefined && funded < needs)
|
|
1386
|
+
soft.under.push({ wallet, funded, needs });
|
|
1387
|
+
}
|
|
1388
|
+
// No escape in the narration: at preflight nothing has been created yet, so there is nothing
|
|
1389
|
+
// to close. The final report re-renders this note with the escape its own state supports. The
|
|
1390
|
+
// tense here is forward-looking — every short wallet is about to be funded and run — so both
|
|
1391
|
+
// per-row sets are full and the narration reads "funds …; will run DEGRADED".
|
|
1392
|
+
const note = buildSoftBudgetNote(pkg, result, soft.under, "", forwardTense(soft.under));
|
|
1393
|
+
if (note === null)
|
|
1394
|
+
return;
|
|
1395
|
+
log(`deploy: ${note}`);
|
|
1396
|
+
emit({
|
|
1397
|
+
type: "job",
|
|
1398
|
+
phase: "preflight",
|
|
1399
|
+
detail: note,
|
|
1400
|
+
evidence: {
|
|
1401
|
+
minBudget: result.minBudget,
|
|
1402
|
+
walletCount: result.walletCount,
|
|
1403
|
+
bindingWallet: result.bindingWallet,
|
|
1404
|
+
unresolvedWallets: result.unresolvedWallets,
|
|
1405
|
+
underfunded: soft.under,
|
|
1406
|
+
},
|
|
1407
|
+
});
|
|
1408
|
+
}
|
|
1409
|
+
catch (err) {
|
|
1410
|
+
log(`deploy: calculated-minimum check skipped (${errorText(err)}) — advisory only`);
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
// ── create ────────────────────────────────────────────────────────────────
|
|
1414
|
+
const createTargets = runs.filter((run) => run.needsWallet || run.awaitingActive);
|
|
1415
|
+
if (!halted() && createTargets.length > 0) {
|
|
1416
|
+
const fresh = createTargets.filter((run) => run.needsWallet).length;
|
|
1417
|
+
const waiting = createTargets.length - fresh;
|
|
1418
|
+
emitPhase("create", waiting === 0
|
|
1419
|
+
? `creating and funding ${fresh} wallet(s)`
|
|
1420
|
+
: `creating and funding ${fresh} wallet(s); waiting for ${waiting} existing strategy(ies) to become ACTIVE`);
|
|
1421
|
+
await span("deploy.create", { "senpi.deploy.step": "create" }, async () => {
|
|
1422
|
+
for (const run of createTargets) {
|
|
1423
|
+
if (boundaryAbandoned())
|
|
1424
|
+
break;
|
|
1425
|
+
await guarded(run, "create", () => createInstance(run));
|
|
1426
|
+
}
|
|
1427
|
+
return phaseStatus(runs, "create");
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
/**
|
|
1431
|
+
* A create call that never throws, so one code path records the outcome.
|
|
1432
|
+
*
|
|
1433
|
+
* `threw` is part of the result, not a detail of how it was produced: it is the difference
|
|
1434
|
+
* between "the backend answered and created nothing" and "nobody knows whether a wallet was
|
|
1435
|
+
* created and funded", and {@link isNameRejection} refuses to retry on the second. Without it a
|
|
1436
|
+
* transport failure is indistinguishable here from a refusal — which is exactly how the retry
|
|
1437
|
+
* guard came to fire only on the class it must not fire on.
|
|
1438
|
+
*/
|
|
1439
|
+
async function callCreate(params) {
|
|
1440
|
+
try {
|
|
1441
|
+
return await deps.createCustomStrategy(params);
|
|
1442
|
+
}
|
|
1443
|
+
catch (err) {
|
|
1444
|
+
// Residual window, stated precisely because it is not closed here: a create abandoned at its
|
|
1445
|
+
// per-call deadline is recorded WITHOUT a journaled create key — the key only exists in the
|
|
1446
|
+
// response that never arrived. The deadline bounds this job's wait, not the backend's work,
|
|
1447
|
+
// so the create may still succeed afterwards. A re-run then finds that wallet by NAME, which
|
|
1448
|
+
// is enough in every case but one: the custom name was also rejected, so the wallet landed
|
|
1449
|
+
// under a backend-assigned name AND has no journaled key. That wallet is invisible to both
|
|
1450
|
+
// reconcile paths and a re-run funds a second one beside it. Narrow (needs a >90s create and
|
|
1451
|
+
// a rejected name) and unchanged by the C2 carve-out, which could only ever journal a key it
|
|
1452
|
+
// received; closing it needs an idempotency key on the create tool, not a client-side fix.
|
|
1453
|
+
// Until then the failure detail below tells the operator exactly what to check.
|
|
1454
|
+
return { success: false, message: errorText(err), threw: true, aborted: isDeadlineAbort(err) };
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
async function createInstance(run) {
|
|
1458
|
+
const strategyId = run.awaitingActive ?? (await createWallet(run));
|
|
1459
|
+
if (!strategyId)
|
|
1460
|
+
return;
|
|
1461
|
+
run.report.strategyId = strategyId;
|
|
1462
|
+
await pollUntilActive(run, strategyId);
|
|
1463
|
+
}
|
|
1464
|
+
/** Create + fund a fresh wallet. Returns its strategyId, or undefined when the step already failed. */
|
|
1465
|
+
async function createWallet(run) {
|
|
1466
|
+
const initialBudget = run.fundAmount ?? 0;
|
|
1467
|
+
const attribution = { skillName: pkg.id, skillVersion: pkg.version };
|
|
1468
|
+
let result = await callCreate({ initialBudget, strategyName: run.strategyName, ...attribution });
|
|
1469
|
+
// Whether the abandoned create (if it comes to that) is the UNNAMED second attempt. Known here
|
|
1470
|
+
// and nowhere else — and it is the whole difference between "a re-run adopts it by name" and
|
|
1471
|
+
// "no route can ever find it", so the report must not have to hedge about which happened.
|
|
1472
|
+
let nameFallbackFired = false;
|
|
1473
|
+
if (!result.success && isNameRejection(result)) {
|
|
1474
|
+
// The ONE money-moving call in this file with no step boundary ahead of it — every other one
|
|
1475
|
+
// is preceded by `boundaryAbandoned()`. It needs its own, because `callCreate` mints a FRESH
|
|
1476
|
+
// `MCP_CALL_TIMEOUT_MS` deadline: a retry issued after the soft stop can still be awaiting a
|
|
1477
|
+
// funded create 90s later, while the grace (`GRACE_MS`, sized to outlast exactly one in-flight
|
|
1478
|
+
// call) has already force-terminated the job and handed the single-flight slot on. The next
|
|
1479
|
+
// deploy then runs its preflight against a balance this one is still spending — the overlap
|
|
1480
|
+
// the grace arithmetic exists to make impossible.
|
|
1481
|
+
//
|
|
1482
|
+
// Recorded BEFORE the sweep, so this instance's create keeps the detail that says which call
|
|
1483
|
+
// was issued and which was not; `honorDeadline` only fills steps nothing has recorded.
|
|
1484
|
+
if (deps.shouldAbandon()) {
|
|
1485
|
+
record(run, "create", {
|
|
1486
|
+
status: "failed",
|
|
1487
|
+
detail: buildCreateNameRetryNotIssuedDetail({
|
|
1488
|
+
instance: run.inst.name,
|
|
1489
|
+
strategyName: run.strategyName,
|
|
1490
|
+
fundAmount: initialBudget,
|
|
1491
|
+
message: result.message,
|
|
1492
|
+
pkg,
|
|
1493
|
+
budget,
|
|
1494
|
+
}),
|
|
1495
|
+
evidence: { success: false, message: result.message, retryIssued: false },
|
|
1496
|
+
});
|
|
1497
|
+
boundaryAbandoned();
|
|
1498
|
+
return undefined;
|
|
1499
|
+
}
|
|
1500
|
+
// deploy.py:447 parity — naming is best-effort legibility and must never block a deploy.
|
|
1501
|
+
// Narrated as a job event: it is not the create step's recorded outcome.
|
|
1502
|
+
emit({
|
|
1503
|
+
type: "job",
|
|
1504
|
+
phase: "create",
|
|
1505
|
+
detail: `name "${run.strategyName}" rejected (${result.message}); creating without a custom name`,
|
|
1506
|
+
});
|
|
1507
|
+
nameFallbackFired = true;
|
|
1508
|
+
result = await callCreate({ initialBudget, ...attribution });
|
|
1509
|
+
}
|
|
1510
|
+
if (!result.success || !result.strategyId) {
|
|
1511
|
+
record(run, "create", {
|
|
1512
|
+
status: "failed",
|
|
1513
|
+
detail: result.aborted
|
|
1514
|
+
? buildCreateAbandonedDetail({
|
|
1515
|
+
instance: run.inst.name,
|
|
1516
|
+
strategyName: run.strategyName,
|
|
1517
|
+
nameFallbackFired,
|
|
1518
|
+
fundAmount: run.fundAmount ?? 0,
|
|
1519
|
+
message: result.message,
|
|
1520
|
+
pkg,
|
|
1521
|
+
budget,
|
|
1522
|
+
})
|
|
1523
|
+
: `wallet creation failed: ${result.message ?? "backend returned no strategyId"}`,
|
|
1524
|
+
evidence: {
|
|
1525
|
+
success: result.success,
|
|
1526
|
+
message: result.message,
|
|
1527
|
+
strategyId: result.strategyId,
|
|
1528
|
+
...(result.aborted ? { outcome: "unknown" } : {}),
|
|
1529
|
+
},
|
|
1530
|
+
});
|
|
1531
|
+
return undefined;
|
|
1532
|
+
}
|
|
1533
|
+
// C2: journal the create key BEFORE the ACTIVE poll — a crash during that poll is exactly the
|
|
1534
|
+
// window in which the create step's own outcome never gets recorded, and (when the custom name
|
|
1535
|
+
// was rejected) the strategy is unfindable by name afterwards. This event is the only thing
|
|
1536
|
+
// that lets a re-run adopt it instead of funding a second wallet beside it.
|
|
1537
|
+
//
|
|
1538
|
+
// `requested` rides the same line for the same reason, one narration tier down: the ask exists
|
|
1539
|
+
// nowhere else once this process dies, and the run that later ADOPTS this strategy needs it to
|
|
1540
|
+
// notice that the backend landed less than was asked for. It is a lookup input to a WARN only
|
|
1541
|
+
// (see DeployParams.priorRequested) — never an orchestration decision, and never the funded
|
|
1542
|
+
// figure, which is always re-read live.
|
|
1543
|
+
emit({
|
|
1544
|
+
type: "job",
|
|
1545
|
+
phase: "create",
|
|
1546
|
+
detail: `wallet created for instance "${run.inst.name}" (strategyId ${result.strategyId})`,
|
|
1547
|
+
evidence: { instance: run.inst.name, strategyId: result.strategyId, requested: initialBudget },
|
|
1548
|
+
});
|
|
1549
|
+
return result.strategyId;
|
|
1550
|
+
}
|
|
1551
|
+
/** The bounded ACTIVE poll — shared by a freshly created wallet and an adopted initializing one. */
|
|
1552
|
+
async function pollUntilActive(run, strategyId) {
|
|
1553
|
+
const deadline = deps.now() + maxWaitSec * 1000;
|
|
1554
|
+
for (;;) {
|
|
1555
|
+
const [live] = await deps.listStrategies({ strategyIds: [strategyId] });
|
|
1556
|
+
if (live && live.status.toUpperCase() === "ACTIVE" && live.strategyWalletAddress) {
|
|
1557
|
+
run.report.wallet = live.strategyWalletAddress;
|
|
1558
|
+
run.report.funded = live.totalFunded;
|
|
1559
|
+
record(run, "create", {
|
|
1560
|
+
status: "ok",
|
|
1561
|
+
// An adopted instance was never funded by THIS run, so it carries the same
|
|
1562
|
+
// budget-not-applied note the ACTIVE-adopt path renders; a freshly created wallet was
|
|
1563
|
+
// funded to its planned share and needs none.
|
|
1564
|
+
detail: run.awaitingActive
|
|
1565
|
+
? buildAdoptedWalletDetail(pkg, run.inst, live, live.strategyWalletAddress, budget)
|
|
1566
|
+
: `wallet ${live.strategyWalletAddress} is ACTIVE, ${fundedClause(live.totalFunded)}`,
|
|
1567
|
+
evidence: {
|
|
1568
|
+
strategyId,
|
|
1569
|
+
strategyName: live.strategyName,
|
|
1570
|
+
status: live.status,
|
|
1571
|
+
strategyWalletAddress: live.strategyWalletAddress,
|
|
1572
|
+
totalFunded: live.totalFunded,
|
|
1573
|
+
},
|
|
1574
|
+
});
|
|
1575
|
+
return;
|
|
1576
|
+
}
|
|
1577
|
+
// Only INITIALIZING statuses resolve on their own, so only they are worth waiting on (the
|
|
1578
|
+
// same rule reconcile applies two screens up). Everything else that is not ACTIVE burns the
|
|
1579
|
+
// whole --max-wait and is then reported `pending` with "re-run; reconciliation waits for it
|
|
1580
|
+
// to reach ACTIVE" — a promise none of them can honour. Dead first: it is a subset with its
|
|
1581
|
+
// own, opposite re-run consequence.
|
|
1582
|
+
if (live && isDead(live.status)) {
|
|
1583
|
+
record(run, "create", {
|
|
1584
|
+
status: "failed",
|
|
1585
|
+
detail: buildCreateDeadDetail(pkg, budget, live, run.awaitingActive === undefined),
|
|
1586
|
+
evidence: {
|
|
1587
|
+
strategyId,
|
|
1588
|
+
status: live.status,
|
|
1589
|
+
strategyWalletAddress: live.strategyWalletAddress,
|
|
1590
|
+
totalFunded: live.totalFunded,
|
|
1591
|
+
},
|
|
1592
|
+
});
|
|
1593
|
+
return;
|
|
1594
|
+
}
|
|
1595
|
+
if (live && live.status.toUpperCase() !== "ACTIVE" && !isInitializing(live.status)) {
|
|
1596
|
+
record(run, "create", {
|
|
1597
|
+
status: "failed",
|
|
1598
|
+
detail: buildCreateNotDeployableDetail(pkg, budget, live, run.awaitingActive === undefined,
|
|
1599
|
+
// Scoped HERE, where the sibling runs are in scope — the detail builder is pure.
|
|
1600
|
+
buildPackageCloseOffer(pkg, runs, run, live)),
|
|
1601
|
+
evidence: {
|
|
1602
|
+
strategyId,
|
|
1603
|
+
status: live.status,
|
|
1604
|
+
strategyWalletAddress: live.strategyWalletAddress,
|
|
1605
|
+
totalFunded: live.totalFunded,
|
|
1606
|
+
},
|
|
1607
|
+
});
|
|
1608
|
+
return;
|
|
1609
|
+
}
|
|
1610
|
+
if (deps.now() >= deadline) {
|
|
1611
|
+
record(run, "create", {
|
|
1612
|
+
status: "pending",
|
|
1613
|
+
// `budget` verbatim, never a per-instance fallback: an adopted instance has no
|
|
1614
|
+
// fundAmount, and `--budget 0` is a command no one can usefully run.
|
|
1615
|
+
// `run.strategyName` is the name a re-run's reconcile will look this row up by, so the
|
|
1616
|
+
// detail can check whether that route is even open before promising what it does.
|
|
1617
|
+
detail: buildCreatePendingDetail(pkg, budget, maxWaitSec, live, deps.now(), run.strategyName),
|
|
1618
|
+
evidence: {
|
|
1619
|
+
strategyId,
|
|
1620
|
+
status: live?.status ?? null,
|
|
1621
|
+
createdAt: live?.createdAt ?? null,
|
|
1622
|
+
// The fact the sentence branches on, in the machine-readable half too: a `--json`
|
|
1623
|
+
// reader must not have to diff two prose names to learn the name route is closed.
|
|
1624
|
+
// `strategyName` follows this file's absence rule — a field the backend did not report
|
|
1625
|
+
// is OMITTED, never handed over as `""` — because the prose for that same row says
|
|
1626
|
+
// "under no readable name of its own", and the two readers must not reach different
|
|
1627
|
+
// conclusions about whether a name exists.
|
|
1628
|
+
...(live?.strategyName?.trim() ? { strategyName: live.strategyName } : {}),
|
|
1629
|
+
...(live ? { matchableByName: reconcileCanMatchByName(run.strategyName, live) } : {}),
|
|
1630
|
+
},
|
|
1631
|
+
});
|
|
1632
|
+
return;
|
|
1633
|
+
}
|
|
1634
|
+
await deps.sleep(CREATE_POLL_MS);
|
|
1635
|
+
if (deps.shouldAbandon()) {
|
|
1636
|
+
record(run, "create", {
|
|
1637
|
+
status: "pending",
|
|
1638
|
+
detail: `deploy deadline hit while waiting for wallet ${strategyId} to become ACTIVE — the wallet ` +
|
|
1639
|
+
`may still be funding. Nothing was rolled back; re-run \`${resumeCommand(pkg.dir, budget)}\` to resume`,
|
|
1640
|
+
evidence: { strategyId },
|
|
1641
|
+
});
|
|
1642
|
+
honorDeadline();
|
|
1643
|
+
return;
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
}
|
|
1647
|
+
// ── install ───────────────────────────────────────────────────────────────
|
|
1648
|
+
const installable = runs.filter((run) => run.report.wallet);
|
|
1649
|
+
if (!halted() && installable.length > 0) {
|
|
1650
|
+
emitPhase("install", `installing ${installable.length} runtime(s)`);
|
|
1651
|
+
await span("deploy.install", { "senpi.deploy.step": "install" }, async () => {
|
|
1652
|
+
for (const run of installable) {
|
|
1653
|
+
if (boundaryAbandoned())
|
|
1654
|
+
break;
|
|
1655
|
+
await guarded(run, "install", () => installInstance(run));
|
|
1656
|
+
}
|
|
1657
|
+
return phaseStatus(runs, "install");
|
|
1658
|
+
});
|
|
1659
|
+
}
|
|
1660
|
+
async function installInstance(run) {
|
|
1661
|
+
const wallet = run.report.wallet;
|
|
1662
|
+
const runtimeId = sanitizeStrategyName(`${pkg.id}-${run.inst.name}`);
|
|
1663
|
+
/** Prefixed onto the install detail when a dead registry row was cleared out of the way. */
|
|
1664
|
+
let replacedDeadRow = "";
|
|
1665
|
+
const onWallet = await deps.registryFindByWallet(wallet);
|
|
1666
|
+
if (onWallet) {
|
|
1667
|
+
// Asked before the live-handle question, because a row whose install is still running answers
|
|
1668
|
+
// that question "no" and would take the dead-row branch below — deleting the row the in-flight
|
|
1669
|
+
// install is about to bind, then installing a second runtime on the same margin account.
|
|
1670
|
+
// Never unwound: this wallet is being bound right now, and closing it would race that install.
|
|
1671
|
+
if (await deps.installPendingOnWallet(wallet)) {
|
|
1672
|
+
record(run, "install", {
|
|
1673
|
+
status: "failed",
|
|
1674
|
+
detail: buildInstallInFlightDetail({ wallet, runtimeId: onWallet.id }),
|
|
1675
|
+
evidence: { wallet, id: onWallet.id, installPending: true },
|
|
1676
|
+
});
|
|
1677
|
+
return;
|
|
1678
|
+
}
|
|
1679
|
+
if (await deps.runtimeIsLive(onWallet.id)) {
|
|
1680
|
+
run.installedAtMs = deps.now();
|
|
1681
|
+
run.report.runtimeId = onWallet.id;
|
|
1682
|
+
record(run, "install", {
|
|
1683
|
+
status: "skipped",
|
|
1684
|
+
detail: `runtime already registered AND running on this wallet (id "${onWallet.id}")`,
|
|
1685
|
+
evidence: { id: onWallet.id, wallet, live: true },
|
|
1686
|
+
});
|
|
1687
|
+
return;
|
|
1688
|
+
}
|
|
1689
|
+
// H4: a registry row with no live handle is a previous install that died before start —
|
|
1690
|
+
// reporting it `skipped` would be a clean line over a dead runtime, the DOA shape again.
|
|
1691
|
+
await deps.deleteRuntime(onWallet.id);
|
|
1692
|
+
emit({
|
|
1693
|
+
type: "job",
|
|
1694
|
+
phase: "install",
|
|
1695
|
+
detail: `registry entry "${onWallet.id}" had no live runtime — deleted; reinstalling on this wallet`,
|
|
1696
|
+
});
|
|
1697
|
+
replacedDeadRow = `registry entry "${onWallet.id}" had no live runtime — deleted and reinstalled. `;
|
|
1698
|
+
// fall through to the normal install path below
|
|
1699
|
+
}
|
|
1700
|
+
// The decision-model gates ran before reconcile — by here they are settled, so no refusal
|
|
1701
|
+
// can originate at the install step. Thrown failures from any call below are unwound by the
|
|
1702
|
+
// D-6 backstop in `guarded`.
|
|
1703
|
+
// The verb never reuses an old wallet: a same-id runtime bound elsewhere is deleted and
|
|
1704
|
+
// recreated on the fresh wallet rather than left pointing at stale money.
|
|
1705
|
+
let replaced = replacedDeadRow;
|
|
1706
|
+
const byId = await deps.registryFindById(runtimeId);
|
|
1707
|
+
if (byId && byId.wallet?.toLowerCase() !== wallet.toLowerCase()) {
|
|
1708
|
+
// Includes an entry whose wallet is unreadable: leaving it in place makes installRuntime
|
|
1709
|
+
// refuse with a bare "already exists" plus a `runtime delete` hint, instead of this verb's
|
|
1710
|
+
// own never-reuse-an-old-wallet outcome.
|
|
1711
|
+
await deps.deleteRuntime(runtimeId);
|
|
1712
|
+
replaced +=
|
|
1713
|
+
`existing runtime was on an old wallet (${byId.wallet ?? "address unreadable"}) — deleting ` +
|
|
1714
|
+
`and recreating on the fresh wallet; never reusing the old one. `;
|
|
1715
|
+
}
|
|
1716
|
+
let runtimeYamlContent;
|
|
1717
|
+
try {
|
|
1718
|
+
runtimeYamlContent = renderRuntimeYaml(run.inst, wallet, decisionModel && pkg.decisionModelEnv
|
|
1719
|
+
? { env: pkg.decisionModelEnv, model: decisionModel }
|
|
1720
|
+
: undefined);
|
|
1721
|
+
}
|
|
1722
|
+
catch (err) {
|
|
1723
|
+
record(run, "install", {
|
|
1724
|
+
status: "failed",
|
|
1725
|
+
detail: `${replaced}render failed: ${errorText(err)}`,
|
|
1726
|
+
});
|
|
1727
|
+
if (run.needsWallet)
|
|
1728
|
+
await rollbackWallet(run);
|
|
1729
|
+
return;
|
|
1730
|
+
}
|
|
1731
|
+
const result = await deps.installRuntime({
|
|
1732
|
+
runtimeYamlContent,
|
|
1733
|
+
runtimeYamlDir: run.inst.runtimeYamlDir,
|
|
1734
|
+
id: runtimeId,
|
|
1735
|
+
});
|
|
1736
|
+
if (!result.ok || !result.payload) {
|
|
1737
|
+
record(run, "install", {
|
|
1738
|
+
status: "failed",
|
|
1739
|
+
detail: `${replaced}install refused: ${result.error?.code ?? "UNKNOWN"} — ${result.error?.message ?? "no message"}`,
|
|
1740
|
+
evidence: { code: result.error?.code, message: result.error?.message },
|
|
1741
|
+
});
|
|
1742
|
+
if (run.needsWallet)
|
|
1743
|
+
await rollbackWallet(run);
|
|
1744
|
+
return;
|
|
1745
|
+
}
|
|
1746
|
+
const payload = result.payload;
|
|
1747
|
+
if (payload.unwired) {
|
|
1748
|
+
// The install succeeded and the runtime is UP — but with no live entry scanners, so it can
|
|
1749
|
+
// never produce a signal. Every one of the three phases is deterministic on this box, so
|
|
1750
|
+
// waiting on the observe step would only turn a certain failure into `installed-unobserved`
|
|
1751
|
+
// over a wallet this job just funded. It is an install failure, and it rolls back like one.
|
|
1752
|
+
run.report.runtimeId = payload.id;
|
|
1753
|
+
// The runtime is UP: `installRuntimeCore` set a live handle and wrote a registry row before
|
|
1754
|
+
// it reported the unwired phase. Closing the wallet under it (D-6, just below) would leave a
|
|
1755
|
+
// RUNNING runtime — DSL monitor, position reconcile, MCP polls — bound to a strategy this
|
|
1756
|
+
// same job closed. A re-run would clean it up, but the interim is exactly the zombie shape,
|
|
1757
|
+
// deliberately created on the money path. Delete it here, and delete it BEFORE the close so
|
|
1758
|
+
// nothing is ever live against a closing wallet. Unconditional: an install that failed must
|
|
1759
|
+
// leave nothing installed, adopted wallet or not.
|
|
1760
|
+
const removal = await removeBlindRuntime(payload.id);
|
|
1761
|
+
record(run, "install", {
|
|
1762
|
+
status: "failed",
|
|
1763
|
+
detail: `${replaced}${buildUnwiredInstallDetail({
|
|
1764
|
+
phase: payload.unwired,
|
|
1765
|
+
runtimeId: payload.id,
|
|
1766
|
+
wallet,
|
|
1767
|
+
createdByThisJob: run.needsWallet,
|
|
1768
|
+
removal,
|
|
1769
|
+
})}`,
|
|
1770
|
+
evidence: {
|
|
1771
|
+
...payload,
|
|
1772
|
+
runtimeDeleted: removal.ok,
|
|
1773
|
+
...(removal.ok ? {} : { deleteError: removal.error }),
|
|
1774
|
+
},
|
|
1775
|
+
});
|
|
1776
|
+
if (run.needsWallet)
|
|
1777
|
+
await rollbackWallet(run);
|
|
1778
|
+
return;
|
|
1779
|
+
}
|
|
1780
|
+
run.installedAtMs = deps.now();
|
|
1781
|
+
run.report.runtimeId = payload.id;
|
|
1782
|
+
record(run, "install", {
|
|
1783
|
+
status: "ok",
|
|
1784
|
+
detail: `${replaced}installed runtime "${payload.id}" on wallet ${payload.wallet}`,
|
|
1785
|
+
evidence: { ...payload },
|
|
1786
|
+
});
|
|
1787
|
+
}
|
|
1788
|
+
/**
|
|
1789
|
+
* Tear down a runtime whose install is being recorded `failed`. Never throws: a cleanup that
|
|
1790
|
+
* cannot run must not swallow the D-6 rollback that follows it — a stranded FUNDED wallet is the
|
|
1791
|
+
* worse outcome by far. The result is what the detail and the evidence are computed from, so the
|
|
1792
|
+
* report says what is actually left on the box rather than what was attempted.
|
|
1793
|
+
*/
|
|
1794
|
+
async function removeBlindRuntime(id) {
|
|
1795
|
+
try {
|
|
1796
|
+
await deps.deleteRuntime(id);
|
|
1797
|
+
return { ok: true };
|
|
1798
|
+
}
|
|
1799
|
+
catch (err) {
|
|
1800
|
+
return { ok: false, error: errorText(err) };
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
/**
|
|
1804
|
+
* D-6: unwind ONLY a wallet this job created and funded, ONLY when its install failed.
|
|
1805
|
+
* Never an adopted wallet (it predates this deploy), never on observe failure (the runtime is
|
|
1806
|
+
* installed and may open a position any moment), never on a refusal (nothing was created).
|
|
1807
|
+
* A close that cannot run or fails is E_ROLLBACK_INCOMPLETE — a stranded wallet is never silent.
|
|
1808
|
+
*
|
|
1809
|
+
* D-7: `strategy_close` returns the funds to the owner embedded wallet on its own, so the unwind
|
|
1810
|
+
* is one call — there is no separate withdraw step to get half-done.
|
|
1811
|
+
*/
|
|
1812
|
+
async function rollbackWallet(run) {
|
|
1813
|
+
await span("deploy.rollback", { "senpi.deploy.step": "rollback" }, () => rollbackWalletBody(run));
|
|
1814
|
+
}
|
|
1815
|
+
/** Resolves with the rollback's result — the wiring turns it into `senpi.deploy.rollback.count`. */
|
|
1816
|
+
async function rollbackWalletBody(run) {
|
|
1817
|
+
const wallet = run.report.wallet;
|
|
1818
|
+
const funded = rollbackFundedClause(run.report.funded);
|
|
1819
|
+
emit({
|
|
1820
|
+
type: "job",
|
|
1821
|
+
phase: "install",
|
|
1822
|
+
detail: `install failed on wallet ${wallet}, which this job created and ${funded} — ` +
|
|
1823
|
+
`closing it and returning the funds`,
|
|
1824
|
+
});
|
|
1825
|
+
const incomplete = (reason) => record(run, "create", {
|
|
1826
|
+
status: "failed",
|
|
1827
|
+
detail: `[E_ROLLBACK_INCOMPLETE] wallet ${wallet} was created by this deploy and ${funded}, ` +
|
|
1828
|
+
`its install failed, and the automatic close did not complete: ${reason}. The wallet is ` +
|
|
1829
|
+
`live and unwatched. Reclaim the funds from THIS wallet: MCP \`strategy_close\` on ` +
|
|
1830
|
+
`${wallet}${buildStrandedCloseCaveat(pkg, runs, run)}`,
|
|
1831
|
+
evidence: { wallet, funded: run.report.funded, reason },
|
|
1832
|
+
});
|
|
1833
|
+
try {
|
|
1834
|
+
if (!(await deps.walletIsFlat(wallet))) {
|
|
1835
|
+
incomplete("the wallet holds open positions — refusing to auto-close");
|
|
1836
|
+
return "incomplete";
|
|
1837
|
+
}
|
|
1838
|
+
const result = await deps.closeStrategy(wallet);
|
|
1839
|
+
// A resolved close proves nothing on its own: `strategy_close` reports a tool-level failure
|
|
1840
|
+
// by RESOLVING `{ success: false }`, never by throwing. Treating that as done zeroed `funded`
|
|
1841
|
+
// and told the operator the funds were back while the wallet was still live and funded.
|
|
1842
|
+
const closeFailure = describeCloseFailure(result);
|
|
1843
|
+
if (closeFailure) {
|
|
1844
|
+
incomplete(closeFailure);
|
|
1845
|
+
return "incomplete";
|
|
1846
|
+
}
|
|
1847
|
+
// The structured facts the report leads with must agree with the step detail: this wallet is
|
|
1848
|
+
// closed and the funds are back with the owner, so it is no longer a funded wallet. The
|
|
1849
|
+
// amount survives in the step evidence, where it reads as history rather than as a holding.
|
|
1850
|
+
const fundedBeforeRollback = run.report.funded;
|
|
1851
|
+
run.report.funded = 0;
|
|
1852
|
+
run.report.rolledBack = true;
|
|
1853
|
+
record(run, "create", {
|
|
1854
|
+
status: "ok",
|
|
1855
|
+
detail: `wallet ${wallet} was created by this deploy and its install failed — rolled back: strategy ` +
|
|
1856
|
+
`closed, funds returned to the owner wallet (close result: ${JSON.stringify(result)})`,
|
|
1857
|
+
evidence: { wallet, funded: fundedBeforeRollback, rolledBack: true, closeResult: result },
|
|
1858
|
+
});
|
|
1859
|
+
return "ok";
|
|
1860
|
+
}
|
|
1861
|
+
catch (err) {
|
|
1862
|
+
incomplete(errorText(err));
|
|
1863
|
+
return "incomplete";
|
|
1864
|
+
}
|
|
1865
|
+
}
|
|
1866
|
+
// ── observe ───────────────────────────────────────────────────────────────
|
|
1867
|
+
const observable = runs.filter((run) => run.report.runtimeId && ["ok", "skipped"].includes(run.report.steps.install.status));
|
|
1868
|
+
if (!halted() && observable.length > 0) {
|
|
1869
|
+
emitPhase("observe", tickWaitSec > 0
|
|
1870
|
+
? `waiting up to ${tickWaitSec}s for one scanner tick — or, for an external scanner, one fresh liveness post`
|
|
1871
|
+
: "tick observation skipped by --tick-wait 0");
|
|
1872
|
+
await span("deploy.observe", { "senpi.deploy.step": "observe" }, async () => {
|
|
1873
|
+
for (const run of observable) {
|
|
1874
|
+
if (boundaryAbandoned())
|
|
1875
|
+
break;
|
|
1876
|
+
await guarded(run, "observe", () => observeInstance(run));
|
|
1877
|
+
}
|
|
1878
|
+
return phaseStatus(runs, "observe");
|
|
1879
|
+
});
|
|
1880
|
+
}
|
|
1881
|
+
async function observeInstance(run) {
|
|
1882
|
+
const runtimeId = run.report.runtimeId;
|
|
1883
|
+
if (tickWaitSec <= 0) {
|
|
1884
|
+
// D-3 / M3: skipping observation must never read as verified — one argument away in a wrapper.
|
|
1885
|
+
record(run, "observe", {
|
|
1886
|
+
status: "unobserved",
|
|
1887
|
+
detail: `tick observation skipped (--tick-wait 0) — liveness NOT verified. ` +
|
|
1888
|
+
`Check it yourself: \`openclaw senpi scanner -r ${runtimeId}\``,
|
|
1889
|
+
});
|
|
1890
|
+
return;
|
|
1891
|
+
}
|
|
1892
|
+
const sinceMs = run.installedAtMs ?? deps.now();
|
|
1893
|
+
const deadline = deps.now() + tickWaitSec * 1000;
|
|
1894
|
+
let rowsLastSeen = [];
|
|
1895
|
+
/** Every distinct lastRunStatus seen across polls — the production data for tuning --tick-wait. */
|
|
1896
|
+
const statusesSeen = [];
|
|
1897
|
+
for (;;) {
|
|
1898
|
+
const rows = (await deps.getScannerRows(runtimeId)) ?? [];
|
|
1899
|
+
rowsLastSeen = rows;
|
|
1900
|
+
for (const row of rows) {
|
|
1901
|
+
if (row.lastRunStatus !== null && !statusesSeen.includes(row.lastRunStatus)) {
|
|
1902
|
+
statusesSeen.push(row.lastRunStatus);
|
|
1903
|
+
}
|
|
1904
|
+
}
|
|
1905
|
+
const active = rows.filter((row) => row.enabled);
|
|
1906
|
+
if (rows.length > 0 && active.length === 0) {
|
|
1907
|
+
record(run, "observe", {
|
|
1908
|
+
status: "unobserved",
|
|
1909
|
+
detail: `all ${rows.length} scanner(s) are disabled in the live engine — the runtime will ` +
|
|
1910
|
+
`never tick. Only the runtime API disables a scanner (a runtime.yaml \`enabled\` key is ` +
|
|
1911
|
+
`refused at deploy and never reaches the engine), so something disabled these after ` +
|
|
1912
|
+
`install. Check: \`openclaw senpi scanner -r ${runtimeId}\``,
|
|
1913
|
+
evidence: {
|
|
1914
|
+
scanners: rows.map((row) => ({ scannerId: row.scannerId, enabled: row.enabled })),
|
|
1915
|
+
statusesSeen,
|
|
1916
|
+
},
|
|
1917
|
+
});
|
|
1918
|
+
return;
|
|
1919
|
+
}
|
|
1920
|
+
const ticked = active.find((row) => isFreshTick(row, sinceMs));
|
|
1921
|
+
if (ticked) {
|
|
1922
|
+
// Two different things are proven here and the line has to name which one it got.
|
|
1923
|
+
// `isFreshTick` accepts liveness for an external scanner on purpose (see it for why), so
|
|
1924
|
+
// `verified tick:` in front of an external row was a claim its own evidence contradicted:
|
|
1925
|
+
// the production case is `runCount=0 lastRunStatus=null statusesSeen=[]`, because the
|
|
1926
|
+
// scanner's interval is longer than the observe window. Both are legitimate grounds to
|
|
1927
|
+
// call the deploy live; only one of them is a tick.
|
|
1928
|
+
//
|
|
1929
|
+
// Branch on the ROW, never on `scheduleMode`. `isFreshTick`'s external branch short-circuits
|
|
1930
|
+
// on `lastAliveAt` before it reads the run fields, so an external scanner that HAS recorded
|
|
1931
|
+
// a fresh run arrives here too — and a mode-keyed wording printed "not that a run was
|
|
1932
|
+
// recorded" beside `runCount=3`, which is the same verdict-versus-evidence contradiction
|
|
1933
|
+
// inverted. `recordedFreshRun` asks the row what it proves; only when the answer is no does
|
|
1934
|
+
// the line fall back to what liveness alone establishes.
|
|
1935
|
+
const proven = recordedFreshRun(ticked, sinceMs)
|
|
1936
|
+
? `verified tick: ${ticked.scannerId} lastRunStatus=${ticked.lastRunStatus}`
|
|
1937
|
+
: `verified alive: ${ticked.scannerId} is posting (external, lastAliveAt fresh) — this ` +
|
|
1938
|
+
`proves the scanner is running, not that a run was recorded;`;
|
|
1939
|
+
record(run, "observe", {
|
|
1940
|
+
status: "ok",
|
|
1941
|
+
detail: `${proven} runCount=${ticked.runCount} ` +
|
|
1942
|
+
`signalsProduced=${ticked.signalsProduced} health=${ticked.health}`,
|
|
1943
|
+
evidence: {
|
|
1944
|
+
scannerId: ticked.scannerId,
|
|
1945
|
+
scheduleMode: ticked.scheduleMode,
|
|
1946
|
+
lastRunStatus: ticked.lastRunStatus,
|
|
1947
|
+
lastRunFinishedAt: ticked.lastRunFinishedAt,
|
|
1948
|
+
lastAliveAt: ticked.lastAliveAt,
|
|
1949
|
+
runCount: ticked.runCount,
|
|
1950
|
+
signalsProduced: ticked.signalsProduced,
|
|
1951
|
+
health: ticked.health,
|
|
1952
|
+
statusesSeen,
|
|
1953
|
+
},
|
|
1954
|
+
});
|
|
1955
|
+
return;
|
|
1956
|
+
}
|
|
1957
|
+
const failedRow = active.find((row) => isFreshError(row, sinceMs));
|
|
1958
|
+
if (failedRow) {
|
|
1959
|
+
// D-1: an error tick is a terminal answer — polling the rest of the budget adds nothing.
|
|
1960
|
+
record(run, "observe", {
|
|
1961
|
+
status: "failed",
|
|
1962
|
+
detail: buildObserveFailedDetail(failedRow, runtimeId),
|
|
1963
|
+
evidence: {
|
|
1964
|
+
scannerId: failedRow.scannerId,
|
|
1965
|
+
lastRunStatus: failedRow.lastRunStatus,
|
|
1966
|
+
errorCount: failedRow.errorCount,
|
|
1967
|
+
lastError: failedRow.lastError,
|
|
1968
|
+
health: failedRow.health,
|
|
1969
|
+
statusesSeen,
|
|
1970
|
+
},
|
|
1971
|
+
});
|
|
1972
|
+
return;
|
|
1973
|
+
}
|
|
1974
|
+
if (deps.now() >= deadline)
|
|
1975
|
+
break;
|
|
1976
|
+
await deps.sleep(OBSERVE_POLL_MS);
|
|
1977
|
+
if (deps.shouldAbandon()) {
|
|
1978
|
+
record(run, "observe", {
|
|
1979
|
+
status: "unobserved",
|
|
1980
|
+
detail: `deploy deadline hit before a tick was observed — the runtime is installed and may be ` +
|
|
1981
|
+
`ticking. Check it yourself: \`openclaw senpi scanner -r ${runtimeId}\``,
|
|
1982
|
+
});
|
|
1983
|
+
honorDeadline();
|
|
1984
|
+
return;
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
// H2: a scanner on a long interval is not broken, it is early. When the registry says the first
|
|
1988
|
+
// tick is still ahead of the budget we just burned, say when — a bare "no tick observed" reads
|
|
1989
|
+
// as a fault and invites a close/recreate.
|
|
1990
|
+
const scheduled = rowsLastSeen.find((row) => row.enabled && row.nextRunAt !== null && row.nextRunAt > deadline);
|
|
1991
|
+
record(run, "observe", {
|
|
1992
|
+
status: "unobserved",
|
|
1993
|
+
detail: scheduled
|
|
1994
|
+
? `no tick observed within ${tickWaitSec}s — scanner "${scheduled.scannerId}" is scheduled, ` +
|
|
1995
|
+
`first tick due ${new Date(scheduled.nextRunAt).toISOString()}. ` +
|
|
1996
|
+
`Check then: \`openclaw senpi scanner -r ${runtimeId}\``
|
|
1997
|
+
: buildUnobservedDetail(tickWaitSec, runtimeId),
|
|
1998
|
+
evidence: { statusesSeen },
|
|
1999
|
+
});
|
|
2000
|
+
}
|
|
2001
|
+
// ── overall ───────────────────────────────────────────────────────────────
|
|
2002
|
+
const overall = decideOverall(runs, refused, abandoned);
|
|
2003
|
+
// The soft-tier fields ride the FINAL report, not a step detail, so the warn is still on the page
|
|
2004
|
+
// for a `live` / `installed-unobserved` deploy — where the capped event tail has long since
|
|
2005
|
+
// scrolled past the preflight narration that first raised it.
|
|
2006
|
+
// The escape's precondition is the RUNTIME, not the wallet: `close.py --instance` resolves a
|
|
2007
|
+
// sleeve through its runtime. The note's tense reads the same terminal facts, but PER ROW —
|
|
2008
|
+
// "funds" follows each wallet, "will run" follows each runtime — so a mixed report cannot claim
|
|
2009
|
+
// one state about sleeves its own escape describes differently.
|
|
2010
|
+
const byName = new Map(runs.map((run) => [run.inst.name, run]));
|
|
2011
|
+
const hasRuntime = (name) => {
|
|
2012
|
+
const run = byName.get(name);
|
|
2013
|
+
return Boolean(run?.report.runtimeId) && ["ok", "skipped"].includes(run?.report.steps.install.status ?? "");
|
|
2014
|
+
};
|
|
2015
|
+
const hasLiveWallet = (name) => {
|
|
2016
|
+
const run = byName.get(name);
|
|
2017
|
+
return Boolean(run?.report.wallet) && !run?.report.rolledBack;
|
|
2018
|
+
};
|
|
2019
|
+
const isStranded = (name) => hasLiveWallet(name) && !hasRuntime(name);
|
|
2020
|
+
const isZeroShare = (name) => pkg.instances.find((inst) => inst.name === name)?.fundingShare === 0;
|
|
2021
|
+
const escapeState = {
|
|
2022
|
+
closable: soft.under.map((row) => row.wallet).filter((name) => hasRuntime(name) && !isZeroShare(name)),
|
|
2023
|
+
strandedNames: soft.under.map((row) => row.wallet).filter(isStranded),
|
|
2024
|
+
walletless: walletlessAfter(pkg, runs),
|
|
2025
|
+
zeroShare: soft.under.map((row) => row.wallet).filter(isZeroShare),
|
|
2026
|
+
rollbackIncomplete: runs.some((run) => run.report.steps.create.status === "failed" &&
|
|
2027
|
+
run.report.steps.create.detail.startsWith("[E_ROLLBACK_INCOMPLETE]")),
|
|
2028
|
+
};
|
|
2029
|
+
const underNames = soft.under.map((row) => row.wallet);
|
|
2030
|
+
const softNote = soft.min
|
|
2031
|
+
? buildSoftBudgetNote(pkg, soft.min, soft.under, buildBudgetEscape(pkg, soft.min, escapeState), {
|
|
2032
|
+
funded: new Set(underNames.filter(hasLiveWallet)),
|
|
2033
|
+
running: new Set(underNames.filter(hasRuntime)),
|
|
2034
|
+
})
|
|
2035
|
+
: null;
|
|
2036
|
+
const softTier = {
|
|
2037
|
+
...(soft.min
|
|
2038
|
+
? {
|
|
2039
|
+
minBudget: soft.min.minBudget,
|
|
2040
|
+
minWalletCount: soft.min.walletCount,
|
|
2041
|
+
...(soft.min.unresolvedWallets.length > 0
|
|
2042
|
+
? { minBudgetUnresolved: soft.min.unresolvedWallets }
|
|
2043
|
+
: {}),
|
|
2044
|
+
}
|
|
2045
|
+
: {}),
|
|
2046
|
+
...(soft.under.length > 0 ? { belowMin: true } : {}),
|
|
2047
|
+
...(softNote ? { minBudgetNote: softNote } : {}),
|
|
2048
|
+
};
|
|
2049
|
+
/**
|
|
2050
|
+
* The OUTCOME check the plan-side W_BUDGET_* tiers cannot make: what the backend actually landed,
|
|
2051
|
+
* against what this job asked it for. Computed HERE, in the single place that has both halves —
|
|
2052
|
+
* the ask (the amount `planFunding` decided and the create call carried) and `report.funded`
|
|
2053
|
+
* (quoted from the post-create `strategy_list` read, the same figure the create step prints).
|
|
2054
|
+
*
|
|
2055
|
+
* The ask comes from this run's own `fundAmount` when this run funded, and otherwise from
|
|
2056
|
+
* {@link DeployParams.priorRequested} — the amount a prior run of this package journaled for the
|
|
2057
|
+
* very strategyId this run just ADOPTED. Without that fallback a crash between a partial fund and
|
|
2058
|
+
* report assembly resumed as an adoption with no ask, and the shortfall vanished from the report.
|
|
2059
|
+
* The journal supplies the ask ONLY; `funded` is always the live read from this run.
|
|
2060
|
+
*
|
|
2061
|
+
* Every guard below exists so the pair can only ever be a real, comparable one:
|
|
2062
|
+
* - the ask — a wallet nobody ever planned to fund has none, and inventing one would announce a
|
|
2063
|
+
* shortfall against money that was never requested; the create step's own "the requested …
|
|
2064
|
+
* was NOT added" line already owns that case. A non-positive ask is not a comparison either.
|
|
2065
|
+
* - `report.wallet` — set on, and only on, the ACTIVE strategy read. Absent means the read never
|
|
2066
|
+
* happened (create pending, failed, abandoned), so there is nothing to say about a wallet
|
|
2067
|
+
* nobody looked at. Silence is the fail-closed answer.
|
|
2068
|
+
* - a rollback that was ATTEMPTED, proven (`rolledBack`) or not. A proven close zeroes `funded`,
|
|
2069
|
+
* so the pair would compare $0 against the ask on a wallet that no longer exists. An
|
|
2070
|
+
* INCOMPLETE one is the sharper case: `funded` keeps its partial value, and the create step is
|
|
2071
|
+
* already carrying `[E_ROLLBACK_INCOMPLETE]` — "reclaim the funds from THIS wallet". A
|
|
2072
|
+
* top-the-wallet-up warn beside it makes one report the author of two opposite money moves on
|
|
2073
|
+
* one wallet, and an agent reconciling them runs whichever executes. When the rollback text is
|
|
2074
|
+
* present it is the single remediation authority, exactly as the funding-round doctrine says.
|
|
2075
|
+
*
|
|
2076
|
+
* `report.funded` absent WITH a wallet present is the third outcome, not silence: the read
|
|
2077
|
+
* happened and the backend reported no readable figure. That is
|
|
2078
|
+
* {@link buildFundedUnreadableNote}, never a `$0.00` claim.
|
|
2079
|
+
*/
|
|
2080
|
+
const rollbackAttempted = (run) => Boolean(run.report.rolledBack) ||
|
|
2081
|
+
(run.report.steps.create.status === "failed" &&
|
|
2082
|
+
run.report.steps.create.detail.startsWith("[E_ROLLBACK_INCOMPLETE]"));
|
|
2083
|
+
const askFor = (run) => run.fundAmount ??
|
|
2084
|
+
(run.report.strategyId !== undefined ? params.priorRequested?.get(run.report.strategyId) : undefined);
|
|
2085
|
+
const partialFunded = [];
|
|
2086
|
+
const unreadableFunded = [];
|
|
2087
|
+
for (const run of runs) {
|
|
2088
|
+
const requested = askFor(run);
|
|
2089
|
+
const funded = run.report.funded;
|
|
2090
|
+
const wallet = run.report.wallet;
|
|
2091
|
+
if (requested === undefined || requested <= 0)
|
|
2092
|
+
continue;
|
|
2093
|
+
if (wallet === undefined || rollbackAttempted(run))
|
|
2094
|
+
continue;
|
|
2095
|
+
if (funded === undefined) {
|
|
2096
|
+
unreadableFunded.push({
|
|
2097
|
+
instance: run.inst.name,
|
|
2098
|
+
wallet,
|
|
2099
|
+
requested,
|
|
2100
|
+
created: run.needsWallet,
|
|
2101
|
+
});
|
|
2102
|
+
continue;
|
|
2103
|
+
}
|
|
2104
|
+
if (funded >= requested * BUDGET_TOLERANCE)
|
|
2105
|
+
continue;
|
|
2106
|
+
partialFunded.push({
|
|
2107
|
+
instance: run.inst.name,
|
|
2108
|
+
wallet,
|
|
2109
|
+
requested,
|
|
2110
|
+
funded,
|
|
2111
|
+
created: run.needsWallet,
|
|
2112
|
+
});
|
|
2113
|
+
}
|
|
2114
|
+
/**
|
|
2115
|
+
* Whether a budget note may claim "nothing is broken — the strategy is LIVE".
|
|
2116
|
+
*
|
|
2117
|
+
* One condition, because one is all there is: the report as a WHOLE must be `live`. That verdict
|
|
2118
|
+
* is the sentence's whole subject, and {@link decideOverall} only reaches it when EVERY step of
|
|
2119
|
+
* every instance is `ok` or `skipped` — no refusal, no abandonment, nothing failed, pending or
|
|
2120
|
+
* unobserved anywhere in the package. The gate used to admit `installed-unobserved` too, and its
|
|
2121
|
+
* every() only ever looked at the rows the note names, so a fully-funded sibling sleeve that
|
|
2122
|
+
* never ticked left the note asserting "the strategy is LIVE" two lines under a verdict saying
|
|
2123
|
+
* the opposite.
|
|
2124
|
+
*
|
|
2125
|
+
* A per-row "installed and observed ticking" check rode along after that fix, justified by "live
|
|
2126
|
+
* tolerates a skipped observe". It does not, and the check was therefore dead: `observeInstance`
|
|
2127
|
+
* records only `ok`, `unobserved` or `failed`, the sole writer of a `skipped` observe is the
|
|
2128
|
+
* deadline honour — which sets `abandoned`, and an abandoned deploy is `failed` — and an instance
|
|
2129
|
+
* that was never observed at all keeps `pending`. Under `live`, every named row's observe is
|
|
2130
|
+
* already `ok`. A dead check kept alive by a comment that asserts the opposite is worse than no
|
|
2131
|
+
* check: the next maintainer adds a real skipped-observe path believing this gate covers it. If
|
|
2132
|
+
* one is ever added, `live` stops proving "observed ticking" and this is the site to revisit.
|
|
2133
|
+
*/
|
|
2134
|
+
const reassures = () => overall === "live";
|
|
2135
|
+
return {
|
|
2136
|
+
packageId: pkg.id,
|
|
2137
|
+
version: pkg.version,
|
|
2138
|
+
overall,
|
|
2139
|
+
instances: runs.map((run) => run.report),
|
|
2140
|
+
next: buildNext(overall, pkg, budget, runs),
|
|
2141
|
+
...softTier,
|
|
2142
|
+
...(partialFunded.length > 0
|
|
2143
|
+
? { partialFundNote: buildPartialFundNote(pkg, partialFunded, reassures()) }
|
|
2144
|
+
: {}),
|
|
2145
|
+
...(unreadableFunded.length > 0
|
|
2146
|
+
? { fundedUnreadableNote: buildFundedUnreadableNote(pkg, unreadableFunded, reassures()) }
|
|
2147
|
+
: {}),
|
|
2148
|
+
};
|
|
2149
|
+
}
|
|
2150
|
+
/**
|
|
2151
|
+
* Refusals and abandonment outrank outcomes, and an unobserved tick can never
|
|
2152
|
+
* read as success — the precedence order is the fail-closed one.
|
|
2153
|
+
*
|
|
2154
|
+
* An abandoned deploy is `failed`, not a status of its own: it ran out of wall clock without
|
|
2155
|
+
* finishing, and the resume path is the same one every failure gets.
|
|
2156
|
+
*/
|
|
2157
|
+
function decideOverall(runs, refused, abandoned) {
|
|
2158
|
+
if (refused)
|
|
2159
|
+
return "refused";
|
|
2160
|
+
if (abandoned)
|
|
2161
|
+
return "failed";
|
|
2162
|
+
const statuses = runs.flatMap((run) => STEP_ORDER.map((step) => run.report.steps[step].status));
|
|
2163
|
+
if (statuses.includes("failed"))
|
|
2164
|
+
return "failed";
|
|
2165
|
+
if (statuses.includes("pending"))
|
|
2166
|
+
return "pending";
|
|
2167
|
+
if (statuses.includes("unobserved"))
|
|
2168
|
+
return "installed-unobserved";
|
|
2169
|
+
return "live";
|
|
2170
|
+
}
|
|
2171
|
+
//# sourceMappingURL=orchestrator.js.map
|