@senpi/runtime 3.0.58 → 3.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -2
- package/dist/actions/action-events.d.ts +3 -3
- package/dist/actions/action-events.d.ts.map +1 -1
- package/dist/actions/action-events.js +24 -16
- package/dist/actions/action-events.js.map +1 -1
- package/dist/actions/action-runtime-module.d.ts.map +1 -1
- package/dist/actions/action-runtime-module.js +18 -1
- package/dist/actions/action-runtime-module.js.map +1 -1
- package/dist/actions/close-position/close-position.action.d.ts.map +1 -1
- package/dist/actions/close-position/close-position.action.js +84 -11
- package/dist/actions/close-position/close-position.action.js.map +1 -1
- package/dist/actions/decision-engine.d.ts +0 -2
- package/dist/actions/decision-engine.d.ts.map +1 -1
- package/dist/actions/decision-engine.js +1 -79
- package/dist/actions/decision-engine.js.map +1 -1
- package/dist/actions/event-derivation.d.ts +40 -2
- package/dist/actions/event-derivation.d.ts.map +1 -1
- package/dist/actions/event-derivation.js +178 -5
- package/dist/actions/event-derivation.js.map +1 -1
- package/dist/actions/open-position/open-position.action.d.ts.map +1 -1
- package/dist/actions/open-position/open-position.action.js +103 -16
- package/dist/actions/open-position/open-position.action.js.map +1 -1
- package/dist/actions/position-tracker/position-tracker.action.d.ts +2 -1
- package/dist/actions/position-tracker/position-tracker.action.d.ts.map +1 -1
- package/dist/actions/position-tracker/position-tracker.action.js +47 -15
- package/dist/actions/position-tracker/position-tracker.action.js.map +1 -1
- package/dist/actions/skipped-signal-results.d.ts +16 -0
- package/dist/actions/skipped-signal-results.d.ts.map +1 -0
- package/dist/actions/skipped-signal-results.js +47 -0
- package/dist/actions/skipped-signal-results.js.map +1 -0
- package/dist/cli/deploy-render.d.ts +91 -0
- package/dist/cli/deploy-render.d.ts.map +1 -0
- package/dist/cli/deploy-render.js +216 -0
- package/dist/cli/deploy-render.js.map +1 -0
- package/dist/cli/guide-commands.d.ts.map +1 -1
- package/dist/cli/guide-commands.js +78 -6
- package/dist/cli/guide-commands.js.map +1 -1
- package/dist/cli/senpi-commands.d.ts +6 -1
- package/dist/cli/senpi-commands.d.ts.map +1 -1
- package/dist/cli/senpi-commands.js +164 -5
- package/dist/cli/senpi-commands.js.map +1 -1
- package/dist/cli/validate-command.d.ts.map +1 -1
- package/dist/cli/validate-command.js +32 -1
- package/dist/cli/validate-command.js.map +1 -1
- package/dist/deploy/deadlines.d.ts +47 -0
- package/dist/deploy/deadlines.d.ts.map +1 -0
- package/dist/deploy/deadlines.js +56 -0
- package/dist/deploy/deadlines.js.map +1 -0
- package/dist/deploy/error-text.d.ts +25 -0
- package/dist/deploy/error-text.d.ts.map +1 -0
- package/dist/deploy/error-text.js +42 -0
- package/dist/deploy/error-text.js.map +1 -0
- package/dist/deploy/funding.d.ts +87 -0
- package/dist/deploy/funding.d.ts.map +1 -0
- package/dist/deploy/funding.js +229 -0
- package/dist/deploy/funding.js.map +1 -0
- package/dist/deploy/install-gate-runner.d.ts +12 -0
- package/dist/deploy/install-gate-runner.d.ts.map +1 -0
- package/dist/deploy/install-gate-runner.js +17 -0
- package/dist/deploy/install-gate-runner.js.map +1 -0
- package/dist/deploy/job.d.ts +199 -0
- package/dist/deploy/job.d.ts.map +1 -0
- package/dist/deploy/job.js +463 -0
- package/dist/deploy/job.js.map +1 -0
- package/dist/deploy/min-budget.d.ts +68 -0
- package/dist/deploy/min-budget.d.ts.map +1 -0
- package/dist/deploy/min-budget.js +297 -0
- package/dist/deploy/min-budget.js.map +1 -0
- package/dist/deploy/orchestrator.d.ts +305 -0
- package/dist/deploy/orchestrator.d.ts.map +1 -0
- package/dist/deploy/orchestrator.js +2910 -0
- package/dist/deploy/orchestrator.js.map +1 -0
- package/dist/deploy/package.d.ts +130 -0
- package/dist/deploy/package.d.ts.map +1 -0
- package/dist/deploy/package.js +351 -0
- package/dist/deploy/package.js.map +1 -0
- package/dist/deploy/proof-gate.d.ts +82 -0
- package/dist/deploy/proof-gate.d.ts.map +1 -0
- package/dist/deploy/proof-gate.js +269 -0
- package/dist/deploy/proof-gate.js.map +1 -0
- package/dist/deploy/python-float.d.ts +34 -0
- package/dist/deploy/python-float.d.ts.map +1 -0
- package/dist/deploy/python-float.js +46 -0
- package/dist/deploy/python-float.js.map +1 -0
- package/dist/deploy/recheck-commands.d.ts +50 -0
- package/dist/deploy/recheck-commands.d.ts.map +1 -0
- package/dist/deploy/recheck-commands.js +59 -0
- package/dist/deploy/recheck-commands.js.map +1 -0
- package/dist/deploy/register.d.ts +99 -0
- package/dist/deploy/register.d.ts.map +1 -0
- package/dist/deploy/register.js +689 -0
- package/dist/deploy/register.js.map +1 -0
- package/dist/deploy/taxonomy.d.ts +24 -0
- package/dist/deploy/taxonomy.d.ts.map +1 -0
- package/dist/deploy/taxonomy.js +31 -0
- package/dist/deploy/taxonomy.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +244 -132
- package/dist/index.js.map +1 -1
- package/dist/runtime/lifecycle-events.d.ts +13 -0
- package/dist/runtime/lifecycle-events.d.ts.map +1 -1
- package/dist/runtime/lifecycle-events.js +32 -0
- package/dist/runtime/lifecycle-events.js.map +1 -1
- package/dist/runtime/runtime.d.ts.map +1 -1
- package/dist/runtime/runtime.js +3 -1
- package/dist/runtime/runtime.js.map +1 -1
- package/dist/runtime/strategy-registry.d.ts +8 -0
- package/dist/runtime/strategy-registry.d.ts.map +1 -1
- package/dist/runtime/strategy-registry.js +14 -1
- package/dist/runtime/strategy-registry.js.map +1 -1
- package/dist/runtime/supervisor/long-running-supervisor.d.ts +8 -0
- package/dist/runtime/supervisor/long-running-supervisor.d.ts.map +1 -1
- package/dist/runtime/supervisor/long-running-supervisor.js +1 -0
- package/dist/runtime/supervisor/long-running-supervisor.js.map +1 -1
- package/dist/runtime-api/intake/intake-events.d.ts +19 -3
- package/dist/runtime-api/intake/intake-events.d.ts.map +1 -1
- package/dist/runtime-api/intake/intake-events.js +64 -2
- package/dist/runtime-api/intake/intake-events.js.map +1 -1
- package/dist/runtime-api/intake/intake-mount.d.ts.map +1 -1
- package/dist/runtime-api/intake/intake-mount.js +7 -4
- package/dist/runtime-api/intake/intake-mount.js.map +1 -1
- package/dist/runtime-api/intake/process-signal-post.d.ts.map +1 -1
- package/dist/runtime-api/intake/process-signal-post.js +7 -1
- package/dist/runtime-api/intake/process-signal-post.js.map +1 -1
- package/dist/runtime-api/intake/scanner-launcher.d.ts.map +1 -1
- package/dist/runtime-api/intake/scanner-launcher.js +12 -0
- package/dist/runtime-api/intake/scanner-launcher.js.map +1 -1
- package/dist/runtime-api/intake/types.d.ts +24 -0
- package/dist/runtime-api/intake/types.d.ts.map +1 -1
- package/dist/runtime-api/routes/signals.d.ts.map +1 -1
- package/dist/runtime-api/routes/signals.js +10 -1
- package/dist/runtime-api/routes/signals.js.map +1 -1
- package/dist/runtime-api/routes/signals.schema.d.ts +10 -0
- package/dist/runtime-api/routes/signals.schema.d.ts.map +1 -1
- package/dist/runtime-api/routes/signals.schema.js +14 -0
- package/dist/runtime-api/routes/signals.schema.js.map +1 -1
- package/dist/scanners/events.d.ts +7 -0
- package/dist/scanners/events.d.ts.map +1 -1
- package/dist/scanners/events.js.map +1 -1
- package/dist/scanners/external/scaffold/delivery.py +23 -12
- package/dist/scanners/external/scaffold/scaffold.py +55 -7
- package/dist/scanners/implementations/market-regime.d.ts +1 -1
- package/dist/scanners/runtime-module.d.ts.map +1 -1
- package/dist/scanners/runtime-module.js +10 -2
- package/dist/scanners/runtime-module.js.map +1 -1
- package/dist/senpi/client.d.ts +58 -4
- package/dist/senpi/client.d.ts.map +1 -1
- package/dist/senpi/client.js +192 -15
- package/dist/senpi/client.js.map +1 -1
- package/dist/senpi/strategy-status.d.ts +32 -0
- package/dist/senpi/strategy-status.d.ts.map +1 -0
- package/dist/senpi/strategy-status.js +70 -0
- package/dist/senpi/strategy-status.js.map +1 -0
- package/dist/senpi/types.d.ts +47 -3
- package/dist/senpi/types.d.ts.map +1 -1
- package/dist/skills-manager/manifest.js +1 -1
- package/dist/skills-manager/skills-manifest.json +1 -1
- package/dist/telemetry/event-builders.d.ts +45 -6
- package/dist/telemetry/event-builders.d.ts.map +1 -1
- package/dist/telemetry/event-builders.js +99 -11
- package/dist/telemetry/event-builders.js.map +1 -1
- package/dist/telemetry/metrics.d.ts +37 -2
- package/dist/telemetry/metrics.d.ts.map +1 -1
- package/dist/telemetry/metrics.js +84 -4
- package/dist/telemetry/metrics.js.map +1 -1
- package/dist/telemetry/scanner-tick-event.d.ts +32 -0
- package/dist/telemetry/scanner-tick-event.d.ts.map +1 -0
- package/dist/telemetry/scanner-tick-event.js +32 -0
- package/dist/telemetry/scanner-tick-event.js.map +1 -0
- package/dist/types/strategy.d.ts +9 -1
- package/dist/types/strategy.d.ts.map +1 -1
- package/dist/universe/package-universe.d.ts +97 -0
- package/dist/universe/package-universe.d.ts.map +1 -0
- package/dist/universe/package-universe.js +275 -0
- package/dist/universe/package-universe.js.map +1 -0
- package/dist/utils/event-catalog.d.ts +58 -0
- package/dist/utils/event-catalog.d.ts.map +1 -1
- package/dist/utils/event-catalog.js +1 -0
- package/dist/utils/event-catalog.js.map +1 -1
- package/dist/validate/codes.d.ts +20 -0
- package/dist/validate/codes.d.ts.map +1 -1
- package/dist/validate/codes.js +36 -0
- package/dist/validate/codes.js.map +1 -1
- package/dist/validate/import-stage.js +5 -5
- package/dist/validate/import-stage.js.map +1 -1
- package/dist/validate/index.d.ts +1 -1
- package/dist/validate/index.d.ts.map +1 -1
- package/dist/validate/index.js.map +1 -1
- package/dist/validate/install-gate.d.ts.map +1 -1
- package/dist/validate/install-gate.js +10 -2
- package/dist/validate/install-gate.js.map +1 -1
- package/dist/validate/live-instruments.d.ts +44 -0
- package/dist/validate/live-instruments.d.ts.map +1 -0
- package/dist/validate/live-instruments.js +71 -0
- package/dist/validate/live-instruments.js.map +1 -0
- package/dist/validate/live-stage.js +5 -5
- package/dist/validate/live-stage.js.map +1 -1
- package/dist/validate/proof.d.ts +11 -1
- package/dist/validate/proof.d.ts.map +1 -1
- package/dist/validate/proof.js +7 -6
- package/dist/validate/proof.js.map +1 -1
- package/dist/validate/recipe-checks.d.ts.map +1 -1
- package/dist/validate/recipe-checks.js +6 -4
- package/dist/validate/recipe-checks.js.map +1 -1
- package/dist/validate/run.d.ts +5 -0
- package/dist/validate/run.d.ts.map +1 -1
- package/dist/validate/run.js +10 -1
- package/dist/validate/run.js.map +1 -1
- package/dist/validate/target.d.ts +41 -0
- package/dist/validate/target.d.ts.map +1 -1
- package/dist/validate/target.js +192 -16
- package/dist/validate/target.js.map +1 -1
- package/dist/validate/universe-stage.d.ts +71 -0
- package/dist/validate/universe-stage.d.ts.map +1 -0
- package/dist/validate/universe-stage.js +183 -0
- package/dist/validate/universe-stage.js.map +1 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/dist/runtime/managed-wallet-guard.d.ts +0 -47
- package/dist/runtime/managed-wallet-guard.d.ts.map +0 -1
- package/dist/runtime/managed-wallet-guard.js +0 -106
- package/dist/runtime/managed-wallet-guard.js.map +0 -1
- package/dist/runtime/protection-nudge.d.ts +0 -67
- package/dist/runtime/protection-nudge.d.ts.map +0 -1
- package/dist/runtime/protection-nudge.js +0 -244
- package/dist/runtime/protection-nudge.js.map +0 -1
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import type { DeployEvent, DeployEventBody, DeployReport, JobPhase } from "./orchestrator.js";
|
|
2
|
+
export type JobState =
|
|
3
|
+
/** `stopRequested` = the deadline watchdog has asked the run to stop at its next step boundary. */
|
|
4
|
+
{
|
|
5
|
+
status: "running";
|
|
6
|
+
phase: JobPhase;
|
|
7
|
+
stopRequested: boolean;
|
|
8
|
+
} | {
|
|
9
|
+
status: "done";
|
|
10
|
+
overall: DeployReport["overall"];
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Stamped by `scanOnBoot`, and SYNTHESIZED by {@link DeployJobController.readJournal} for any
|
|
14
|
+
* journal with no terminal line — which is the same thing in almost every case, and not in one:
|
|
15
|
+
* `append` swallows write failures (see its doc), so a deploy that ran to `live` on a full or
|
|
16
|
+
* read-only volume never got its terminal line written and reads `interrupted` forever after.
|
|
17
|
+
* A live process still never produces this state for its OWN job — `status` answers from memory
|
|
18
|
+
* while the job is running.
|
|
19
|
+
*/
|
|
20
|
+
| {
|
|
21
|
+
status: "interrupted";
|
|
22
|
+
};
|
|
23
|
+
export interface DeployJobMeta {
|
|
24
|
+
/** "dpl-" + 8 lowercase hex */
|
|
25
|
+
deployId: string;
|
|
26
|
+
packageId: string;
|
|
27
|
+
version: string;
|
|
28
|
+
packageDir: string;
|
|
29
|
+
budget?: number;
|
|
30
|
+
/** ISO */
|
|
31
|
+
startedAt: string;
|
|
32
|
+
}
|
|
33
|
+
export interface JobSnapshot {
|
|
34
|
+
meta: DeployJobMeta;
|
|
35
|
+
state: JobState;
|
|
36
|
+
/** tail, newest last (capped in memory; the journal holds the full history) */
|
|
37
|
+
events: DeployEvent[];
|
|
38
|
+
/** terminal jobs only */
|
|
39
|
+
report?: DeployReport;
|
|
40
|
+
}
|
|
41
|
+
/** What `run` is handed: its own id, narration in, the deadline stop out. Nothing else. */
|
|
42
|
+
export interface JobHandle {
|
|
43
|
+
/** This job's id — the `senpi.deploy.id` span attribute the run's own spans carry. */
|
|
44
|
+
deployId: string;
|
|
45
|
+
emit(event: DeployEventBody): void;
|
|
46
|
+
shouldAbandon(): boolean;
|
|
47
|
+
}
|
|
48
|
+
export interface DeployJobControllerOptions {
|
|
49
|
+
deploysDir: string;
|
|
50
|
+
now(): number;
|
|
51
|
+
log(line: string): void;
|
|
52
|
+
/** journals retained by the boot scan; default 20 */
|
|
53
|
+
keep?: number;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* H6: every MCP call a deploy makes is separately deadline-bounded. Owned here, beside the grace it
|
|
57
|
+
* determines, so the two can never be hand-synced apart; `register.ts` imports it for its own
|
|
58
|
+
* per-call signals.
|
|
59
|
+
*/
|
|
60
|
+
export declare const MCP_CALL_TIMEOUT_MS = 90000;
|
|
61
|
+
/**
|
|
62
|
+
* How long the soft stop gets to land at a step boundary before the slot is taken back by force.
|
|
63
|
+
*
|
|
64
|
+
* DERIVED, never a bare number: force-terminate releases the single-flight slot while the abandoned
|
|
65
|
+
* run may still be waiting on a call. Sized past the per-call deadline so that by the time the slot
|
|
66
|
+
* frees, every call the abandoned run could still be awaiting has already been abandoned by its own
|
|
67
|
+
* deadline — a NEW deploy never starts its preflight/create while the old job is still parked on a
|
|
68
|
+
* money-moving call against the same funding waterfall. The headroom covers the abort landing and
|
|
69
|
+
* the step-boundary check that follows it.
|
|
70
|
+
*
|
|
71
|
+
* What this does NOT buy: a deadline bounds the WAIT, not the backend's work (`boundAwait` in
|
|
72
|
+
* `senpi/client.ts` — the gateway-injected MCP transport accepts no abort option). A create or fund
|
|
73
|
+
* this job stopped waiting on may still complete server-side after the slot has been handed on.
|
|
74
|
+
* What the grace guarantees is that the job is no longer *waiting* on a money-moving call when the
|
|
75
|
+
* slot comes back; the residue is reconciled by the next deploy, not prevented here.
|
|
76
|
+
*
|
|
77
|
+
* (This supersedes the plan's locked 60s: 60 < the locked 90s call timeout, so the two locked
|
|
78
|
+
* defaults were mutually inconsistent.)
|
|
79
|
+
*/
|
|
80
|
+
export declare const GRACE_MS: number;
|
|
81
|
+
/**
|
|
82
|
+
* The single-flight slot is IN-MEMORY ONLY (`this.live`). It must never be persisted: an
|
|
83
|
+
* interrupted job belongs to a dead process and leaves the slot free by construction — persisting
|
|
84
|
+
* it would wedge every deploy on the box after one crash.
|
|
85
|
+
*/
|
|
86
|
+
export declare class DeployJobController {
|
|
87
|
+
private readonly deploysDir;
|
|
88
|
+
private readonly now;
|
|
89
|
+
private readonly log;
|
|
90
|
+
private readonly keep;
|
|
91
|
+
private live;
|
|
92
|
+
constructor(opts: DeployJobControllerOptions);
|
|
93
|
+
/** The running job's identity and progress, or null. The single-flight gate reads this. */
|
|
94
|
+
running(): (DeployJobMeta & {
|
|
95
|
+
phase: JobPhase;
|
|
96
|
+
stopRequested: boolean;
|
|
97
|
+
}) | null;
|
|
98
|
+
/**
|
|
99
|
+
* Claim the single job slot and fire `run` un-awaited.
|
|
100
|
+
*
|
|
101
|
+
* Single-flight is a money invariant, not a convenience: concurrent deploys
|
|
102
|
+
* share one funding waterfall, so two preflights can both pass while jointly
|
|
103
|
+
* overdrawing. Throws when a job is already running. The ordinary
|
|
104
|
+
* `E_DEPLOY_IN_PROGRESS` refusal comes from the gateway's own `running()`
|
|
105
|
+
* check BEFORE it gets here; this throw is the lost race between that check
|
|
106
|
+
* and this call, and the gateway catches it and renders the same refusal.
|
|
107
|
+
*
|
|
108
|
+
* `capMs` is the job's wall-clock cap (H6). Every MCP call the run makes is separately
|
|
109
|
+
* timeout-bounded, but a cap is what guarantees the slot comes back: one hung await used to hold
|
|
110
|
+
* it until the gateway restarted, and a boundary-honored stop flag never protected that case.
|
|
111
|
+
*/
|
|
112
|
+
start(meta: Omit<DeployJobMeta, "deployId" | "startedAt">, capMs: number, run: (job: JobHandle) => Promise<DeployReport>): DeployJobMeta;
|
|
113
|
+
/**
|
|
114
|
+
* The running job (from memory) when it is the one being asked about — by id or by omission —
|
|
115
|
+
* else the newest journal, or the named one. Unknown id → null.
|
|
116
|
+
*
|
|
117
|
+
* The live check MUST come first even for an explicit id: a
|
|
118
|
+
* running job's journal has no terminal line yet, so falling through to {@link readJournal}
|
|
119
|
+
* would render it `interrupted` — a state only the boot scan may ever assign, and a flat lie
|
|
120
|
+
* about a job that is actively moving money.
|
|
121
|
+
*/
|
|
122
|
+
status(deployId?: string): Promise<JobSnapshot | null>;
|
|
123
|
+
/**
|
|
124
|
+
* C2 carve-out: the ONE sanctioned journal read that feeds the orchestrator — and only as a
|
|
125
|
+
* LOOKUP KEY. Every id returned here is verified against the live backend (strategy_list
|
|
126
|
+
* strategyIds:[…]) before reconcile acts on it; the journal never decides anything.
|
|
127
|
+
*
|
|
128
|
+
* It exists because a create whose custom name the backend rejected lands under a
|
|
129
|
+
* backend-assigned name, which the reconcile name match can never find again: without the key,
|
|
130
|
+
* a re-run creates and funds a second wallet beside the first.
|
|
131
|
+
*
|
|
132
|
+
* Returns instance name → journaled created-strategy ids, newest journals first.
|
|
133
|
+
*
|
|
134
|
+
* `deployId` scopes the answer to ONE journal. Reconcile deliberately passes nothing: every
|
|
135
|
+
* journal of the package is a legitimate key source there, because any of them may hold the id
|
|
136
|
+
* of the wallet a re-run must adopt instead of funding a second one (D-9). The interrupted RENDER
|
|
137
|
+
* passes its own id, because it answers a narrower question — what did THIS job leave behind —
|
|
138
|
+
* and a still-live wallet from an earlier completed deploy is not that. Rendering it there would
|
|
139
|
+
* offer to close a strategy that is running exactly as intended.
|
|
140
|
+
*/
|
|
141
|
+
priorCreatedStrategyIds(packageId: string, deployId?: string): Map<string, string[]>;
|
|
142
|
+
/**
|
|
143
|
+
* The second D-9 lookup input off the SAME create-key event: strategyId → the amount that create
|
|
144
|
+
* call asked the backend to fund.
|
|
145
|
+
*
|
|
146
|
+
* Narrower than {@link priorCreatedStrategyIds} in what it may do, and deliberately so. It feeds
|
|
147
|
+
* exactly one thing — the report's funded-vs-requested comparison — and only the REQUESTED half
|
|
148
|
+
* of it; the funded half is always re-read live from `strategy_list` on the current run. No
|
|
149
|
+
* orchestration branch reads it, so a stale, missing or wrong entry costs a warn, never an action.
|
|
150
|
+
*
|
|
151
|
+
* It exists because the ask lives only in the memory of the run that funded. A deploy that funds
|
|
152
|
+
* $60 of a $500 ask and dies before assembling its report resumes as an adoption, and without
|
|
153
|
+
* this the shortfall is simply never mentioned again.
|
|
154
|
+
*
|
|
155
|
+
* Newest journal wins on a repeated strategyId — the ask that most recently produced that wallet.
|
|
156
|
+
*/
|
|
157
|
+
priorRequestedFunding(packageId: string): Map<string, number>;
|
|
158
|
+
/**
|
|
159
|
+
* Boot janitor: stamp every unterminated journal `interrupted`, then prune to
|
|
160
|
+
* the newest `keep`. Discovery is pull-based — the next `openclaw senpi deploy status`
|
|
161
|
+
* renders the interruption; nothing is resumed or notified here.
|
|
162
|
+
*/
|
|
163
|
+
scanOnBoot(): Promise<void>;
|
|
164
|
+
private recordEvent;
|
|
165
|
+
/**
|
|
166
|
+
* Close a job out. The slot is released in `finally`: a journal-write failure is narration lost,
|
|
167
|
+
* never a wedged single-flight slot that would refuse every future deploy with a stale
|
|
168
|
+
* `E_DEPLOY_IN_PROGRESS`.
|
|
169
|
+
*/
|
|
170
|
+
private terminate;
|
|
171
|
+
private journalPath;
|
|
172
|
+
/**
|
|
173
|
+
* Append one journal line. **Never throws**: the journal is narration, and the narration-only
|
|
174
|
+
* rule cuts both ways — a full or read-only state volume must not be able to change what a
|
|
175
|
+
* deploy does (or wedge its slot). A failed write is logged and the line is lost.
|
|
176
|
+
*/
|
|
177
|
+
private append;
|
|
178
|
+
private journalIds;
|
|
179
|
+
private readLines;
|
|
180
|
+
/** Journals newest-first by header `startedAt` — the only ordering the controller trusts. */
|
|
181
|
+
private journalIdsNewestFirst;
|
|
182
|
+
private newestJournalId;
|
|
183
|
+
/**
|
|
184
|
+
* Journal → snapshot. A journal with no terminal line IS an interrupted job.
|
|
185
|
+
*
|
|
186
|
+
* The FIRST terminal line wins. That ordering is load-bearing: when the deadline watchdog
|
|
187
|
+
* force-terminates a wedged job, the zombie `run` promise may settle later and append a second
|
|
188
|
+
* terminal line — the watchdog's verdict is the true one, and reading the first keeps it.
|
|
189
|
+
*/
|
|
190
|
+
private readJournal;
|
|
191
|
+
/** Retain the newest `keep` journals. Runs on every terminate and on boot, so a long-lived
|
|
192
|
+
* gateway does not grow `deploys/` without bound between restarts. Never throws.
|
|
193
|
+
*
|
|
194
|
+
* Unlocked by design: prune and every journal read are fully synchronous on one thread, so they
|
|
195
|
+
* cannot interleave in-process; a torn last line is already tolerated by `readLines`. Do not add
|
|
196
|
+
* locking without a cross-process caller. */
|
|
197
|
+
private prune;
|
|
198
|
+
}
|
|
199
|
+
//# sourceMappingURL=job.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"job.d.ts","sourceRoot":"","sources":["../../src/deploy/job.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAsB9F,MAAM,MAAM,QAAQ;AAClB,mGAAmG;AACjG;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,QAAQ,CAAC;IAAC,aAAa,EAAE,OAAO,CAAA;CAAE,GAC9D;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAC,SAAS,CAAC,CAAA;CAAE;AACtD;;;;;;;GAOG;GACD;IAAE,MAAM,EAAE,aAAa,CAAA;CAAE,CAAC;AAE9B,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU;IACV,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,QAAQ,CAAC;IAChB,+EAA+E;IAC/E,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,yBAAyB;IACzB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED,2FAA2F;AAC3F,MAAM,WAAW,SAAS;IACxB,sFAAsF;IACtF,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,KAAK,EAAE,eAAe,GAAG,IAAI,CAAC;IACnC,aAAa,IAAI,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,0BAA0B;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,IAAI,MAAM,CAAC;IACd,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,qDAAqD;IACrD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AA6BD;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,QAAS,CAAC;AAE1C;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,QAAQ,QAA+B,CAAC;AAErD;;;;GAIG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAyB;IAC7C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,IAAI,CAAwB;gBAExB,IAAI,EAAE,0BAA0B;IAO5C,2FAA2F;IAC3F,OAAO,IAAI,CAAC,aAAa,GAAG;QAAE,KAAK,EAAE,QAAQ,CAAC;QAAC,aAAa,EAAE,OAAO,CAAA;KAAE,CAAC,GAAG,IAAI;IAK/E;;;;;;;;;;;;;OAaG;IACH,KAAK,CACH,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,UAAU,GAAG,WAAW,CAAC,EACnD,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,YAAY,CAAC,GAC7C,aAAa;IAuFhB;;;;;;;;OAQG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAiB5D;;;;;;;;;;;;;;;;;OAiBG;IACH,uBAAuB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;IAuBpF;;;;;;;;;;;;;;OAcG;IACH,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAkB7D;;;;OAIG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBjC,OAAO,CAAC,WAAW;IASnB;;;;OAIG;IACH,OAAO,CAAC,SAAS;IA8BjB,OAAO,CAAC,WAAW;IAInB;;;;OAIG;IACH,OAAO,CAAC,MAAM;IAQd,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,SAAS;IAejB,6FAA6F;IAC7F,OAAO,CAAC,qBAAqB;IAY7B,OAAO,CAAC,eAAe;IAIvB;;;;;;OAMG;IACH,OAAO,CAAC,WAAW;IAsBnB;;;;;iDAK6C;IAC7C,OAAO,CAAC,KAAK;CAQd"}
|
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Everything async about a deploy: the single-flight job slot, the append-only
|
|
3
|
+
* JSONL journal, the wall-clock deadline watchdog, the boot-time interrupted
|
|
4
|
+
* scan, and status snapshots. It runs whatever `run` closure the gateway hands
|
|
5
|
+
* it and knows nothing about MCP or orchestration.
|
|
6
|
+
*
|
|
7
|
+
* **There is no cancel** (D-4): undeploying a strategy is closing it, not
|
|
8
|
+
* stopping the job. The only thing that stops a running deploy early is the
|
|
9
|
+
* deadline watchdog armed in {@link DeployJobController.start} — a soft stop the
|
|
10
|
+
* orchestrator honours at its next step boundary, followed by a force-terminate
|
|
11
|
+
* that frees the slot even when the run is wedged inside an await.
|
|
12
|
+
*
|
|
13
|
+
* **The journal is narration-only.** No code path may read `deploys/*.jsonl` to
|
|
14
|
+
* decide an action. Journals are read for exactly three purposes: rendering
|
|
15
|
+
* `status` history, the boot scan that stamps unterminated journals
|
|
16
|
+
* `interrupted`, and the one sanctioned carve-out — {@link
|
|
17
|
+
* DeployJobController.priorCreatedStrategyIds}, which yields a LOOKUP KEY (a
|
|
18
|
+
* strategyId this box journaled at create time) that reconcile must verify
|
|
19
|
+
* against the live backend before acting on it. Every orchestrator decision
|
|
20
|
+
* still reads the live surfaces (backend strategies + runtime registry) — that
|
|
21
|
+
* is what keeps the E_STATE drift class dead.
|
|
22
|
+
*
|
|
23
|
+
* **No auto-resume.** `scanOnBoot` is a janitor: it stamps and prunes, and never
|
|
24
|
+
* re-enters a state machine. Only a fresh agent-initiated `senpi deploy`
|
|
25
|
+
* resumes, via reconciliation.
|
|
26
|
+
*/
|
|
27
|
+
import { randomBytes } from "node:crypto";
|
|
28
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
29
|
+
import { join } from "node:path";
|
|
30
|
+
import { recordDeploy } from "../telemetry/metrics.js";
|
|
31
|
+
import { extractTaxonomyCode } from "./taxonomy.js";
|
|
32
|
+
/**
|
|
33
|
+
* The taxonomy code of the FIRST refused step in the report, for the deploy metric's dimension.
|
|
34
|
+
*
|
|
35
|
+
* Reads the shared extractor, not a private pattern: this was the third reader, and its narrower
|
|
36
|
+
* one silently dropped any code carrying a digit — so `[E_MIN_10]` would exit 2 at the CLI and
|
|
37
|
+
* carry a `senpi.deploy.refusal.code` span attribute while producing no metric dimension at all.
|
|
38
|
+
*/
|
|
39
|
+
function firstRefusalCode(report) {
|
|
40
|
+
for (const instance of report.instances) {
|
|
41
|
+
for (const outcome of Object.values(instance.steps)) {
|
|
42
|
+
if (outcome.status !== "refused")
|
|
43
|
+
continue;
|
|
44
|
+
const code = extractTaxonomyCode(outcome.detail);
|
|
45
|
+
if (code)
|
|
46
|
+
return code;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
const EVENT_TAIL_MAX = 50;
|
|
52
|
+
const DEFAULT_KEEP = 20;
|
|
53
|
+
const DEPLOY_ID_RE = /^dpl-[0-9a-f]{8}$/;
|
|
54
|
+
function errorText(err) {
|
|
55
|
+
return err instanceof Error ? err.message : String(err);
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* H6: every MCP call a deploy makes is separately deadline-bounded. Owned here, beside the grace it
|
|
59
|
+
* determines, so the two can never be hand-synced apart; `register.ts` imports it for its own
|
|
60
|
+
* per-call signals.
|
|
61
|
+
*/
|
|
62
|
+
export const MCP_CALL_TIMEOUT_MS = 90_000;
|
|
63
|
+
/**
|
|
64
|
+
* How long the soft stop gets to land at a step boundary before the slot is taken back by force.
|
|
65
|
+
*
|
|
66
|
+
* DERIVED, never a bare number: force-terminate releases the single-flight slot while the abandoned
|
|
67
|
+
* run may still be waiting on a call. Sized past the per-call deadline so that by the time the slot
|
|
68
|
+
* frees, every call the abandoned run could still be awaiting has already been abandoned by its own
|
|
69
|
+
* deadline — a NEW deploy never starts its preflight/create while the old job is still parked on a
|
|
70
|
+
* money-moving call against the same funding waterfall. The headroom covers the abort landing and
|
|
71
|
+
* the step-boundary check that follows it.
|
|
72
|
+
*
|
|
73
|
+
* What this does NOT buy: a deadline bounds the WAIT, not the backend's work (`boundAwait` in
|
|
74
|
+
* `senpi/client.ts` — the gateway-injected MCP transport accepts no abort option). A create or fund
|
|
75
|
+
* this job stopped waiting on may still complete server-side after the slot has been handed on.
|
|
76
|
+
* What the grace guarantees is that the job is no longer *waiting* on a money-moving call when the
|
|
77
|
+
* slot comes back; the residue is reconciled by the next deploy, not prevented here.
|
|
78
|
+
*
|
|
79
|
+
* (This supersedes the plan's locked 60s: 60 < the locked 90s call timeout, so the two locked
|
|
80
|
+
* defaults were mutually inconsistent.)
|
|
81
|
+
*/
|
|
82
|
+
export const GRACE_MS = MCP_CALL_TIMEOUT_MS + 30_000;
|
|
83
|
+
/**
|
|
84
|
+
* The single-flight slot is IN-MEMORY ONLY (`this.live`). It must never be persisted: an
|
|
85
|
+
* interrupted job belongs to a dead process and leaves the slot free by construction — persisting
|
|
86
|
+
* it would wedge every deploy on the box after one crash.
|
|
87
|
+
*/
|
|
88
|
+
export class DeployJobController {
|
|
89
|
+
deploysDir;
|
|
90
|
+
now;
|
|
91
|
+
log;
|
|
92
|
+
keep;
|
|
93
|
+
live = null;
|
|
94
|
+
constructor(opts) {
|
|
95
|
+
this.deploysDir = opts.deploysDir;
|
|
96
|
+
this.now = opts.now;
|
|
97
|
+
this.log = opts.log;
|
|
98
|
+
this.keep = opts.keep ?? DEFAULT_KEEP;
|
|
99
|
+
}
|
|
100
|
+
/** The running job's identity and progress, or null. The single-flight gate reads this. */
|
|
101
|
+
running() {
|
|
102
|
+
if (!this.live)
|
|
103
|
+
return null;
|
|
104
|
+
return { ...this.live.meta, phase: this.live.phase, stopRequested: this.live.stopRequested };
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Claim the single job slot and fire `run` un-awaited.
|
|
108
|
+
*
|
|
109
|
+
* Single-flight is a money invariant, not a convenience: concurrent deploys
|
|
110
|
+
* share one funding waterfall, so two preflights can both pass while jointly
|
|
111
|
+
* overdrawing. Throws when a job is already running. The ordinary
|
|
112
|
+
* `E_DEPLOY_IN_PROGRESS` refusal comes from the gateway's own `running()`
|
|
113
|
+
* check BEFORE it gets here; this throw is the lost race between that check
|
|
114
|
+
* and this call, and the gateway catches it and renders the same refusal.
|
|
115
|
+
*
|
|
116
|
+
* `capMs` is the job's wall-clock cap (H6). Every MCP call the run makes is separately
|
|
117
|
+
* timeout-bounded, but a cap is what guarantees the slot comes back: one hung await used to hold
|
|
118
|
+
* it until the gateway restarted, and a boundary-honored stop flag never protected that case.
|
|
119
|
+
*/
|
|
120
|
+
start(meta, capMs, run) {
|
|
121
|
+
if (this.live) {
|
|
122
|
+
throw new Error(`deploy ${this.live.meta.deployId} is already running (${this.live.phase})`);
|
|
123
|
+
}
|
|
124
|
+
const full = {
|
|
125
|
+
...meta,
|
|
126
|
+
deployId: `dpl-${randomBytes(4).toString("hex")}`,
|
|
127
|
+
startedAt: new Date(this.now()).toISOString(),
|
|
128
|
+
};
|
|
129
|
+
const job = { meta: full, phase: "reconcile", stopRequested: false, terminated: false, events: [], seq: 0 };
|
|
130
|
+
// Header first, slot second: the journal is narration, so a write failure must not be able to
|
|
131
|
+
// claim (and then never release) the single-flight slot. `append` never throws — see its JSDoc.
|
|
132
|
+
try {
|
|
133
|
+
mkdirSync(this.deploysDir, { recursive: true });
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
this.log(`deploy ${full.deployId}: could not create ${this.deploysDir} — journaling disabled (${errorText(err)})`);
|
|
137
|
+
}
|
|
138
|
+
this.append(full.deployId, { kind: "header", ...full });
|
|
139
|
+
this.live = job;
|
|
140
|
+
const handle = {
|
|
141
|
+
deployId: full.deployId,
|
|
142
|
+
emit: (event) => this.recordEvent(job, event),
|
|
143
|
+
shouldAbandon: () => job.stopRequested,
|
|
144
|
+
};
|
|
145
|
+
// Soft stop at the cap, hard stop `GRACE_MS` later. Both timers are unref'd so a deploy never
|
|
146
|
+
// keeps the gateway process alive, and both are cleared in `terminate`.
|
|
147
|
+
job.watchdog = setTimeout(() => {
|
|
148
|
+
job.stopRequested = true;
|
|
149
|
+
this.recordEvent(job, {
|
|
150
|
+
type: "job",
|
|
151
|
+
detail: `deploy deadline (${Math.round(capMs / 1000)}s) exceeded — abandoning at the next step boundary`,
|
|
152
|
+
});
|
|
153
|
+
job.grace = setTimeout(() => {
|
|
154
|
+
if (this.live === job) {
|
|
155
|
+
this.terminate(job, { status: "done", overall: "failed" }, {
|
|
156
|
+
packageId: full.packageId,
|
|
157
|
+
version: full.version,
|
|
158
|
+
overall: "failed",
|
|
159
|
+
instances: [],
|
|
160
|
+
next: `The deploy exceeded its deadline and was abandoned; a step may still have been in ` +
|
|
161
|
+
`flight. Nothing was rolled back. Re-run \`openclaw senpi deploy -p '${full.packageDir}'\` — ` +
|
|
162
|
+
`reconciliation adopts what exists.`,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}, GRACE_MS);
|
|
166
|
+
job.grace.unref?.();
|
|
167
|
+
}, capMs);
|
|
168
|
+
job.watchdog.unref?.();
|
|
169
|
+
// Deferred a microtask so `start` stays pure bookkeeping: a synchronously throwing `run`
|
|
170
|
+
// becomes a journaled terminal state like any other failure, never an exception at the caller.
|
|
171
|
+
void Promise.resolve()
|
|
172
|
+
.then(() => run(handle))
|
|
173
|
+
.then((report) => this.terminate(job, { status: "done", overall: report.overall }, report), (err) => {
|
|
174
|
+
if (job.terminated)
|
|
175
|
+
return; // already force-terminated; do not narrate past the terminal line
|
|
176
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
177
|
+
this.recordEvent(job, { type: "job", detail: `deploy failed: ${message}` });
|
|
178
|
+
this.terminate(job, { status: "done", overall: "failed" }, {
|
|
179
|
+
packageId: full.packageId,
|
|
180
|
+
version: full.version,
|
|
181
|
+
overall: "failed",
|
|
182
|
+
instances: [],
|
|
183
|
+
next: `The deploy died mid-flight: ${message}. Nothing was rolled back. ` +
|
|
184
|
+
`Re-run \`openclaw senpi deploy -p '${full.packageDir}'\` — reconciliation adopts what exists.`,
|
|
185
|
+
});
|
|
186
|
+
});
|
|
187
|
+
return full;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* The running job (from memory) when it is the one being asked about — by id or by omission —
|
|
191
|
+
* else the newest journal, or the named one. Unknown id → null.
|
|
192
|
+
*
|
|
193
|
+
* The live check MUST come first even for an explicit id: a
|
|
194
|
+
* running job's journal has no terminal line yet, so falling through to {@link readJournal}
|
|
195
|
+
* would render it `interrupted` — a state only the boot scan may ever assign, and a flat lie
|
|
196
|
+
* about a job that is actively moving money.
|
|
197
|
+
*/
|
|
198
|
+
async status(deployId) {
|
|
199
|
+
// A caller's id is joined into a path by `journalPath`, so it is shape-checked against the same
|
|
200
|
+
// pattern `journalIds` applies to what it reads back. An unknown id already answers "no such
|
|
201
|
+
// deploy"; this is what keeps `../…` from being READ before answering it.
|
|
202
|
+
if (deployId !== undefined && !DEPLOY_ID_RE.test(deployId))
|
|
203
|
+
return null;
|
|
204
|
+
if (this.live && (!deployId || deployId === this.live.meta.deployId)) {
|
|
205
|
+
return {
|
|
206
|
+
meta: this.live.meta,
|
|
207
|
+
state: { status: "running", phase: this.live.phase, stopRequested: this.live.stopRequested },
|
|
208
|
+
events: [...this.live.events],
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
const id = deployId ?? this.newestJournalId();
|
|
212
|
+
if (!id)
|
|
213
|
+
return null;
|
|
214
|
+
return this.readJournal(id);
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* C2 carve-out: the ONE sanctioned journal read that feeds the orchestrator — and only as a
|
|
218
|
+
* LOOKUP KEY. Every id returned here is verified against the live backend (strategy_list
|
|
219
|
+
* strategyIds:[…]) before reconcile acts on it; the journal never decides anything.
|
|
220
|
+
*
|
|
221
|
+
* It exists because a create whose custom name the backend rejected lands under a
|
|
222
|
+
* backend-assigned name, which the reconcile name match can never find again: without the key,
|
|
223
|
+
* a re-run creates and funds a second wallet beside the first.
|
|
224
|
+
*
|
|
225
|
+
* Returns instance name → journaled created-strategy ids, newest journals first.
|
|
226
|
+
*
|
|
227
|
+
* `deployId` scopes the answer to ONE journal. Reconcile deliberately passes nothing: every
|
|
228
|
+
* journal of the package is a legitimate key source there, because any of them may hold the id
|
|
229
|
+
* of the wallet a re-run must adopt instead of funding a second one (D-9). The interrupted RENDER
|
|
230
|
+
* passes its own id, because it answers a narrower question — what did THIS job leave behind —
|
|
231
|
+
* and a still-live wallet from an earlier completed deploy is not that. Rendering it there would
|
|
232
|
+
* offer to close a strategy that is running exactly as intended.
|
|
233
|
+
*/
|
|
234
|
+
priorCreatedStrategyIds(packageId, deployId) {
|
|
235
|
+
const byInstance = new Map();
|
|
236
|
+
for (const id of this.journalIdsNewestFirst()) {
|
|
237
|
+
if (deployId !== undefined && id !== deployId)
|
|
238
|
+
continue;
|
|
239
|
+
const lines = this.readLines(id);
|
|
240
|
+
const header = lines.find((line) => line.kind === "header");
|
|
241
|
+
if (!header || header.kind !== "header" || header.packageId !== packageId)
|
|
242
|
+
continue;
|
|
243
|
+
for (const line of lines) {
|
|
244
|
+
// ONLY the create-key event. Narrowing the shape keeps the carve-out explicit: any future
|
|
245
|
+
// job event that happens to carry an instance + strategyId must not silently become a
|
|
246
|
+
// reconcile input.
|
|
247
|
+
if (line.kind !== "event" || line.type !== "job" || line.phase !== "create")
|
|
248
|
+
continue;
|
|
249
|
+
const instance = line.evidence?.instance;
|
|
250
|
+
const strategyId = line.evidence?.strategyId;
|
|
251
|
+
if (typeof instance !== "string" || typeof strategyId !== "string")
|
|
252
|
+
continue;
|
|
253
|
+
const list = byInstance.get(instance) ?? [];
|
|
254
|
+
if (!list.includes(strategyId))
|
|
255
|
+
list.push(strategyId);
|
|
256
|
+
byInstance.set(instance, list);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
return byInstance;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* The second D-9 lookup input off the SAME create-key event: strategyId → the amount that create
|
|
263
|
+
* call asked the backend to fund.
|
|
264
|
+
*
|
|
265
|
+
* Narrower than {@link priorCreatedStrategyIds} in what it may do, and deliberately so. It feeds
|
|
266
|
+
* exactly one thing — the report's funded-vs-requested comparison — and only the REQUESTED half
|
|
267
|
+
* of it; the funded half is always re-read live from `strategy_list` on the current run. No
|
|
268
|
+
* orchestration branch reads it, so a stale, missing or wrong entry costs a warn, never an action.
|
|
269
|
+
*
|
|
270
|
+
* It exists because the ask lives only in the memory of the run that funded. A deploy that funds
|
|
271
|
+
* $60 of a $500 ask and dies before assembling its report resumes as an adoption, and without
|
|
272
|
+
* this the shortfall is simply never mentioned again.
|
|
273
|
+
*
|
|
274
|
+
* Newest journal wins on a repeated strategyId — the ask that most recently produced that wallet.
|
|
275
|
+
*/
|
|
276
|
+
priorRequestedFunding(packageId) {
|
|
277
|
+
const byStrategy = new Map();
|
|
278
|
+
for (const id of this.journalIdsNewestFirst()) {
|
|
279
|
+
const lines = this.readLines(id);
|
|
280
|
+
const header = lines.find((line) => line.kind === "header");
|
|
281
|
+
if (!header || header.kind !== "header" || header.packageId !== packageId)
|
|
282
|
+
continue;
|
|
283
|
+
for (const line of lines) {
|
|
284
|
+
if (line.kind !== "event" || line.type !== "job" || line.phase !== "create")
|
|
285
|
+
continue;
|
|
286
|
+
const strategyId = line.evidence?.strategyId;
|
|
287
|
+
const requested = line.evidence?.requested;
|
|
288
|
+
if (typeof strategyId !== "string" || typeof requested !== "number")
|
|
289
|
+
continue;
|
|
290
|
+
if (!(requested > 0) || byStrategy.has(strategyId))
|
|
291
|
+
continue;
|
|
292
|
+
byStrategy.set(strategyId, requested);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return byStrategy;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Boot janitor: stamp every unterminated journal `interrupted`, then prune to
|
|
299
|
+
* the newest `keep`. Discovery is pull-based — the next `openclaw senpi deploy status`
|
|
300
|
+
* renders the interruption; nothing is resumed or notified here.
|
|
301
|
+
*/
|
|
302
|
+
async scanOnBoot() {
|
|
303
|
+
if (!existsSync(this.deploysDir))
|
|
304
|
+
return;
|
|
305
|
+
for (const id of this.journalIds()) {
|
|
306
|
+
const lines = this.readLines(id);
|
|
307
|
+
if (lines.length === 0 || lines.some((line) => line.kind === "terminal"))
|
|
308
|
+
continue;
|
|
309
|
+
this.append(id, {
|
|
310
|
+
kind: "terminal",
|
|
311
|
+
state: { status: "interrupted" },
|
|
312
|
+
endedAt: new Date(this.now()).toISOString(),
|
|
313
|
+
});
|
|
314
|
+
this.log(`deploy ${id} interrupted by restart — surfaced on next openclaw senpi deploy status`);
|
|
315
|
+
}
|
|
316
|
+
this.prune();
|
|
317
|
+
}
|
|
318
|
+
// ── internals ───────────────────────────────────────────────────────────────
|
|
319
|
+
recordEvent(job, body) {
|
|
320
|
+
job.seq += 1;
|
|
321
|
+
const event = { ...body, seq: job.seq, ts: new Date(this.now()).toISOString() };
|
|
322
|
+
if (body.type === "job" && body.phase)
|
|
323
|
+
job.phase = body.phase;
|
|
324
|
+
job.events.push(event);
|
|
325
|
+
if (job.events.length > EVENT_TAIL_MAX)
|
|
326
|
+
job.events.splice(0, job.events.length - EVENT_TAIL_MAX);
|
|
327
|
+
this.append(job.meta.deployId, { kind: "event", ...event });
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Close a job out. The slot is released in `finally`: a journal-write failure is narration lost,
|
|
331
|
+
* never a wedged single-flight slot that would refuse every future deploy with a stale
|
|
332
|
+
* `E_DEPLOY_IN_PROGRESS`.
|
|
333
|
+
*/
|
|
334
|
+
terminate(job, state, report) {
|
|
335
|
+
// Idempotent: after a force-terminate the abandoned `run` promise settles later and calls in
|
|
336
|
+
// again. `readJournal` already tolerates the second terminal line (first wins), but the deploy
|
|
337
|
+
// METRIC does not — without this guard one wedged deploy counts twice, under two different
|
|
338
|
+
// outcomes, on precisely the path where the count matters most.
|
|
339
|
+
if (job.terminated)
|
|
340
|
+
return;
|
|
341
|
+
job.terminated = true;
|
|
342
|
+
try {
|
|
343
|
+
if (job.watchdog)
|
|
344
|
+
clearTimeout(job.watchdog);
|
|
345
|
+
if (job.grace)
|
|
346
|
+
clearTimeout(job.grace);
|
|
347
|
+
// M6: `refused` is recorded as an outcome like any other, with its taxonomy code — a refusal
|
|
348
|
+
// is a correct answer, never an error.
|
|
349
|
+
const refusalCode = firstRefusalCode(report);
|
|
350
|
+
recordDeploy({
|
|
351
|
+
outcome: state.status === "done" ? state.overall : state.status,
|
|
352
|
+
...(refusalCode ? { refusalCode } : {}),
|
|
353
|
+
durationSec: Math.max(0, (this.now() - Date.parse(job.meta.startedAt)) / 1000),
|
|
354
|
+
});
|
|
355
|
+
this.append(job.meta.deployId, {
|
|
356
|
+
kind: "terminal",
|
|
357
|
+
state,
|
|
358
|
+
endedAt: new Date(this.now()).toISOString(),
|
|
359
|
+
report,
|
|
360
|
+
});
|
|
361
|
+
this.prune();
|
|
362
|
+
}
|
|
363
|
+
finally {
|
|
364
|
+
if (this.live === job)
|
|
365
|
+
this.live = null;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
journalPath(deployId) {
|
|
369
|
+
return join(this.deploysDir, `${deployId}.jsonl`);
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Append one journal line. **Never throws**: the journal is narration, and the narration-only
|
|
373
|
+
* rule cuts both ways — a full or read-only state volume must not be able to change what a
|
|
374
|
+
* deploy does (or wedge its slot). A failed write is logged and the line is lost.
|
|
375
|
+
*/
|
|
376
|
+
append(deployId, line) {
|
|
377
|
+
try {
|
|
378
|
+
appendFileSync(this.journalPath(deployId), `${JSON.stringify(line)}\n`);
|
|
379
|
+
}
|
|
380
|
+
catch (err) {
|
|
381
|
+
this.log(`deploy ${deployId}: journal write failed (${errorText(err)}) — narration lost, deploy unaffected`);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
journalIds() {
|
|
385
|
+
if (!existsSync(this.deploysDir))
|
|
386
|
+
return [];
|
|
387
|
+
return readdirSync(this.deploysDir)
|
|
388
|
+
.filter((name) => name.endsWith(".jsonl"))
|
|
389
|
+
.map((name) => name.slice(0, -".jsonl".length))
|
|
390
|
+
.filter((id) => DEPLOY_ID_RE.test(id));
|
|
391
|
+
}
|
|
392
|
+
readLines(deployId) {
|
|
393
|
+
const path = this.journalPath(deployId);
|
|
394
|
+
if (!existsSync(path))
|
|
395
|
+
return [];
|
|
396
|
+
const lines = [];
|
|
397
|
+
for (const raw of readFileSync(path, "utf8").split("\n")) {
|
|
398
|
+
if (raw.trim() === "")
|
|
399
|
+
continue;
|
|
400
|
+
try {
|
|
401
|
+
lines.push(JSON.parse(raw));
|
|
402
|
+
}
|
|
403
|
+
catch {
|
|
404
|
+
// A torn last line (killed mid-append) is narration lost, not state lost.
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
return lines;
|
|
408
|
+
}
|
|
409
|
+
/** Journals newest-first by header `startedAt` — the only ordering the controller trusts. */
|
|
410
|
+
journalIdsNewestFirst() {
|
|
411
|
+
return this.journalIds()
|
|
412
|
+
.map((id) => {
|
|
413
|
+
const header = this.readLines(id).find((line) => line.kind === "header");
|
|
414
|
+
return { id, startedAt: header?.startedAt ?? "" };
|
|
415
|
+
})
|
|
416
|
+
.sort((a, b) => (a.startedAt < b.startedAt ? 1 : a.startedAt > b.startedAt ? -1 : 0))
|
|
417
|
+
.map((entry) => entry.id);
|
|
418
|
+
}
|
|
419
|
+
newestJournalId() {
|
|
420
|
+
return this.journalIdsNewestFirst()[0] ?? null;
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* Journal → snapshot. A journal with no terminal line IS an interrupted job.
|
|
424
|
+
*
|
|
425
|
+
* The FIRST terminal line wins. That ordering is load-bearing: when the deadline watchdog
|
|
426
|
+
* force-terminates a wedged job, the zombie `run` promise may settle later and append a second
|
|
427
|
+
* terminal line — the watchdog's verdict is the true one, and reading the first keeps it.
|
|
428
|
+
*/
|
|
429
|
+
readJournal(deployId) {
|
|
430
|
+
const lines = this.readLines(deployId);
|
|
431
|
+
const header = lines.find((line) => line.kind === "header");
|
|
432
|
+
if (!header)
|
|
433
|
+
return null;
|
|
434
|
+
const { kind: _kind, ...meta } = header;
|
|
435
|
+
const events = lines
|
|
436
|
+
.filter((line) => line.kind === "event")
|
|
437
|
+
.map(({ kind: _eventKind, ...event }) => event);
|
|
438
|
+
const terminal = lines.find((line) => line.kind === "terminal");
|
|
439
|
+
return {
|
|
440
|
+
meta,
|
|
441
|
+
state: terminal?.state ?? { status: "interrupted" },
|
|
442
|
+
events: events.slice(-EVENT_TAIL_MAX),
|
|
443
|
+
...(terminal?.report ? { report: terminal.report } : {}),
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
/** Retain the newest `keep` journals. Runs on every terminate and on boot, so a long-lived
|
|
447
|
+
* gateway does not grow `deploys/` without bound between restarts. Never throws.
|
|
448
|
+
*
|
|
449
|
+
* Unlocked by design: prune and every journal read are fully synchronous on one thread, so they
|
|
450
|
+
* cannot interleave in-process; a torn last line is already tolerated by `readLines`. Do not add
|
|
451
|
+
* locking without a cross-process caller. */
|
|
452
|
+
prune() {
|
|
453
|
+
try {
|
|
454
|
+
const stale = this.journalIdsNewestFirst().slice(this.keep);
|
|
455
|
+
for (const id of stale)
|
|
456
|
+
rmSync(this.journalPath(id), { force: true });
|
|
457
|
+
}
|
|
458
|
+
catch (err) {
|
|
459
|
+
this.log(`deploy journal prune failed (${errorText(err)}) — harmless, retried on the next terminate`);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
//# sourceMappingURL=job.js.map
|