@theford/runly 0.0.0-stage → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +157 -2
- package/dist/cli/client.d.ts +80 -0
- package/dist/cli/client.js +294 -0
- package/dist/cli/client.js.map +1 -0
- package/dist/cli/daemon-unit.d.ts +58 -0
- package/dist/cli/daemon-unit.js +289 -0
- package/dist/cli/daemon-unit.js.map +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +1482 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/install.d.ts +9 -0
- package/dist/cli/install.js +53 -0
- package/dist/cli/install.js.map +1 -0
- package/dist/cli/logs.d.ts +43 -0
- package/dist/cli/logs.js +232 -0
- package/dist/cli/logs.js.map +1 -0
- package/dist/cli/tether.d.ts +3 -0
- package/dist/cli/tether.js +65 -0
- package/dist/cli/tether.js.map +1 -0
- package/dist/cli/watchdog.d.ts +1 -0
- package/dist/cli/watchdog.js +81 -0
- package/dist/cli/watchdog.js.map +1 -0
- package/dist/core/builds.d.ts +20 -0
- package/dist/core/builds.js +93 -0
- package/dist/core/builds.js.map +1 -0
- package/dist/core/caller-env.d.ts +17 -0
- package/dist/core/caller-env.js +77 -0
- package/dist/core/caller-env.js.map +1 -0
- package/dist/core/diagnostics.d.ts +68 -0
- package/dist/core/diagnostics.js +34 -0
- package/dist/core/diagnostics.js.map +1 -0
- package/dist/core/env-vars.d.ts +35 -0
- package/dist/core/env-vars.js +32 -0
- package/dist/core/env-vars.js.map +1 -0
- package/dist/core/events.d.ts +10 -0
- package/dist/core/events.js +42 -0
- package/dist/core/events.js.map +1 -0
- package/dist/core/exec.d.ts +22 -0
- package/dist/core/exec.js +84 -0
- package/dist/core/exec.js.map +1 -0
- package/dist/core/journal.d.ts +227 -0
- package/dist/core/journal.js +395 -0
- package/dist/core/journal.js.map +1 -0
- package/dist/core/logs.d.ts +56 -0
- package/dist/core/logs.js +259 -0
- package/dist/core/logs.js.map +1 -0
- package/dist/core/manifest.d.ts +292 -0
- package/dist/core/manifest.js +179 -0
- package/dist/core/manifest.js.map +1 -0
- package/dist/core/namespaces.d.ts +7 -0
- package/dist/core/namespaces.js +59 -0
- package/dist/core/namespaces.js.map +1 -0
- package/dist/core/paths.d.ts +31 -0
- package/dist/core/paths.js +65 -0
- package/dist/core/paths.js.map +1 -0
- package/dist/core/policy.d.ts +94 -0
- package/dist/core/policy.js +122 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/ports.d.ts +52 -0
- package/dist/core/ports.js +94 -0
- package/dist/core/ports.js.map +1 -0
- package/dist/core/presets.d.ts +25 -0
- package/dist/core/presets.js +86 -0
- package/dist/core/presets.js.map +1 -0
- package/dist/core/procscan.d.ts +111 -0
- package/dist/core/procscan.js +342 -0
- package/dist/core/procscan.js.map +1 -0
- package/dist/core/retention.d.ts +97 -0
- package/dist/core/retention.js +322 -0
- package/dist/core/retention.js.map +1 -0
- package/dist/core/sleep.d.ts +51 -0
- package/dist/core/sleep.js +99 -0
- package/dist/core/sleep.js.map +1 -0
- package/dist/core/tree-ledger.d.ts +39 -0
- package/dist/core/tree-ledger.js +92 -0
- package/dist/core/tree-ledger.js.map +1 -0
- package/dist/core/types.d.ts +21 -0
- package/dist/core/types.js +6 -0
- package/dist/core/types.js.map +1 -0
- package/dist/core/units.d.ts +20 -0
- package/dist/core/units.js +69 -0
- package/dist/core/units.js.map +1 -0
- package/dist/core/upkeep.d.ts +90 -0
- package/dist/core/upkeep.js +176 -0
- package/dist/core/upkeep.js.map +1 -0
- package/dist/core/util.d.ts +46 -0
- package/dist/core/util.js +168 -0
- package/dist/core/util.js.map +1 -0
- package/dist/core/version.d.ts +39 -0
- package/dist/core/version.js +144 -0
- package/dist/core/version.js.map +1 -0
- package/dist/core/worktree.d.ts +51 -0
- package/dist/core/worktree.js +309 -0
- package/dist/core/worktree.js.map +1 -0
- package/dist/daemon/budget.d.ts +155 -0
- package/dist/daemon/budget.js +288 -0
- package/dist/daemon/budget.js.map +1 -0
- package/dist/daemon/election.d.ts +9 -0
- package/dist/daemon/election.js +140 -0
- package/dist/daemon/election.js.map +1 -0
- package/dist/daemon/engine.d.ts +1703 -0
- package/dist/daemon/engine.js +5305 -0
- package/dist/daemon/engine.js.map +1 -0
- package/dist/daemon/index.d.ts +2 -0
- package/dist/daemon/index.js +412 -0
- package/dist/daemon/index.js.map +1 -0
- package/dist/daemon/proxy.d.ts +108 -0
- package/dist/daemon/proxy.js +615 -0
- package/dist/daemon/proxy.js.map +1 -0
- package/dist/daemon/supervisor.d.ts +101 -0
- package/dist/daemon/supervisor.js +493 -0
- package/dist/daemon/supervisor.js.map +1 -0
- package/dist/drivers/appliances.d.ts +11 -0
- package/dist/drivers/appliances.js +174 -0
- package/dist/drivers/appliances.js.map +1 -0
- package/dist/drivers/datastores.d.ts +144 -0
- package/dist/drivers/datastores.js +775 -0
- package/dist/drivers/datastores.js.map +1 -0
- package/dist/drivers/preview.d.ts +69 -0
- package/dist/drivers/preview.js +605 -0
- package/dist/drivers/preview.js.map +1 -0
- package/dist/drivers/types.d.ts +70 -0
- package/dist/drivers/types.js +13 -0
- package/dist/drivers/types.js.map +1 -0
- package/package.json +63 -4
- package/schema/backlot.schema.json +549 -0
- package/schema/runly.schema.json +556 -0
|
@@ -0,0 +1,1703 @@
|
|
|
1
|
+
import { Journal, type LeaseRow } from '../core/journal.js';
|
|
2
|
+
import { LoadBudget, type Committed } from './budget.js';
|
|
3
|
+
import { type UpkeepStep } from '../core/upkeep.js';
|
|
4
|
+
import { ProxyHub } from './proxy.js';
|
|
5
|
+
import { killGroupVerified } from './supervisor.js';
|
|
6
|
+
import { type BindDiagnostics } from '../core/diagnostics.js';
|
|
7
|
+
import type { Hygiene, LeaseKind } from '../core/types.js';
|
|
8
|
+
/** Streamed bind phases → human progress on stderr (never on the --json stdout). */
|
|
9
|
+
export type Progress = (phase: string) => void;
|
|
10
|
+
/** The answer to `up --data-only` from any client that still sends it (the CLI exits 64 before the daemon). */
|
|
11
|
+
export declare const DATA_ONLY_REMOVED: string;
|
|
12
|
+
export interface UpOptions {
|
|
13
|
+
/**
|
|
14
|
+
* Datastores to RELOAD from their template, each with the preset to load
|
|
15
|
+
* (decision 0034). A datastore not named here keeps whatever it holds.
|
|
16
|
+
*/
|
|
17
|
+
presets?: unknown;
|
|
18
|
+
cwd: string;
|
|
19
|
+
/** Explicit refresh envelope; absent means retain this lease's in-memory inputs. */
|
|
20
|
+
callerEnv?: unknown;
|
|
21
|
+
holder?: string;
|
|
22
|
+
hygiene?: Hygiene;
|
|
23
|
+
kind?: LeaseKind;
|
|
24
|
+
ttlMs?: number;
|
|
25
|
+
/**
|
|
26
|
+
* Services to START, plus their transitive depends_on closure — ADDED to
|
|
27
|
+
* what the environment already runs (decision 0034: `up` is additive and
|
|
28
|
+
* never stops a running service). Empty or omitted is the manifest's default
|
|
29
|
+
* set, which is every service. (Only the internal reset-data rebind adds
|
|
30
|
+
* nothing; it does not come through `up`.)
|
|
31
|
+
*/
|
|
32
|
+
services?: string[];
|
|
33
|
+
/** Removed by decision 0034; a request that still sets it is refused with DATA_ONLY_REMOVED. */
|
|
34
|
+
dataOnly?: boolean;
|
|
35
|
+
/** Run every build, also those whose `when:` inputs are unchanged (decision 0038). */
|
|
36
|
+
rebuild?: boolean;
|
|
37
|
+
/**
|
|
38
|
+
* The CALLER's process, so its lease can be released when it dies.
|
|
39
|
+
* The CLI exits per invocation, so this must be the long-lived agent's pid —
|
|
40
|
+
* supplied via --holder-pid or BACKLOT_HOLDER_PID.
|
|
41
|
+
*/
|
|
42
|
+
holderPid?: number;
|
|
43
|
+
/** Set by the daemon per-request; emits progress frames back to the client. */
|
|
44
|
+
onProgress?: Progress;
|
|
45
|
+
/**
|
|
46
|
+
* Aborted when the requesting client disconnects (the daemon wires it to
|
|
47
|
+
* the response's close). A queued `up` whose caller is gone gives up
|
|
48
|
+
* instead of binding for nobody.
|
|
49
|
+
*/
|
|
50
|
+
signal?: AbortSignal;
|
|
51
|
+
}
|
|
52
|
+
export declare class Engine {
|
|
53
|
+
private readonly reapServiceGroup;
|
|
54
|
+
constructor(reapServiceGroup?: typeof killGroupVerified);
|
|
55
|
+
readonly journal: Journal;
|
|
56
|
+
/** The L4 proxy holding every environment's public ports (decision 0033). */
|
|
57
|
+
readonly proxy: ProxyHub;
|
|
58
|
+
private supervisors;
|
|
59
|
+
private lastSweep;
|
|
60
|
+
private lastRetention;
|
|
61
|
+
private lastGc;
|
|
62
|
+
private lastSweepMono;
|
|
63
|
+
/** waketime (epoch ms) of the last kernel-recorded sleep already pardoned. */
|
|
64
|
+
private lastPardonedWake;
|
|
65
|
+
/** FIFO tickets for capacity waiters, so an early waiter is not starved.
|
|
66
|
+
* PER STACK: capacity is per-stack, so one queue for the whole engine made
|
|
67
|
+
* a waiter on full stack A block stack B's instantly-satisfiable claim. */
|
|
68
|
+
private waitTicket;
|
|
69
|
+
private waiting;
|
|
70
|
+
private poolChain;
|
|
71
|
+
private envChains;
|
|
72
|
+
/**
|
|
73
|
+
* One lock per WORKTREE (keyed by stack id), held around everything runly
|
|
74
|
+
* writes into it: upkeep rules, builds, and a pristine bind clearing the
|
|
75
|
+
* worktree ledger (decision 0032). A worktree has one environment, but
|
|
76
|
+
* `runly warm` runs there with none — and before the first bind creates it —
|
|
77
|
+
* so the environment lock alone cannot serialize them. Ordering rule: an
|
|
78
|
+
* environment lock is always taken BEFORE this one, never after — `warm`
|
|
79
|
+
* takes the environment lock(s) of its stack first, so no cycle exists.
|
|
80
|
+
*/
|
|
81
|
+
private treeChains;
|
|
82
|
+
/** Envs with an operation in flight — the sweeper must not expire/quiesce these. */
|
|
83
|
+
readonly busy: Set<string>;
|
|
84
|
+
/** What is in flight on a busy env, so a deferral can name it. */
|
|
85
|
+
private busyOp;
|
|
86
|
+
/** Never persisted or returned. Lease IDs prevent reuse from inheriting another caller's inputs. */
|
|
87
|
+
private leaseInputs;
|
|
88
|
+
/** Opaque in-memory revisions only; no secret values or hashes in the journal. */
|
|
89
|
+
private appliedInputs;
|
|
90
|
+
private appliedInputSpecs;
|
|
91
|
+
private appliedManifests;
|
|
92
|
+
/**
|
|
93
|
+
* Re-publishing the leases' previews after a restart (decision 0045), per
|
|
94
|
+
* lease id: in flight, or failed with the reason `ctx` reports. In memory:
|
|
95
|
+
* it describes this daemon life; the lease row keeps what to re-publish.
|
|
96
|
+
*/
|
|
97
|
+
private previewRestores;
|
|
98
|
+
/** Settles once every recorded preview was re-published or reported; a restore that outlives a `shutdown` stops what it started. */
|
|
99
|
+
previewsRestored: Promise<void>;
|
|
100
|
+
private inputRevision;
|
|
101
|
+
/** Unsaved proxy activity per env, by port key; flushed throttled and by the sweeper. */
|
|
102
|
+
private activityDirty;
|
|
103
|
+
private activityFlushedAt;
|
|
104
|
+
/** When each running service process was started — a fresh start is activity. */
|
|
105
|
+
private startedAt;
|
|
106
|
+
/** Services stopped for idleness (ps shows them as `idle`). */
|
|
107
|
+
/** The last failed `up` per environment (0.19), for ctx and ps; cleared by a successful one. */
|
|
108
|
+
private readonly bindFailures;
|
|
109
|
+
private idleStopped;
|
|
110
|
+
/**
|
|
111
|
+
* Services of a LEASED environment that crash-looped and were stopped
|
|
112
|
+
* (decision 0039), by env + service: `ps` and `ctx` show them as `failed`
|
|
113
|
+
* with how they ended; a wake skips them; the next `up` starts them again.
|
|
114
|
+
*/
|
|
115
|
+
private failedServices;
|
|
116
|
+
/** In-flight wakes, by env + service, so a burst of connections starts a service once. */
|
|
117
|
+
private waking;
|
|
118
|
+
/** First time a tether was seen dead; it is believed only after the grace. */
|
|
119
|
+
private goneSince;
|
|
120
|
+
/** The manifest each environment's running services were started from (budget accounting). */
|
|
121
|
+
private envManifests;
|
|
122
|
+
/** Appliances runly started in this daemon life, by probe address (budget accounting); pruned when one stops. */
|
|
123
|
+
private startedAppliances;
|
|
124
|
+
/** When each idle-stopped service stopped: a wake soon after skips the CPU gate (decision 0036). */
|
|
125
|
+
private idleStoppedAt;
|
|
126
|
+
/** A sweep is running: the next tick is skipped rather than overlapping it. */
|
|
127
|
+
private sweeping;
|
|
128
|
+
/** shutdown() has begun: no sweep, no wake, nothing new. */
|
|
129
|
+
stopping: boolean;
|
|
130
|
+
/** The server-wide load budget (decision 0036). */
|
|
131
|
+
readonly budget: LoadBudget;
|
|
132
|
+
private poolLocked;
|
|
133
|
+
private envLocked;
|
|
134
|
+
private treeLocked;
|
|
135
|
+
/** Hold several environment locks at once, in a fixed (sorted) order. */
|
|
136
|
+
private envsLocked;
|
|
137
|
+
/**
|
|
138
|
+
* The worktree's upkeep ledger, and a committer that writes back only the
|
|
139
|
+
* worktree's half of a merged fingerprint map. MUST be used inside
|
|
140
|
+
* treeLocked: the read-modify-write is what the lock serializes.
|
|
141
|
+
*/
|
|
142
|
+
private treeLedgerSession;
|
|
143
|
+
/** Recovery (decision 0009): reap recorded PIDs from a previous daemon life; hot -> warm. */
|
|
144
|
+
recover(): Promise<void>;
|
|
145
|
+
/**
|
|
146
|
+
* Reclaim runly-spawned processes that no live environment accounts for.
|
|
147
|
+
*
|
|
148
|
+
* A process is an orphan when it carries this state root's tag but its env
|
|
149
|
+
* either no longer exists in the journal, or exists in a state that must have
|
|
150
|
+
* no services running (warm/recycling). Anything belonging to a hot env, or
|
|
151
|
+
* to an env with an operation in flight, is left strictly alone — a bind
|
|
152
|
+
* racing the sweep must not have its dev-server shot out from under it.
|
|
153
|
+
*/
|
|
154
|
+
poolGc(): Promise<{
|
|
155
|
+
supported: boolean;
|
|
156
|
+
reclaimed: Array<{
|
|
157
|
+
pid: number;
|
|
158
|
+
envId: string;
|
|
159
|
+
service: string;
|
|
160
|
+
}>;
|
|
161
|
+
skipped: number;
|
|
162
|
+
}>;
|
|
163
|
+
/**
|
|
164
|
+
* An environment's PRIVATE state — data dir, logs. There is no tree here:
|
|
165
|
+
* services run in the caller's worktree (decision 0032).
|
|
166
|
+
*/
|
|
167
|
+
private envDirs;
|
|
168
|
+
private createEnv;
|
|
169
|
+
/**
|
|
170
|
+
* A public port nobody holds: inside the public block, recorded on no
|
|
171
|
+
* environment row, not one this daemon listens on, and probed free (which
|
|
172
|
+
* also skips a port a tunnel or another process binds on any address).
|
|
173
|
+
*/
|
|
174
|
+
private allocPublicPort;
|
|
175
|
+
/**
|
|
176
|
+
* A fresh internal port for one start of env/key (decision 0033), reserved
|
|
177
|
+
* in the proxy at once. The probe awaits, so another start can be probing
|
|
178
|
+
* the same port meanwhile: the reservation is re-checked after it and taken
|
|
179
|
+
* with no await in between. Released when the target goes down (a stop, a
|
|
180
|
+
* failed start, a settle) or is owned by the target once it is up.
|
|
181
|
+
*/
|
|
182
|
+
private allocInternalPort;
|
|
183
|
+
/**
|
|
184
|
+
* Make the proxy hold every public port of `env` (decision 0033). Idempotent.
|
|
185
|
+
*
|
|
186
|
+
* A recorded port is kept whenever it can be: the whole point is that it is
|
|
187
|
+
* stable. It moves only when it cannot be held — it lies outside the public
|
|
188
|
+
* block (a journal from runly 0.13, whose ports came from the OS ephemeral
|
|
189
|
+
* range), or another process took it while no listener held it (the daemon
|
|
190
|
+
* was down). A move is saved to the journal at once and reported; returns
|
|
191
|
+
* one line per moved port, empty when nothing moved.
|
|
192
|
+
*/
|
|
193
|
+
private ensureProxies;
|
|
194
|
+
/** One atomic claim attempt — MUST run under the pool lock. */
|
|
195
|
+
private tryClaim;
|
|
196
|
+
/**
|
|
197
|
+
* Does the machine-wide ceiling refuse a NEW environment right now? Named
|
|
198
|
+
* explicitly because a refusal must quote the cap that bound (#47). There
|
|
199
|
+
* is no per-stack cap (one environment per worktree, decision 0032) and no
|
|
200
|
+
* data-only cap (decision 0034: a database without an application is a
|
|
201
|
+
* `runly db` copy, which is not an environment). MUST run under the pool lock.
|
|
202
|
+
*/
|
|
203
|
+
private capacityBinding;
|
|
204
|
+
/**
|
|
205
|
+
* Cold, unleased environments that can be given up to free a MACHINE-WIDE
|
|
206
|
+
* slot, least-recently-used first.
|
|
207
|
+
*
|
|
208
|
+
* Why eviction rather than just not counting them (#46's own suggestion): the
|
|
209
|
+
* caps gate environment CREATION only — rebinding an existing environment is
|
|
210
|
+
* never capacity-checked — so the row count is what bounds worst-case
|
|
211
|
+
* concurrent load. Excluding cold rows from the count would leave nothing to
|
|
212
|
+
* stop N of them being rebound hot at once. Giving the slot back for real
|
|
213
|
+
* keeps the ceiling meaningful, and costs the least-recently-used environment
|
|
214
|
+
* one cold provision the next time it is bound.
|
|
215
|
+
*
|
|
216
|
+
* Unleased + not busy + idle past IDLE_TTL qualifies. A recently released
|
|
217
|
+
* environment is the warm pool doing its job, so it is never taken — if
|
|
218
|
+
* nothing is that idle the caller gets a refusal naming the cap that actually
|
|
219
|
+
* bound, which is still strictly better than the lockout.
|
|
220
|
+
*
|
|
221
|
+
* Deliberately NOT restricted to `warm`. Idle past IDLE_TTL is precisely what
|
|
222
|
+
* the sweeper quiesces, but it only looks every BACKLOT_SWEEP_MS (15s by
|
|
223
|
+
* default) — so an environment released and abandoned a moment ago is still
|
|
224
|
+
* `hot` while already being condemned. Requiring `warm` meant that for a whole
|
|
225
|
+
* sweep interval the caller was refused with "waiting will not help", when the
|
|
226
|
+
* next sweep would in fact have made this environment evictable. Taking a
|
|
227
|
+
* condemned `hot` environment is the same trade one step further, and
|
|
228
|
+
* recycleOne stops its services through the ordinary teardown.
|
|
229
|
+
*/
|
|
230
|
+
private evictionCandidates;
|
|
231
|
+
/**
|
|
232
|
+
* Free a machine-wide slot by giving up the least-recently-used cold
|
|
233
|
+
* environment. Returns the id evicted, or null if nothing qualified.
|
|
234
|
+
*
|
|
235
|
+
* MUST NOT hold the pool lock: recycleOne claims under it (the chain is not
|
|
236
|
+
* re-entrant), and the teardown that follows deletes a tree, which the pool
|
|
237
|
+
* lock must never wait on. claimForTeardown re-reads and refuses anything
|
|
238
|
+
* leased or busy, so a candidate chosen here and leased in the gap is declined
|
|
239
|
+
* rather than taken — never take an environment somebody else is holding (#40).
|
|
240
|
+
*/
|
|
241
|
+
private evictForMachineCapacity;
|
|
242
|
+
/**
|
|
243
|
+
* Is some environment about to become claimable on its own?
|
|
244
|
+
*
|
|
245
|
+
* `busy` is the one that matters here and was missed: the idle quiesce runs
|
|
246
|
+
* under the environment lock, which marks it busy, and decision 0021 keeps its
|
|
247
|
+
* mid-quiesce state as plain `hot` on purpose. So the sweeper reclaiming heat
|
|
248
|
+
* from the only candidate made it briefly unevictable — and the machine-wide
|
|
249
|
+
* refusal, which tells the caller that waiting cannot help, fired on a
|
|
250
|
+
* condition that clears in milliseconds. It reproduced on the macOS runner,
|
|
251
|
+
* where teardown is slower (services get SIGTERM, then a verified SIGKILL).
|
|
252
|
+
*
|
|
253
|
+
* The other three are transient for the reasons the per-stack branch already
|
|
254
|
+
* gives: the sweeper reaps them and frees capacity.
|
|
255
|
+
*/
|
|
256
|
+
private transientlyUnclaimable;
|
|
257
|
+
/**
|
|
258
|
+
* Why this environment could not be given up, per environment.
|
|
259
|
+
*
|
|
260
|
+
* It used to say "too recent to evict" for every unleased row, which is the
|
|
261
|
+
* defect #47 was about in miniature: a message that names one cause for four
|
|
262
|
+
* different situations sends the reader after the wrong one. (It cost an hour
|
|
263
|
+
* here, diagnosing a CI failure whose real cause was `busy`.)
|
|
264
|
+
*/
|
|
265
|
+
private notEvictableBecause;
|
|
266
|
+
/**
|
|
267
|
+
* Is the pool full of environments whose leases outlast our whole wait?
|
|
268
|
+
*
|
|
269
|
+
* If so, queueing cannot possibly succeed, and reporting "waited 60s" blames a
|
|
270
|
+
* timing problem that does not exist. MUST run under the pool lock.
|
|
271
|
+
*/
|
|
272
|
+
private structuralCapacityBlock;
|
|
273
|
+
/**
|
|
274
|
+
* Why this worktree's own environment cannot be claimed, if that is what is
|
|
275
|
+
* in the way (decision 0032: one environment per worktree, so a second
|
|
276
|
+
* holder waits for it instead of getting another).
|
|
277
|
+
*
|
|
278
|
+
* - `undefined`: the worktree has no environment, or it sits unleased and
|
|
279
|
+
* usable — whatever refused the claim is a capacity ceiling, not the
|
|
280
|
+
* worktree.
|
|
281
|
+
* - `null`: it is in the way, but will clear by itself within the wait
|
|
282
|
+
* (a lease that lapses in time, a recycle or reap in progress).
|
|
283
|
+
* - a string: it is leased past the wait window — naming who holds it.
|
|
284
|
+
*/
|
|
285
|
+
private worktreeHold;
|
|
286
|
+
/**
|
|
287
|
+
* One refusal text for both throw sites, quoting real counts and the cap that
|
|
288
|
+
* actually bound.
|
|
289
|
+
*
|
|
290
|
+
* The old messages printed `(${POOL_MAX()}/${POOL_MAX()})` — the cap twice,
|
|
291
|
+
* not a count — so a stack with ZERO environments was told "pool at capacity
|
|
292
|
+
* (6/6)" and pointed at the wrong knob (#47).
|
|
293
|
+
*/
|
|
294
|
+
private capacityRefusal;
|
|
295
|
+
/** Queue at capacity WITHOUT holding the pool lock while sleeping. */
|
|
296
|
+
private acquireEnv;
|
|
297
|
+
private acquireQueued;
|
|
298
|
+
/**
|
|
299
|
+
* The values `{{…}}` resolves to. Everything is PUBLIC (decision 0033): the
|
|
300
|
+
* ports and URLs users, builds, exec and other services see are the proxy's.
|
|
301
|
+
* The one exception is a service's own command line and env, given
|
|
302
|
+
* `own = { key, port }`: there `{{ports.<own key>}}` is the INTERNAL port the
|
|
303
|
+
* service must listen on. `{{public_ports.<key>}}` is always the public one,
|
|
304
|
+
* for a service that has to advertise its own address.
|
|
305
|
+
*/
|
|
306
|
+
private templateCtx;
|
|
307
|
+
private supervisor;
|
|
308
|
+
/**
|
|
309
|
+
* The services a bind must bring up: the named ones plus the transitive
|
|
310
|
+
* `depends_on` closure of each. An unknown name is a manifest work-error,
|
|
311
|
+
* reported like the other service lookups. A depends_on cycle is tolerated
|
|
312
|
+
* here (the visit is closure-guarded) and caught by the start loop's own
|
|
313
|
+
* cycle check.
|
|
314
|
+
*/
|
|
315
|
+
private resolveServiceClosure;
|
|
316
|
+
/**
|
|
317
|
+
* The services `env`'s lease wants up (decision 0034): `up` adds to this set
|
|
318
|
+
* and `down` takes away. Undefined on the row means every declared service,
|
|
319
|
+
* so a service added to the manifest later is included.
|
|
320
|
+
*/
|
|
321
|
+
private desiredServices;
|
|
322
|
+
/** How a wanted set is recorded: undefined for every declared service, else the list (possibly empty). */
|
|
323
|
+
private recordedShape;
|
|
324
|
+
/**
|
|
325
|
+
* The services that use datastore `ds`, read from the manifest: a service
|
|
326
|
+
* uses it when its `run:`, `build:` or `env:` templates `{{datastores.<ds>.…}}`
|
|
327
|
+
* — that is how a service is handed a connection string. `null` when no
|
|
328
|
+
* service references it at all: then runly cannot tell who reads it (a
|
|
329
|
+
* connection string in a config file, a hard-coded path), and the caller
|
|
330
|
+
* must restart every running service to be safe (decision 0034).
|
|
331
|
+
*/
|
|
332
|
+
private datastoreUsers;
|
|
333
|
+
/** The datastores an environment of `stack` provisions: all but the `copies_only` ones (decision 0039). */
|
|
334
|
+
private envDatastores;
|
|
335
|
+
/** `up --preset` and `reset-data --preset` name only datastores an environment has (decision 0039). */
|
|
336
|
+
private refuseCopiesOnlyPresets;
|
|
337
|
+
/**
|
|
338
|
+
* Give back the namespaces of datastores an environment no longer
|
|
339
|
+
* provisions — one made before its datastore became `copies_only` (decision
|
|
340
|
+
* 0039) — by the drop recorded for each. A drop that fails stays on the row
|
|
341
|
+
* for teardown to retry.
|
|
342
|
+
*/
|
|
343
|
+
private retireEnvDatastores;
|
|
344
|
+
/**
|
|
345
|
+
* Create, keep or reload each datastore in `plan` — all of them at once
|
|
346
|
+
* (0.19): each bakes and restores under its own datastore lock, so three
|
|
347
|
+
* datastores cost the slowest one, not their sum. Every completed restore is
|
|
348
|
+
* journalled at once, so a failure still reports the others truthfully, and
|
|
349
|
+
* the call settles only when every datastore has (the first error wins). A
|
|
350
|
+
* datastore not forced and already present is KEPT — whatever it holds, even
|
|
351
|
+
* a preset the catalog no longer offers (decision 0034: no preset means keep
|
|
352
|
+
* the data, never a silent reset to the default).
|
|
353
|
+
*/
|
|
354
|
+
private prepareDatastores;
|
|
355
|
+
/**
|
|
356
|
+
* What an additive `up` builds and may restart: every service it leaves
|
|
357
|
+
* running for a plain `up`; for `up <service…>` only the named services,
|
|
358
|
+
* their depends_on closure and what it (re)starts because it is not running
|
|
359
|
+
* (0.19). A full rebind still stops and starts everything — its reasons
|
|
360
|
+
* (an upkeep rule ran, the manifest changed, …) concern every service.
|
|
361
|
+
*/
|
|
362
|
+
private bindScope;
|
|
363
|
+
private bindAndStart;
|
|
364
|
+
private bindAndStartInner;
|
|
365
|
+
/** The public ports of `services` hold new connections until their service is ready again. */
|
|
366
|
+
private markStarting;
|
|
367
|
+
/**
|
|
368
|
+
* The order the builds of `names` run in: depends_on waves. A build runs
|
|
369
|
+
* after the builds of what its service depends on (transitively), and the
|
|
370
|
+
* builds of one wave run at once — independent services no longer wait for
|
|
371
|
+
* each other. `build: {serial: true}` runs alone; `builds: serial` on the
|
|
372
|
+
* stack runs every build alone, in manifest order (the behaviour before
|
|
373
|
+
* 0.16.1). The load budget holds the costliest wave's sum (needFor).
|
|
374
|
+
*
|
|
375
|
+
* Builds that write one shared output or take one tool lock belong in
|
|
376
|
+
* `serial`. Two Angular projects are fine in parallel: the CLI's persistent
|
|
377
|
+
* cache is keyed per project (`.angular/cache/<version>/<project>`).
|
|
378
|
+
*/
|
|
379
|
+
private buildWaves;
|
|
380
|
+
/**
|
|
381
|
+
* Run `fn` for each service of `waves`, wave after wave, the members of one
|
|
382
|
+
* wave at once. A failure fails the call once its whole wave has settled —
|
|
383
|
+
* never with a sibling build still running behind the caller's back.
|
|
384
|
+
*/
|
|
385
|
+
private inWaves;
|
|
386
|
+
/** Build `names` (those that declare a build) in waves. MUST run under treeLocked. */
|
|
387
|
+
private runBuilds;
|
|
388
|
+
/**
|
|
389
|
+
* Run one service's build in the worktree. MUST run under treeLocked. Its
|
|
390
|
+
* output goes to `<service>.build.log` in the environment's log directory
|
|
391
|
+
* (decision 0038), replaced by each build; `logDir` is undefined only for a
|
|
392
|
+
* `warm` with no environment.
|
|
393
|
+
*/
|
|
394
|
+
private runServiceBuild;
|
|
395
|
+
/**
|
|
396
|
+
* Whether a service's build must run now (decision 0038), and the key a
|
|
397
|
+
* successful run is recorded under. A `when:` build runs only when its
|
|
398
|
+
* inputs changed since its last successful build in this worktree; a
|
|
399
|
+
* string build always runs on a bind, and on a wake only when it never
|
|
400
|
+
* succeeded here (a wake resumes what ran; `up` applies changes).
|
|
401
|
+
*/
|
|
402
|
+
private buildDecision;
|
|
403
|
+
private decideBuild;
|
|
404
|
+
/** Build one service if its build is due; record a success. MUST run under treeLocked. */
|
|
405
|
+
private buildService;
|
|
406
|
+
/** Would this service's build run now? (Recorded in `decided` for the build that follows.) */
|
|
407
|
+
private buildWillRun;
|
|
408
|
+
/**
|
|
409
|
+
* A stat print of a service's declared outputs, as the build ledger keeps
|
|
410
|
+
* it: undefined when it declares none, '' when none exist on disk.
|
|
411
|
+
*/
|
|
412
|
+
private outputsPrint;
|
|
413
|
+
/** The upkeep rules' output of this bind, in `upkeep.build.log` (decision 0038), begun by the first rule that runs. */
|
|
414
|
+
private upkeepLog;
|
|
415
|
+
/**
|
|
416
|
+
* Ensure the stack's appliances (decision 0018), remembering the ones runly
|
|
417
|
+
* started for the budget. A started one is counted as running from now on,
|
|
418
|
+
* so its share leaves the caller's reservation.
|
|
419
|
+
*/
|
|
420
|
+
private ensureAppliances;
|
|
421
|
+
/**
|
|
422
|
+
* Forget appliances that are no longer up (stopped by hand, crashed, a
|
|
423
|
+
* reboot): their cost stayed committed for the rest of the daemon's life.
|
|
424
|
+
* Re-probed every sweep — a TCP connect per appliance runly started.
|
|
425
|
+
*/
|
|
426
|
+
private pruneStartedAppliances;
|
|
427
|
+
/** Declared (or default) run resources of everything runly runs now, across every environment. */
|
|
428
|
+
private committedRunning;
|
|
429
|
+
/**
|
|
430
|
+
* What starting `starts` and building `builds` would cost (decision 0036),
|
|
431
|
+
* item by item: run resources per start, build resources per build that
|
|
432
|
+
* will actually run, and run resources per appliance that is not up.
|
|
433
|
+
*/
|
|
434
|
+
private needFor;
|
|
435
|
+
/**
|
|
436
|
+
* Start `only` (a subset of `active`) in dependency order, readiness-gated,
|
|
437
|
+
* fatal-log fast-fail. The other members of `active` are already running,
|
|
438
|
+
* so they count as started for depends_on — and so does a dependency the
|
|
439
|
+
* lease took `down` on purpose (it is not wanted; the dependent reaches its
|
|
440
|
+
* stable public port). On a failure a bind stops everything (`stop-all`); a
|
|
441
|
+
* wake stops only what it started (`stop-these`). In a LEASED environment a
|
|
442
|
+
* bind's service that fails to boot on its own account (a work-error: it
|
|
443
|
+
* exited, hit a fatal log marker, daemonized) is instead stopped and marked
|
|
444
|
+
* failed like a runtime crash loop (decision 0039, 0.18.1); its dependents
|
|
445
|
+
* are not started, everything else is. Returns those dependents.
|
|
446
|
+
*/
|
|
447
|
+
private startServices;
|
|
448
|
+
/**
|
|
449
|
+
* Who the caller is when it names no holder: its WORKTREE — the directory
|
|
450
|
+
* holding the manifest, found by searching upward — whichever subdirectory
|
|
451
|
+
* the verb runs from. `up` at the root and `ctx --env` from `src/app` are
|
|
452
|
+
* the same caller. A live lease an older runly keyed to a subdirectory is
|
|
453
|
+
* still honoured from that subdirectory until it ends.
|
|
454
|
+
*/
|
|
455
|
+
private callerHolder;
|
|
456
|
+
up(opts: UpOptions): Promise<{
|
|
457
|
+
previewNotice: string | undefined;
|
|
458
|
+
bindDiagnostics: BindDiagnostics;
|
|
459
|
+
stack: string;
|
|
460
|
+
envId: string;
|
|
461
|
+
state: import("../core/types.js").EnvState;
|
|
462
|
+
/**
|
|
463
|
+
* `holderPid`: the process the lease is tethered to (decision 0035) —
|
|
464
|
+
* while it lives the lease does not expire, so `expiresAt` is only the
|
|
465
|
+
* renewal horizon. null for a lease that ends at `expiresAt`.
|
|
466
|
+
*/
|
|
467
|
+
lease: {
|
|
468
|
+
id: string;
|
|
469
|
+
kind: "session";
|
|
470
|
+
hygiene: Hygiene;
|
|
471
|
+
expiresAt: number;
|
|
472
|
+
holderPid: number | null;
|
|
473
|
+
} | null;
|
|
474
|
+
urls: Record<string, string>;
|
|
475
|
+
/** The environment's PUBLIC ports by manifest key (`runly ctx --env` → RUNLY_PORT_<KEY>). */
|
|
476
|
+
ports: {
|
|
477
|
+
[x: string]: number;
|
|
478
|
+
};
|
|
479
|
+
/**
|
|
480
|
+
* The proxy in front of each public port, by port key (decision 0033):
|
|
481
|
+
* state, client→server bytes and the last client byte's time. runly's
|
|
482
|
+
* own readiness probes bypass it and are never counted.
|
|
483
|
+
*/
|
|
484
|
+
proxy: Record<string, import("./proxy.js").ProxyStats>;
|
|
485
|
+
previewUrls: Record<string, string>;
|
|
486
|
+
/**
|
|
487
|
+
* A preview the daemon re-publishes after a restart (decision 0045):
|
|
488
|
+
* `restoring`, or `failed` with the error; null when there is none.
|
|
489
|
+
*/
|
|
490
|
+
previewRestore: {
|
|
491
|
+
error?: string | undefined;
|
|
492
|
+
service: string;
|
|
493
|
+
url: string;
|
|
494
|
+
state: "failed" | "restoring";
|
|
495
|
+
} | null;
|
|
496
|
+
/**
|
|
497
|
+
* Every declared service and where it stands (decision 0034): `running`,
|
|
498
|
+
* `stopped` (wanted, but not running — a quiesce or a daemon restart; the
|
|
499
|
+
* next `up` starts it) or `down` (not wanted: never started, or taken
|
|
500
|
+
* down with `runly down`). `urls` lists the wanted ones.
|
|
501
|
+
*/
|
|
502
|
+
services: Record<string, "down" | "failed" | "running" | "stopped">;
|
|
503
|
+
/** Services that crash-looped and were stopped (decision 0039), with their last exit; the next `up` retries them. */
|
|
504
|
+
failures: Record<string, {
|
|
505
|
+
reason: string;
|
|
506
|
+
exitCode: number | null;
|
|
507
|
+
signal: string | null;
|
|
508
|
+
at: number;
|
|
509
|
+
detail: string;
|
|
510
|
+
hint: string;
|
|
511
|
+
}>;
|
|
512
|
+
/** The last `up` failed and none succeeded since (0.19): what failed; null otherwise. */
|
|
513
|
+
lastUpFailed: {
|
|
514
|
+
at: number | null;
|
|
515
|
+
class: string | null;
|
|
516
|
+
message: string;
|
|
517
|
+
source: string | null;
|
|
518
|
+
hint: string;
|
|
519
|
+
} | null;
|
|
520
|
+
/**
|
|
521
|
+
* `logins` stays the PRIMARY login even for a stack that declares a list, so
|
|
522
|
+
* a consumer reading `ctx.logins.user` is unaffected by the manifest growing
|
|
523
|
+
* more of them; `allLogins` carries the full set. Both are null/omitted when
|
|
524
|
+
* the stack declares none.
|
|
525
|
+
*/
|
|
526
|
+
logins: import("../core/manifest.js").Login | null;
|
|
527
|
+
allLogins: import("../core/manifest.js").Login[];
|
|
528
|
+
/**
|
|
529
|
+
* The manifest's INTERNAL hook, templated and run inside the leased
|
|
530
|
+
* environment — not something to run by hand. It still carries its
|
|
531
|
+
* `{{role}}` placeholder, and run from a worktree it signs with the wrong
|
|
532
|
+
* key, so a token minted that way comes back 401 and reads as a
|
|
533
|
+
* permissions problem. `tokenVia` is the supported path.
|
|
534
|
+
*/
|
|
535
|
+
tokenCommand: string | null;
|
|
536
|
+
tokenVia: string | null;
|
|
537
|
+
/**
|
|
538
|
+
* Every datastore exists for the environment's whole life (decision
|
|
539
|
+
* 0034). `preset` is what it holds now — the last preset restored into
|
|
540
|
+
* it — or null before its first restore. `runly ctx --env` exports it as
|
|
541
|
+
* RUNLY_DATASTORE_<NAME>_PRESET.
|
|
542
|
+
*/
|
|
543
|
+
datastores: {
|
|
544
|
+
[k: string]: {
|
|
545
|
+
url: string;
|
|
546
|
+
ns: string;
|
|
547
|
+
preset: string | null;
|
|
548
|
+
};
|
|
549
|
+
};
|
|
550
|
+
events: import("./supervisor.js").ServiceEvent[];
|
|
551
|
+
}>;
|
|
552
|
+
/**
|
|
553
|
+
* The environment a lease points to. leaseForHolder JOINs envs, so the row
|
|
554
|
+
* existed when the lease was resolved — but a concurrent forced teardown can
|
|
555
|
+
* delete it before this read (deleteEnv is one transaction now, so a torn
|
|
556
|
+
* write can no longer leave a lease naming a deleted row; journals from
|
|
557
|
+
* before that change still can, and the sweeper prunes those). Every verb
|
|
558
|
+
* that asserted `getEnv(lease.envId)!` used to crash with an unclassified
|
|
559
|
+
* TypeError instead of telling the caller what to do about it.
|
|
560
|
+
*/
|
|
561
|
+
private envForLease;
|
|
562
|
+
ctx(cwd: string, holder?: string, envId?: string): {
|
|
563
|
+
stack: string;
|
|
564
|
+
envId: string;
|
|
565
|
+
state: import("../core/types.js").EnvState;
|
|
566
|
+
/**
|
|
567
|
+
* `holderPid`: the process the lease is tethered to (decision 0035) —
|
|
568
|
+
* while it lives the lease does not expire, so `expiresAt` is only the
|
|
569
|
+
* renewal horizon. null for a lease that ends at `expiresAt`.
|
|
570
|
+
*/
|
|
571
|
+
lease: {
|
|
572
|
+
id: string;
|
|
573
|
+
kind: "session";
|
|
574
|
+
hygiene: Hygiene;
|
|
575
|
+
expiresAt: number;
|
|
576
|
+
holderPid: number | null;
|
|
577
|
+
} | null;
|
|
578
|
+
urls: Record<string, string>;
|
|
579
|
+
/** The environment's PUBLIC ports by manifest key (`runly ctx --env` → RUNLY_PORT_<KEY>). */
|
|
580
|
+
ports: {
|
|
581
|
+
[x: string]: number;
|
|
582
|
+
};
|
|
583
|
+
/**
|
|
584
|
+
* The proxy in front of each public port, by port key (decision 0033):
|
|
585
|
+
* state, client→server bytes and the last client byte's time. runly's
|
|
586
|
+
* own readiness probes bypass it and are never counted.
|
|
587
|
+
*/
|
|
588
|
+
proxy: Record<string, import("./proxy.js").ProxyStats>;
|
|
589
|
+
previewUrls: Record<string, string>;
|
|
590
|
+
/**
|
|
591
|
+
* A preview the daemon re-publishes after a restart (decision 0045):
|
|
592
|
+
* `restoring`, or `failed` with the error; null when there is none.
|
|
593
|
+
*/
|
|
594
|
+
previewRestore: {
|
|
595
|
+
error?: string | undefined;
|
|
596
|
+
service: string;
|
|
597
|
+
url: string;
|
|
598
|
+
state: "failed" | "restoring";
|
|
599
|
+
} | null;
|
|
600
|
+
/**
|
|
601
|
+
* Every declared service and where it stands (decision 0034): `running`,
|
|
602
|
+
* `stopped` (wanted, but not running — a quiesce or a daemon restart; the
|
|
603
|
+
* next `up` starts it) or `down` (not wanted: never started, or taken
|
|
604
|
+
* down with `runly down`). `urls` lists the wanted ones.
|
|
605
|
+
*/
|
|
606
|
+
services: Record<string, "down" | "failed" | "running" | "stopped">;
|
|
607
|
+
/** Services that crash-looped and were stopped (decision 0039), with their last exit; the next `up` retries them. */
|
|
608
|
+
failures: Record<string, {
|
|
609
|
+
reason: string;
|
|
610
|
+
exitCode: number | null;
|
|
611
|
+
signal: string | null;
|
|
612
|
+
at: number;
|
|
613
|
+
detail: string;
|
|
614
|
+
hint: string;
|
|
615
|
+
}>;
|
|
616
|
+
/** The last `up` failed and none succeeded since (0.19): what failed; null otherwise. */
|
|
617
|
+
lastUpFailed: {
|
|
618
|
+
at: number | null;
|
|
619
|
+
class: string | null;
|
|
620
|
+
message: string;
|
|
621
|
+
source: string | null;
|
|
622
|
+
hint: string;
|
|
623
|
+
} | null;
|
|
624
|
+
/**
|
|
625
|
+
* `logins` stays the PRIMARY login even for a stack that declares a list, so
|
|
626
|
+
* a consumer reading `ctx.logins.user` is unaffected by the manifest growing
|
|
627
|
+
* more of them; `allLogins` carries the full set. Both are null/omitted when
|
|
628
|
+
* the stack declares none.
|
|
629
|
+
*/
|
|
630
|
+
logins: import("../core/manifest.js").Login | null;
|
|
631
|
+
allLogins: import("../core/manifest.js").Login[];
|
|
632
|
+
/**
|
|
633
|
+
* The manifest's INTERNAL hook, templated and run inside the leased
|
|
634
|
+
* environment — not something to run by hand. It still carries its
|
|
635
|
+
* `{{role}}` placeholder, and run from a worktree it signs with the wrong
|
|
636
|
+
* key, so a token minted that way comes back 401 and reads as a
|
|
637
|
+
* permissions problem. `tokenVia` is the supported path.
|
|
638
|
+
*/
|
|
639
|
+
tokenCommand: string | null;
|
|
640
|
+
tokenVia: string | null;
|
|
641
|
+
/**
|
|
642
|
+
* Every datastore exists for the environment's whole life (decision
|
|
643
|
+
* 0034). `preset` is what it holds now — the last preset restored into
|
|
644
|
+
* it — or null before its first restore. `runly ctx --env` exports it as
|
|
645
|
+
* RUNLY_DATASTORE_<NAME>_PRESET.
|
|
646
|
+
*/
|
|
647
|
+
datastores: {
|
|
648
|
+
[k: string]: {
|
|
649
|
+
url: string;
|
|
650
|
+
ns: string;
|
|
651
|
+
preset: string | null;
|
|
652
|
+
};
|
|
653
|
+
};
|
|
654
|
+
events: import("./supervisor.js").ServiceEvent[];
|
|
655
|
+
};
|
|
656
|
+
/**
|
|
657
|
+
* Where each declared service stands: running, failed (wanted, crash-looped
|
|
658
|
+
* and stopped — decision 0039), stopped (wanted, not running) or down (not
|
|
659
|
+
* wanted).
|
|
660
|
+
*/
|
|
661
|
+
private serviceStates;
|
|
662
|
+
/**
|
|
663
|
+
* The last `up` of `env`, when it failed and none has succeeded since
|
|
664
|
+
* (0.19): what failed, and that `up` redoes it — neither a connection nor a
|
|
665
|
+
* verb starts its services meanwhile. After a daemon restart only the fact
|
|
666
|
+
* survives (the journal's fail streak), not the message.
|
|
667
|
+
*/
|
|
668
|
+
private bindFailureOf;
|
|
669
|
+
/** The failed services of `env` (decision 0039): how each last ended, and the log to read. */
|
|
670
|
+
private serviceFailures;
|
|
671
|
+
/**
|
|
672
|
+
* `runly warm`: run THIS worktree's due upkeep rules and its services'
|
|
673
|
+
* `build:` steps, with no lease and no services (decision 0032). The intended
|
|
674
|
+
* caller is an idle pool slot that was just moved to a new commit
|
|
675
|
+
* (`git checkout <sha> && runly warm`): the installs are done and the build
|
|
676
|
+
* tools' own incremental state is current, so the next agent's bind finds
|
|
677
|
+
* the upkeep fresh and its builds near no-ops.
|
|
678
|
+
*
|
|
679
|
+
* Upkeep rules go through the same worktree ledger a bind reads, so a rule
|
|
680
|
+
* warm ran is a rule the next bind skips. Builds are not recorded anywhere:
|
|
681
|
+
* warm runs every one, and the next bind runs them again — cheaply, because
|
|
682
|
+
* the build tool finds its output current. It holds the environment lock of
|
|
683
|
+
* this worktree (if it has one: a check must not run while its output
|
|
684
|
+
* changes underneath it) and then the worktree lock — the same order every
|
|
685
|
+
* bind uses, so it can wait but never deadlock.
|
|
686
|
+
*
|
|
687
|
+
* What it cannot do without an environment it reports as skipped: a build
|
|
688
|
+
* line that templates an environment's ports or datastores has no values to
|
|
689
|
+
* run with, and `@` built-ins act on an environment's data.
|
|
690
|
+
*/
|
|
691
|
+
warm(cwd: string, onProgress?: Progress): Promise<{
|
|
692
|
+
ok: boolean;
|
|
693
|
+
stack: string;
|
|
694
|
+
root: string;
|
|
695
|
+
steps: (({
|
|
696
|
+
kind: "upkeep";
|
|
697
|
+
} & UpkeepStep) | {
|
|
698
|
+
kind: "build";
|
|
699
|
+
service: string;
|
|
700
|
+
status: "ran" | "skipped" | "failed";
|
|
701
|
+
durationMs: number;
|
|
702
|
+
reason?: string;
|
|
703
|
+
})[];
|
|
704
|
+
failure: null;
|
|
705
|
+
durationMs: number;
|
|
706
|
+
}>;
|
|
707
|
+
resetData(cwd: string, holder?: string, onProgress?: Progress, presets?: unknown): Promise<{
|
|
708
|
+
previewNotice: string | undefined;
|
|
709
|
+
bindDiagnostics: BindDiagnostics;
|
|
710
|
+
stack: string;
|
|
711
|
+
envId: string;
|
|
712
|
+
state: import("../core/types.js").EnvState;
|
|
713
|
+
/**
|
|
714
|
+
* `holderPid`: the process the lease is tethered to (decision 0035) —
|
|
715
|
+
* while it lives the lease does not expire, so `expiresAt` is only the
|
|
716
|
+
* renewal horizon. null for a lease that ends at `expiresAt`.
|
|
717
|
+
*/
|
|
718
|
+
lease: {
|
|
719
|
+
id: string;
|
|
720
|
+
kind: "session";
|
|
721
|
+
hygiene: Hygiene;
|
|
722
|
+
expiresAt: number;
|
|
723
|
+
holderPid: number | null;
|
|
724
|
+
} | null;
|
|
725
|
+
urls: Record<string, string>;
|
|
726
|
+
/** The environment's PUBLIC ports by manifest key (`runly ctx --env` → RUNLY_PORT_<KEY>). */
|
|
727
|
+
ports: {
|
|
728
|
+
[x: string]: number;
|
|
729
|
+
};
|
|
730
|
+
/**
|
|
731
|
+
* The proxy in front of each public port, by port key (decision 0033):
|
|
732
|
+
* state, client→server bytes and the last client byte's time. runly's
|
|
733
|
+
* own readiness probes bypass it and are never counted.
|
|
734
|
+
*/
|
|
735
|
+
proxy: Record<string, import("./proxy.js").ProxyStats>;
|
|
736
|
+
previewUrls: Record<string, string>;
|
|
737
|
+
/**
|
|
738
|
+
* A preview the daemon re-publishes after a restart (decision 0045):
|
|
739
|
+
* `restoring`, or `failed` with the error; null when there is none.
|
|
740
|
+
*/
|
|
741
|
+
previewRestore: {
|
|
742
|
+
error?: string | undefined;
|
|
743
|
+
service: string;
|
|
744
|
+
url: string;
|
|
745
|
+
state: "failed" | "restoring";
|
|
746
|
+
} | null;
|
|
747
|
+
/**
|
|
748
|
+
* Every declared service and where it stands (decision 0034): `running`,
|
|
749
|
+
* `stopped` (wanted, but not running — a quiesce or a daemon restart; the
|
|
750
|
+
* next `up` starts it) or `down` (not wanted: never started, or taken
|
|
751
|
+
* down with `runly down`). `urls` lists the wanted ones.
|
|
752
|
+
*/
|
|
753
|
+
services: Record<string, "down" | "failed" | "running" | "stopped">;
|
|
754
|
+
/** Services that crash-looped and were stopped (decision 0039), with their last exit; the next `up` retries them. */
|
|
755
|
+
failures: Record<string, {
|
|
756
|
+
reason: string;
|
|
757
|
+
exitCode: number | null;
|
|
758
|
+
signal: string | null;
|
|
759
|
+
at: number;
|
|
760
|
+
detail: string;
|
|
761
|
+
hint: string;
|
|
762
|
+
}>;
|
|
763
|
+
/** The last `up` failed and none succeeded since (0.19): what failed; null otherwise. */
|
|
764
|
+
lastUpFailed: {
|
|
765
|
+
at: number | null;
|
|
766
|
+
class: string | null;
|
|
767
|
+
message: string;
|
|
768
|
+
source: string | null;
|
|
769
|
+
hint: string;
|
|
770
|
+
} | null;
|
|
771
|
+
/**
|
|
772
|
+
* `logins` stays the PRIMARY login even for a stack that declares a list, so
|
|
773
|
+
* a consumer reading `ctx.logins.user` is unaffected by the manifest growing
|
|
774
|
+
* more of them; `allLogins` carries the full set. Both are null/omitted when
|
|
775
|
+
* the stack declares none.
|
|
776
|
+
*/
|
|
777
|
+
logins: import("../core/manifest.js").Login | null;
|
|
778
|
+
allLogins: import("../core/manifest.js").Login[];
|
|
779
|
+
/**
|
|
780
|
+
* The manifest's INTERNAL hook, templated and run inside the leased
|
|
781
|
+
* environment — not something to run by hand. It still carries its
|
|
782
|
+
* `{{role}}` placeholder, and run from a worktree it signs with the wrong
|
|
783
|
+
* key, so a token minted that way comes back 401 and reads as a
|
|
784
|
+
* permissions problem. `tokenVia` is the supported path.
|
|
785
|
+
*/
|
|
786
|
+
tokenCommand: string | null;
|
|
787
|
+
tokenVia: string | null;
|
|
788
|
+
/**
|
|
789
|
+
* Every datastore exists for the environment's whole life (decision
|
|
790
|
+
* 0034). `preset` is what it holds now — the last preset restored into
|
|
791
|
+
* it — or null before its first restore. `runly ctx --env` exports it as
|
|
792
|
+
* RUNLY_DATASTORE_<NAME>_PRESET.
|
|
793
|
+
*/
|
|
794
|
+
datastores: {
|
|
795
|
+
[k: string]: {
|
|
796
|
+
url: string;
|
|
797
|
+
ns: string;
|
|
798
|
+
preset: string | null;
|
|
799
|
+
};
|
|
800
|
+
};
|
|
801
|
+
events: import("./supervisor.js").ServiceEvent[];
|
|
802
|
+
}>;
|
|
803
|
+
/**
|
|
804
|
+
* Re-read an environment INSIDE its lock and refuse work on one that is being
|
|
805
|
+
* torn down.
|
|
806
|
+
*
|
|
807
|
+
* Teardown claims the row under the pool lock and then runs slowly outside the
|
|
808
|
+
* env lock, so a request that resolved its lease before the claim can arrive
|
|
809
|
+
* here afterwards and operate on a tree that is about to be deleted (or
|
|
810
|
+
* already is). bindAndStart already re-checks; exec and token
|
|
811
|
+
* phase did not.
|
|
812
|
+
*/
|
|
813
|
+
/**
|
|
814
|
+
* Record that an environment was USED, without extending its lease.
|
|
815
|
+
*
|
|
816
|
+
* Activity is what keeps services awake (decision 0035), so only verbs that
|
|
817
|
+
* USE the environment call this (decision 0039): `up` and `reset-data` (a
|
|
818
|
+
* bind sets the clock itself), `exec`, `token`, `preview` and `down`. The
|
|
819
|
+
* read-only verbs — `ctx`, `ps`, `plan`, `logs`, `status`, `db ls`,
|
|
820
|
+
* `doctor` — do not: an agent polling them is not using a service, and
|
|
821
|
+
* counting them kept idle services running for as long as anyone looked.
|
|
822
|
+
*/
|
|
823
|
+
private touch;
|
|
824
|
+
/** `resumable`: the caller starts idle-stopped services itself (resumeForVerb) instead of being refused. */
|
|
825
|
+
private assertUsable;
|
|
826
|
+
/**
|
|
827
|
+
* What `runly exec` runs with (decision 0040): the environment's variables,
|
|
828
|
+
* with idle-stopped services (and, after a daemon restart, every wanted
|
|
829
|
+
* service) started first and the activity clock touched. The CLI spawns the
|
|
830
|
+
* command itself — inherited stdio, the caller's own environment, the real
|
|
831
|
+
* exit code — so nothing here holds the environment's lock while it runs.
|
|
832
|
+
*/
|
|
833
|
+
execEnv(cwd: string, holder?: string): Promise<{
|
|
834
|
+
envId: string;
|
|
835
|
+
root: string;
|
|
836
|
+
vars: {
|
|
837
|
+
[x: string]: string;
|
|
838
|
+
};
|
|
839
|
+
}>;
|
|
840
|
+
/** A running `runly exec` is activity (decision 0039): the CLI calls this while the command runs. */
|
|
841
|
+
execTouch(envId: string): {
|
|
842
|
+
touched: boolean;
|
|
843
|
+
};
|
|
844
|
+
/** Resolve auth.token with {{role}} and run it in the worktree the environment runs in. */
|
|
845
|
+
token(cwd: string, role: string, holder?: string): Promise<{
|
|
846
|
+
token: string;
|
|
847
|
+
role: string;
|
|
848
|
+
}>;
|
|
849
|
+
/**
|
|
850
|
+
* Where this worktree's environment keeps the logs of `services` (decision
|
|
851
|
+
* 0038) — all of them when none are named. The CLI reads, filters and
|
|
852
|
+
* follows the files itself; the daemon only resolves which ones. With
|
|
853
|
+
* `build`, the build output of each service's last build, and the upkeep
|
|
854
|
+
* output of the last `up` that ran a rule.
|
|
855
|
+
*/
|
|
856
|
+
logsSpec(cwd: string, services: string[], build: boolean, holder?: string): {
|
|
857
|
+
envId: string;
|
|
858
|
+
dir: string;
|
|
859
|
+
build: boolean;
|
|
860
|
+
files: {
|
|
861
|
+
service: string;
|
|
862
|
+
file: string;
|
|
863
|
+
}[];
|
|
864
|
+
};
|
|
865
|
+
/** The pre-0.16 single-service read, kept for RPC clients: the last `lines` lines, without stamps. */
|
|
866
|
+
logs(cwd: string, service: string, lines: number, holder?: string): {
|
|
867
|
+
service: string;
|
|
868
|
+
lines: string;
|
|
869
|
+
};
|
|
870
|
+
/**
|
|
871
|
+
* Hand the environment back. A no-op answer must say WHY.
|
|
872
|
+
*
|
|
873
|
+
* A lease is keyed by holder NAME, which defaults to the worktree path — so
|
|
874
|
+
* releasing from a different directory than the one that bound silently
|
|
875
|
+
* matches nothing. `{released: false}` on its own gave an agent no way to tell
|
|
876
|
+
* that apart from "already released", and the reported behaviour was simply to
|
|
877
|
+
* give up and let the environment linger until its TTL. Name the mismatch, and
|
|
878
|
+
* list who does hold this stack's leases.
|
|
879
|
+
*/
|
|
880
|
+
release(cwd: string, holder?: string): Promise<{
|
|
881
|
+
released: boolean;
|
|
882
|
+
holder: string;
|
|
883
|
+
reason: string;
|
|
884
|
+
otherHolders: string[];
|
|
885
|
+
envId?: undefined;
|
|
886
|
+
} | {
|
|
887
|
+
released: boolean;
|
|
888
|
+
envId: string;
|
|
889
|
+
holder?: undefined;
|
|
890
|
+
reason?: undefined;
|
|
891
|
+
otherHolders?: undefined;
|
|
892
|
+
}>;
|
|
893
|
+
private previewPublisherName;
|
|
894
|
+
private assertPreviewAllowed;
|
|
895
|
+
/**
|
|
896
|
+
* Tagged members of a verified leased preview process group. A publisher's
|
|
897
|
+
* wrapper can fork its tunnel child; excluding only the leader lets a scan
|
|
898
|
+
* select the child and kill the whole leased group. All scan consumers share
|
|
899
|
+
* this classification, using the same state-root-scoped process snapshot.
|
|
900
|
+
*
|
|
901
|
+
* Ownership requires the recorded leader identity AND matching env tags.
|
|
902
|
+
* A stale/reused leader PID, an unreadable identity, or a generic preview:
|
|
903
|
+
* service label cannot exempt another process. Descendants that leave the
|
|
904
|
+
* group (setsid) need a separate ownership mechanism and are not protected.
|
|
905
|
+
*/
|
|
906
|
+
private leasedPreviewPids;
|
|
907
|
+
/**
|
|
908
|
+
* Stop and clear any lease-scoped preview tunnel recorded on this lease.
|
|
909
|
+
*
|
|
910
|
+
* Teardown may NEVER be blocked by a manifest edit: `preview.publisher` is
|
|
911
|
+
* re-read from disk on every call, so a name that resolved at start can be a
|
|
912
|
+
* typo by the time the sweeper gets here — and an unknown name throws a
|
|
913
|
+
* work-error. Thrown from the sweeper's lease loop that aborted every tick,
|
|
914
|
+
* so TTL expiry, dead-holder release and eviction stopped daemon-wide. The
|
|
915
|
+
* teardown path therefore falls back to the default publisher instead.
|
|
916
|
+
*
|
|
917
|
+
* Returns false when the tunnel could NOT be confirmed dead. The record is
|
|
918
|
+
* kept in that case (reapPids' contract): a forgotten preview pid is a
|
|
919
|
+
* public, unauthenticated URL nobody can ever name again.
|
|
920
|
+
*/
|
|
921
|
+
private stopPreviewForLease;
|
|
922
|
+
/**
|
|
923
|
+
* What an operator can actually do about a tunnel that outlived its kill.
|
|
924
|
+
*
|
|
925
|
+
* Pointing at `pool gc` was only ever true on Linux — it is a tag scan, and
|
|
926
|
+
* `procScanSupported()` is Linux-only — which is precisely the platform where
|
|
927
|
+
* the record is NOT recoverable. Name the pid and the public URL instead, so
|
|
928
|
+
* the remedy survives the lease row that is usually about to be deleted.
|
|
929
|
+
*/
|
|
930
|
+
private unreapedPreview;
|
|
931
|
+
/**
|
|
932
|
+
* Release a lease and reap its preview tunnel first.
|
|
933
|
+
*
|
|
934
|
+
* Re-read between the stop and the delete. `release` and the sweeper hold no
|
|
935
|
+
* env lock, and the stop blocks in `killGroupVerified` for up to ~4s — long
|
|
936
|
+
* enough for a concurrent `preview` to take the env lock, publish, and write
|
|
937
|
+
* its pid onto this very row. Deleting the row on the strength of the old
|
|
938
|
+
* snapshot then orphaned the NEW tunnel: no lease names it, `pool gc` skips
|
|
939
|
+
* nothing it can see, and `preview stop` has nothing left to stop. Taking the
|
|
940
|
+
* env lock here instead would nest it inside the pool lock (tryClaim calls
|
|
941
|
+
* this) and stall every claim machine-wide for the length of a bind.
|
|
942
|
+
*/
|
|
943
|
+
private endLease;
|
|
944
|
+
/**
|
|
945
|
+
* Everything about an environment that decides whether publishing `service`
|
|
946
|
+
* is legal, and the local port to publish. Returns that port.
|
|
947
|
+
*
|
|
948
|
+
* One function because it has to run TWICE — once before queueing for the env
|
|
949
|
+
* lock, and again inside it against a re-read row. Every field it reads is one
|
|
950
|
+
* a concurrent bind rewrites.
|
|
951
|
+
*/
|
|
952
|
+
private previewTarget;
|
|
953
|
+
/**
|
|
954
|
+
* Publish every recorded preview again after a restart (decision 0045): same
|
|
955
|
+
* publisher, the settings that pin its address, aimed at the same public
|
|
956
|
+
* port — which the proxy holds again, so the first request wakes the
|
|
957
|
+
* service as usual. A failure is an event and `ctx.previewRestore`, never
|
|
958
|
+
* an error of anyone's verb.
|
|
959
|
+
*/
|
|
960
|
+
private restorePreviews;
|
|
961
|
+
private restorePreview;
|
|
962
|
+
/** The lease's preview is over on purpose: a restart must not bring it back (decision 0045). */
|
|
963
|
+
private forgetPreview;
|
|
964
|
+
previewStart(cwd: string, service: string, holder?: string, ttlMs?: number, httpsPort?: number): Promise<{
|
|
965
|
+
service: string;
|
|
966
|
+
url: string;
|
|
967
|
+
}>;
|
|
968
|
+
previewStop(cwd: string, holder?: string): Promise<{
|
|
969
|
+
stopped: boolean;
|
|
970
|
+
service?: string;
|
|
971
|
+
}>;
|
|
972
|
+
/** Live-probed appliance overview for the stack at cwd. */
|
|
973
|
+
applianceLs(cwd: string): Promise<{
|
|
974
|
+
stack: string;
|
|
975
|
+
appliances: Record<string, {
|
|
976
|
+
probe: string;
|
|
977
|
+
up: boolean;
|
|
978
|
+
startable: boolean;
|
|
979
|
+
stoppable: boolean;
|
|
980
|
+
}>;
|
|
981
|
+
}>;
|
|
982
|
+
/** Ensure one appliance (or all of them) — the same path a bind takes. */
|
|
983
|
+
applianceStart(cwd: string, name?: string): Promise<{
|
|
984
|
+
results: Record<string, string>;
|
|
985
|
+
}>;
|
|
986
|
+
/** Explicit stop — the only path that ever stops an appliance. */
|
|
987
|
+
applianceStop(cwd: string, name: string): Promise<{
|
|
988
|
+
stopped: string;
|
|
989
|
+
}>;
|
|
990
|
+
/**
|
|
991
|
+
* `runly down [service…]` (decision 0034): stop just these services — or,
|
|
992
|
+
* with no names, every service — and keep the lease, the data and the public
|
|
993
|
+
* ports. The proxy keeps listening; connections to a downed service are
|
|
994
|
+
* refused until an `up` brings it back. Dependents of a downed service are
|
|
995
|
+
* NOT stopped ("just those"); they are named in the answer, because they now
|
|
996
|
+
* talk to a port with nothing behind it.
|
|
997
|
+
*/
|
|
998
|
+
down(opts: {
|
|
999
|
+
cwd: string;
|
|
1000
|
+
holder?: string;
|
|
1001
|
+
services: string[];
|
|
1002
|
+
onProgress?: Progress;
|
|
1003
|
+
}): Promise<{
|
|
1004
|
+
stopped: string[];
|
|
1005
|
+
down: string[];
|
|
1006
|
+
dependentsStillRunning: string[];
|
|
1007
|
+
previewNotice: string | undefined;
|
|
1008
|
+
stack: string;
|
|
1009
|
+
envId: string;
|
|
1010
|
+
state: import("../core/types.js").EnvState;
|
|
1011
|
+
/**
|
|
1012
|
+
* `holderPid`: the process the lease is tethered to (decision 0035) —
|
|
1013
|
+
* while it lives the lease does not expire, so `expiresAt` is only the
|
|
1014
|
+
* renewal horizon. null for a lease that ends at `expiresAt`.
|
|
1015
|
+
*/
|
|
1016
|
+
lease: {
|
|
1017
|
+
id: string;
|
|
1018
|
+
kind: "session";
|
|
1019
|
+
hygiene: Hygiene;
|
|
1020
|
+
expiresAt: number;
|
|
1021
|
+
holderPid: number | null;
|
|
1022
|
+
} | null;
|
|
1023
|
+
urls: Record<string, string>;
|
|
1024
|
+
/** The environment's PUBLIC ports by manifest key (`runly ctx --env` → RUNLY_PORT_<KEY>). */
|
|
1025
|
+
ports: {
|
|
1026
|
+
[x: string]: number;
|
|
1027
|
+
};
|
|
1028
|
+
/**
|
|
1029
|
+
* The proxy in front of each public port, by port key (decision 0033):
|
|
1030
|
+
* state, client→server bytes and the last client byte's time. runly's
|
|
1031
|
+
* own readiness probes bypass it and are never counted.
|
|
1032
|
+
*/
|
|
1033
|
+
proxy: Record<string, import("./proxy.js").ProxyStats>;
|
|
1034
|
+
previewUrls: Record<string, string>;
|
|
1035
|
+
/**
|
|
1036
|
+
* A preview the daemon re-publishes after a restart (decision 0045):
|
|
1037
|
+
* `restoring`, or `failed` with the error; null when there is none.
|
|
1038
|
+
*/
|
|
1039
|
+
previewRestore: {
|
|
1040
|
+
error?: string | undefined;
|
|
1041
|
+
service: string;
|
|
1042
|
+
url: string;
|
|
1043
|
+
state: "failed" | "restoring";
|
|
1044
|
+
} | null;
|
|
1045
|
+
/**
|
|
1046
|
+
* Every declared service and where it stands (decision 0034): `running`,
|
|
1047
|
+
* `stopped` (wanted, but not running — a quiesce or a daemon restart; the
|
|
1048
|
+
* next `up` starts it) or `down` (not wanted: never started, or taken
|
|
1049
|
+
* down with `runly down`). `urls` lists the wanted ones.
|
|
1050
|
+
*/
|
|
1051
|
+
services: Record<string, "down" | "failed" | "running" | "stopped">;
|
|
1052
|
+
/** Services that crash-looped and were stopped (decision 0039), with their last exit; the next `up` retries them. */
|
|
1053
|
+
failures: Record<string, {
|
|
1054
|
+
reason: string;
|
|
1055
|
+
exitCode: number | null;
|
|
1056
|
+
signal: string | null;
|
|
1057
|
+
at: number;
|
|
1058
|
+
detail: string;
|
|
1059
|
+
hint: string;
|
|
1060
|
+
}>;
|
|
1061
|
+
/** The last `up` failed and none succeeded since (0.19): what failed; null otherwise. */
|
|
1062
|
+
lastUpFailed: {
|
|
1063
|
+
at: number | null;
|
|
1064
|
+
class: string | null;
|
|
1065
|
+
message: string;
|
|
1066
|
+
source: string | null;
|
|
1067
|
+
hint: string;
|
|
1068
|
+
} | null;
|
|
1069
|
+
/**
|
|
1070
|
+
* `logins` stays the PRIMARY login even for a stack that declares a list, so
|
|
1071
|
+
* a consumer reading `ctx.logins.user` is unaffected by the manifest growing
|
|
1072
|
+
* more of them; `allLogins` carries the full set. Both are null/omitted when
|
|
1073
|
+
* the stack declares none.
|
|
1074
|
+
*/
|
|
1075
|
+
logins: import("../core/manifest.js").Login | null;
|
|
1076
|
+
allLogins: import("../core/manifest.js").Login[];
|
|
1077
|
+
/**
|
|
1078
|
+
* The manifest's INTERNAL hook, templated and run inside the leased
|
|
1079
|
+
* environment — not something to run by hand. It still carries its
|
|
1080
|
+
* `{{role}}` placeholder, and run from a worktree it signs with the wrong
|
|
1081
|
+
* key, so a token minted that way comes back 401 and reads as a
|
|
1082
|
+
* permissions problem. `tokenVia` is the supported path.
|
|
1083
|
+
*/
|
|
1084
|
+
tokenCommand: string | null;
|
|
1085
|
+
tokenVia: string | null;
|
|
1086
|
+
/**
|
|
1087
|
+
* Every datastore exists for the environment's whole life (decision
|
|
1088
|
+
* 0034). `preset` is what it holds now — the last preset restored into
|
|
1089
|
+
* it — or null before its first restore. `runly ctx --env` exports it as
|
|
1090
|
+
* RUNLY_DATASTORE_<NAME>_PRESET.
|
|
1091
|
+
*/
|
|
1092
|
+
datastores: {
|
|
1093
|
+
[k: string]: {
|
|
1094
|
+
url: string;
|
|
1095
|
+
ns: string;
|
|
1096
|
+
preset: string | null;
|
|
1097
|
+
};
|
|
1098
|
+
};
|
|
1099
|
+
events: import("./supervisor.js").ServiceEvent[];
|
|
1100
|
+
}>;
|
|
1101
|
+
/** In-flight creations and drops of database copies; the reaper never touches these. */
|
|
1102
|
+
private dbBusy;
|
|
1103
|
+
/** What `db new` / `db ls` / `ps` show for one copy. */
|
|
1104
|
+
private dbCopyView;
|
|
1105
|
+
/**
|
|
1106
|
+
* Is a recorded holder process gone? One rule for leases and database copies
|
|
1107
|
+
* alike (decision 0034: copies are reaped exactly like environments), so a
|
|
1108
|
+
* grace period added here applies to both.
|
|
1109
|
+
*/
|
|
1110
|
+
private holderGone;
|
|
1111
|
+
/**
|
|
1112
|
+
* `runly db new <datastore>` (decision 0034): a fresh copy of the datastore,
|
|
1113
|
+
* restored from the SAME template the environments use (baked first if it is
|
|
1114
|
+
* missing), outside any environment — no lease, no ports, no services. Many
|
|
1115
|
+
* may exist at once. The row is journalled BEFORE the restore, as
|
|
1116
|
+
* `creating`, so a daemon that dies mid-restore leaves a record the next one
|
|
1117
|
+
* reaps.
|
|
1118
|
+
*/
|
|
1119
|
+
dbNew(opts: {
|
|
1120
|
+
cwd: string;
|
|
1121
|
+
datastore: string;
|
|
1122
|
+
preset?: string;
|
|
1123
|
+
holder?: string;
|
|
1124
|
+
holderPid?: number;
|
|
1125
|
+
onProgress?: Progress;
|
|
1126
|
+
}): Promise<{
|
|
1127
|
+
name: string;
|
|
1128
|
+
datastore: string;
|
|
1129
|
+
preset: string;
|
|
1130
|
+
url: string;
|
|
1131
|
+
ns: string;
|
|
1132
|
+
state: "ready" | "creating" | "dropping";
|
|
1133
|
+
worktree: string;
|
|
1134
|
+
holder: string;
|
|
1135
|
+
holderPid: number | null;
|
|
1136
|
+
/** null = no agent tether was given: the copy lives until its worktree goes or it is dropped. */
|
|
1137
|
+
holderAlive: boolean | null;
|
|
1138
|
+
createdAt: number;
|
|
1139
|
+
}>;
|
|
1140
|
+
/**
|
|
1141
|
+
* `runly db with` records the command it runs against a copy (decision
|
|
1142
|
+
* 0039), so a drop can take that command down first when its CLI died.
|
|
1143
|
+
*/
|
|
1144
|
+
dbAttach(name: string, child: {
|
|
1145
|
+
pid: number;
|
|
1146
|
+
start?: number;
|
|
1147
|
+
pgid?: number;
|
|
1148
|
+
}): {
|
|
1149
|
+
attached: string;
|
|
1150
|
+
};
|
|
1151
|
+
/**
|
|
1152
|
+
* Stop whatever still runs against copy `row` (decision 0039): the command
|
|
1153
|
+
* `runly db with` recorded — its process group when it has its own, else
|
|
1154
|
+
* the verified pid — and, on Linux, every process carrying the copy's tag.
|
|
1155
|
+
* Only a pid whose start time still matches is signalled. Returns how many
|
|
1156
|
+
* processes were stopped.
|
|
1157
|
+
*/
|
|
1158
|
+
private stopCopyUsers;
|
|
1159
|
+
/** Is `path` a copy's own directory under the state root (never anything else)? */
|
|
1160
|
+
private isPrivateDbDir;
|
|
1161
|
+
/**
|
|
1162
|
+
* Drop one copy with what its row recorded — the command (run in the
|
|
1163
|
+
* worktree, or the state root once that is gone) or its private directory.
|
|
1164
|
+
* Needs neither the manifest nor the worktree. A failed drop keeps the row
|
|
1165
|
+
* (`dropping`) with a backoff, so the reaper retries instead of forgetting a
|
|
1166
|
+
* database nothing else can name. Callers hold `dbBusy` for the name.
|
|
1167
|
+
*/
|
|
1168
|
+
private dropDbCopy;
|
|
1169
|
+
/** `runly db drop <name>`: drop one copy now, whoever made it. */
|
|
1170
|
+
dbDrop(name: string): Promise<{
|
|
1171
|
+
dropped: string;
|
|
1172
|
+
}>;
|
|
1173
|
+
/** `runly db ls`: this worktree's copies, or every copy with `all`. */
|
|
1174
|
+
dbLs(cwd: string, all: boolean): {
|
|
1175
|
+
scope: string;
|
|
1176
|
+
copies: {
|
|
1177
|
+
name: string;
|
|
1178
|
+
datastore: string;
|
|
1179
|
+
preset: string;
|
|
1180
|
+
url: string;
|
|
1181
|
+
ns: string;
|
|
1182
|
+
state: "ready" | "creating" | "dropping";
|
|
1183
|
+
worktree: string;
|
|
1184
|
+
holder: string;
|
|
1185
|
+
holderPid: number | null;
|
|
1186
|
+
/** null = no agent tether was given: the copy lives until its worktree goes or it is dropped. */
|
|
1187
|
+
holderAlive: boolean | null;
|
|
1188
|
+
createdAt: number;
|
|
1189
|
+
}[];
|
|
1190
|
+
};
|
|
1191
|
+
/** The caller's worktree root, for the per-worktree views (`ps`, `db ls`). */
|
|
1192
|
+
private callerWorktree;
|
|
1193
|
+
/** Why a copy must go now, or null (decision 0034: reaped exactly like environments; no TTL). */
|
|
1194
|
+
private dbCopyOrphanReason;
|
|
1195
|
+
/** Drop every copy whose holder or worktree is gone. Runs in every sweep and after recovery. */
|
|
1196
|
+
private reapDbCopies;
|
|
1197
|
+
/** Resident memory per (env, service), summed over its tagged processes. Linux only; empty elsewhere. */
|
|
1198
|
+
private rssByService;
|
|
1199
|
+
/**
|
|
1200
|
+
* `runly ps` (decision 0034): one view of what runs for the caller — the
|
|
1201
|
+
* services of this worktree's environment and its database copies — or,
|
|
1202
|
+
* with `all`, for the whole server.
|
|
1203
|
+
*/
|
|
1204
|
+
ps(cwd: string, all: boolean): {
|
|
1205
|
+
scope: string;
|
|
1206
|
+
services: Record<string, unknown>[];
|
|
1207
|
+
databases: {
|
|
1208
|
+
name: string;
|
|
1209
|
+
datastore: string;
|
|
1210
|
+
preset: string;
|
|
1211
|
+
url: string;
|
|
1212
|
+
ns: string;
|
|
1213
|
+
state: "ready" | "creating" | "dropping";
|
|
1214
|
+
worktree: string;
|
|
1215
|
+
holder: string;
|
|
1216
|
+
holderPid: number | null;
|
|
1217
|
+
/** null = no agent tether was given: the copy lives until its worktree goes or it is dropped. */
|
|
1218
|
+
holderAlive: boolean | null;
|
|
1219
|
+
createdAt: number;
|
|
1220
|
+
}[];
|
|
1221
|
+
/** Environments whose last `up` failed (0.19), with what failed. */
|
|
1222
|
+
failedUps: Record<string, unknown>[];
|
|
1223
|
+
/** The server-wide load budget (decision 0036). */
|
|
1224
|
+
budget: {
|
|
1225
|
+
enabled: boolean;
|
|
1226
|
+
memoryBytes: number;
|
|
1227
|
+
cpu: number;
|
|
1228
|
+
committedMemoryBytes: number;
|
|
1229
|
+
committedCpu: number;
|
|
1230
|
+
waiting: number;
|
|
1231
|
+
};
|
|
1232
|
+
};
|
|
1233
|
+
/**
|
|
1234
|
+
* `runly plan [service…]` (decision 0036): what an `up` with these
|
|
1235
|
+
* services would build and start now, what that costs (declared, or the
|
|
1236
|
+
* default, said so), what the box and the budget have free, and whether it
|
|
1237
|
+
* would start now or wait — without doing any of it.
|
|
1238
|
+
*/
|
|
1239
|
+
plan(cwd: string, services: string[], rebuild: boolean, holder?: string): Promise<{
|
|
1240
|
+
stack: string;
|
|
1241
|
+
envId: string | null;
|
|
1242
|
+
items: {
|
|
1243
|
+
kind: "start" | "appliance" | "build";
|
|
1244
|
+
name: string;
|
|
1245
|
+
memoryBytes: number;
|
|
1246
|
+
cpu: number;
|
|
1247
|
+
declared: boolean;
|
|
1248
|
+
skipped: string | null;
|
|
1249
|
+
}[];
|
|
1250
|
+
need: {
|
|
1251
|
+
memoryBytes: number;
|
|
1252
|
+
cpu: number;
|
|
1253
|
+
start: {
|
|
1254
|
+
memoryBytes: number;
|
|
1255
|
+
cpu: number;
|
|
1256
|
+
};
|
|
1257
|
+
build: {
|
|
1258
|
+
memoryBytes: number;
|
|
1259
|
+
cpu: number;
|
|
1260
|
+
};
|
|
1261
|
+
};
|
|
1262
|
+
machine: import("./budget.js").Machine;
|
|
1263
|
+
budget: {
|
|
1264
|
+
enabled: boolean;
|
|
1265
|
+
memoryBytes: number;
|
|
1266
|
+
cpu: number;
|
|
1267
|
+
reserveBytes: number;
|
|
1268
|
+
loadPerCore: number;
|
|
1269
|
+
waitMs: number;
|
|
1270
|
+
};
|
|
1271
|
+
committed: Committed;
|
|
1272
|
+
queue: number;
|
|
1273
|
+
startsNow: boolean;
|
|
1274
|
+
verdict: string;
|
|
1275
|
+
defaultsAssumed: string[];
|
|
1276
|
+
}>;
|
|
1277
|
+
/**
|
|
1278
|
+
* `runly destroy` (decision 0035): everything runly holds for this worktree
|
|
1279
|
+
* goes now — its environment (services, data, preview, ports, lease) and its
|
|
1280
|
+
* database copies. What a worktree pool calls when it takes a worktree back;
|
|
1281
|
+
* the sweeper does the same by itself when the worktree is removed or its
|
|
1282
|
+
* agent's tether dies. The baked templates and the build records stay
|
|
1283
|
+
* (decision 0039; a build's `outputs:` are checked before it is skipped).
|
|
1284
|
+
* Of the upkeep ledger only the rules that declare `outputs:` stay: the pool
|
|
1285
|
+
* usually resets the worktree next (`git clean -fdx`), which removes what
|
|
1286
|
+
* the other rules produced, and nothing else could tell.
|
|
1287
|
+
*/
|
|
1288
|
+
destroy(cwd: string): Promise<{
|
|
1289
|
+
worktree: string;
|
|
1290
|
+
environments: string[];
|
|
1291
|
+
copies: string[];
|
|
1292
|
+
copiesNotDropped: string[];
|
|
1293
|
+
upkeepForgotten: number;
|
|
1294
|
+
}>;
|
|
1295
|
+
/**
|
|
1296
|
+
* `runly pool doctor [--fix]` (decision 0037): what this state root owns
|
|
1297
|
+
* that nothing references any more. Dry run by default; `--fix` removes it.
|
|
1298
|
+
* Only runly's own: files under the state root in runly's naming,
|
|
1299
|
+
* processes carrying this state root's tag, listeners of this daemon, and
|
|
1300
|
+
* server-side namespaces a datastore's `list:` hook reports that carry
|
|
1301
|
+
* runly's naming AND a stack id this state root has a record of.
|
|
1302
|
+
*/
|
|
1303
|
+
poolDoctor(cwd: string, fix: boolean): Promise<{
|
|
1304
|
+
fix: boolean;
|
|
1305
|
+
clean: boolean;
|
|
1306
|
+
findings: {
|
|
1307
|
+
kind: string;
|
|
1308
|
+
what: string;
|
|
1309
|
+
detail: string;
|
|
1310
|
+
fixed?: boolean;
|
|
1311
|
+
error?: string;
|
|
1312
|
+
}[];
|
|
1313
|
+
}>;
|
|
1314
|
+
status(): {
|
|
1315
|
+
pid: number;
|
|
1316
|
+
/**
|
|
1317
|
+
* Who restarts this daemon when it crashes (decision 0039): `systemd` or
|
|
1318
|
+
* `launchd` after `runly daemon install` (the unit sets
|
|
1319
|
+
* RUNLY_SUPERVISOR), `autospawn` (the next CLI command) otherwise. Not
|
|
1320
|
+
* INVOCATION_ID: a daemon autospawned from anything under ANY systemd
|
|
1321
|
+
* service inherits that.
|
|
1322
|
+
*/
|
|
1323
|
+
supervisor: string;
|
|
1324
|
+
envs: {
|
|
1325
|
+
id: string;
|
|
1326
|
+
stack: string;
|
|
1327
|
+
state: import("../core/types.js").EnvState;
|
|
1328
|
+
ports: Record<string, number>;
|
|
1329
|
+
bindCount: number;
|
|
1330
|
+
/** Proxy state and traffic per public port (decision 0033). */
|
|
1331
|
+
proxy: Record<string, import("./proxy.js").ProxyStats>;
|
|
1332
|
+
/** Where its services run: the caller's worktree (decision 0032). */
|
|
1333
|
+
worktree: string;
|
|
1334
|
+
lease: LeaseRow | null;
|
|
1335
|
+
/** null = every service (the default set); otherwise the services the lease wants up — [] after `runly down` (decision 0034). */
|
|
1336
|
+
activeServices: string[] | null;
|
|
1337
|
+
idleMs: number;
|
|
1338
|
+
/** null = the holder never identified itself, so liveness is unknowable. */
|
|
1339
|
+
holderAlive: boolean | null;
|
|
1340
|
+
/** Why this environment is still holding its services, in one word. */
|
|
1341
|
+
heat: string;
|
|
1342
|
+
/**
|
|
1343
|
+
* Will the next bind take this environment as-is?
|
|
1344
|
+
*
|
|
1345
|
+
* `heat: 'cold'` means quiesced — services stopped, everything else
|
|
1346
|
+
* intact — which is a HEALTHY free pool entry, not a stuck one. Read as
|
|
1347
|
+
* "dead", it sent an operator to `pool recycle` to unstick a pool that
|
|
1348
|
+
* was already fine, and that recycle took other people's live leases
|
|
1349
|
+
* with it. So state the conclusion instead of leaving it to be inferred.
|
|
1350
|
+
*/
|
|
1351
|
+
available: boolean;
|
|
1352
|
+
/** Plain language for the two fields above, so no one has to infer it. */
|
|
1353
|
+
summary: string;
|
|
1354
|
+
}[];
|
|
1355
|
+
poolMaxTotal: number;
|
|
1356
|
+
ports: {
|
|
1357
|
+
public: import("../core/ports.js").PortBlock;
|
|
1358
|
+
internal: import("../core/ports.js").PortBlock;
|
|
1359
|
+
tunnel: import("../core/ports.js").PortBlock;
|
|
1360
|
+
ephemeral: import("../core/ports.js").PortBlock;
|
|
1361
|
+
conflicts: string[];
|
|
1362
|
+
};
|
|
1363
|
+
/** The server-wide load budget (decision 0036). */
|
|
1364
|
+
budget: {
|
|
1365
|
+
committedMemoryBytes: number;
|
|
1366
|
+
committedCpu: number;
|
|
1367
|
+
committed: {
|
|
1368
|
+
what: string;
|
|
1369
|
+
memoryBytes: number;
|
|
1370
|
+
cpu: number;
|
|
1371
|
+
}[];
|
|
1372
|
+
waiting: number;
|
|
1373
|
+
enabled: boolean;
|
|
1374
|
+
memoryBytes: number;
|
|
1375
|
+
cpu: number;
|
|
1376
|
+
reserveBytes: number;
|
|
1377
|
+
cpuPressure: number;
|
|
1378
|
+
loadPerCore: number;
|
|
1379
|
+
waitMs: number;
|
|
1380
|
+
maxQueue: number;
|
|
1381
|
+
defaultRun: {
|
|
1382
|
+
memoryBytes: number;
|
|
1383
|
+
cpu: number;
|
|
1384
|
+
};
|
|
1385
|
+
defaultBuild: {
|
|
1386
|
+
memoryBytes: number;
|
|
1387
|
+
cpu: number;
|
|
1388
|
+
};
|
|
1389
|
+
};
|
|
1390
|
+
/** Client bytes every proxied connection holds for a replay right now; capped by BACKLOT_PROXY_REPLAY_CAP_BYTES (0.19, default 64 MiB). */
|
|
1391
|
+
proxyReplayCarriedBytes: number;
|
|
1392
|
+
events: import("../core/events.js").DaemonEvent[];
|
|
1393
|
+
};
|
|
1394
|
+
/** Who a restart would make rebind: every live lease, named. */
|
|
1395
|
+
leaseHolders(): Array<{
|
|
1396
|
+
envId: string;
|
|
1397
|
+
holder: string;
|
|
1398
|
+
kind: string;
|
|
1399
|
+
}>;
|
|
1400
|
+
/**
|
|
1401
|
+
* What a `runly update` restart would cost, without doing it.
|
|
1402
|
+
*
|
|
1403
|
+
* Reported rather than inferred by the CLI because the two facts that decide
|
|
1404
|
+
* whether a restart is safe — which environments are BUSY, and who holds a
|
|
1405
|
+
* lease — only exist in the daemon's memory and journal.
|
|
1406
|
+
*/
|
|
1407
|
+
updatePlan(cliVersion?: string): {
|
|
1408
|
+
daemon: string;
|
|
1409
|
+
daemonBuild: string;
|
|
1410
|
+
cli: string | null;
|
|
1411
|
+
daemonPid: number;
|
|
1412
|
+
journalSchema: number;
|
|
1413
|
+
skew: import("../core/version.js").VersionSkew | null;
|
|
1414
|
+
busy: string[];
|
|
1415
|
+
leases: {
|
|
1416
|
+
envId: string;
|
|
1417
|
+
holder: string;
|
|
1418
|
+
kind: string;
|
|
1419
|
+
}[];
|
|
1420
|
+
};
|
|
1421
|
+
/**
|
|
1422
|
+
* Refuse a restart that would destroy work, and allow the one that costs only
|
|
1423
|
+
* a rebind.
|
|
1424
|
+
*
|
|
1425
|
+
* BUSY is the refusal. An `exec` runs detached so the command itself
|
|
1426
|
+
* survives the daemon, but the CALLER is blocked on this socket waiting for
|
|
1427
|
+
* its result — restarting hands it a dead connection and no result. Every other reclaim path already treats busy as inviolable
|
|
1428
|
+
* (claimForTeardown, both sweeper branches, pool gc, and shutdown's own reap
|
|
1429
|
+
* since 0.8.0), so this one does too.
|
|
1430
|
+
*
|
|
1431
|
+
* A LIVE LEASE is deliberately NOT a refusal. A restart stops services, keeps
|
|
1432
|
+
* the lease, and the holder's next verb rebinds — which is exactly what the
|
|
1433
|
+
* idle quiesce already does to leased environments without anyone's consent,
|
|
1434
|
+
* and what assertUsable already tells the holder to do ("run 'runly up' to
|
|
1435
|
+
* rebind"). Refusing here would mean an update on any busy shared box always
|
|
1436
|
+
* needs --force, and a flag you always pass is a flag that stops meaning
|
|
1437
|
+
* anything: that habituation is what made issue #40 destructive. Holders are
|
|
1438
|
+
* named in the plan instead.
|
|
1439
|
+
*
|
|
1440
|
+
* A DOWNGRADE is refused separately. Restarting from an older CLI replaces a
|
|
1441
|
+
* newer daemon with older code, which is the direction that strands state —
|
|
1442
|
+
* the journal stamp exists because of exactly that (JOURNAL_SCHEMA_VERSION).
|
|
1443
|
+
*/
|
|
1444
|
+
assertRestartable(opts: {
|
|
1445
|
+
force: boolean;
|
|
1446
|
+
cliVersion?: string;
|
|
1447
|
+
}): void;
|
|
1448
|
+
/**
|
|
1449
|
+
* doctor: actively check for the failure shapes the review surfaced —
|
|
1450
|
+
* orphaned ports, journal/reality pid divergence, envs stuck recycling, and a
|
|
1451
|
+
* CLI talking to a daemon that is not the installed build.
|
|
1452
|
+
*/
|
|
1453
|
+
doctor(cliVersion?: string): Promise<{
|
|
1454
|
+
ok: boolean;
|
|
1455
|
+
issues: {
|
|
1456
|
+
level: string;
|
|
1457
|
+
envId?: string;
|
|
1458
|
+
issue: string;
|
|
1459
|
+
}[];
|
|
1460
|
+
version: string;
|
|
1461
|
+
journalSchema: number;
|
|
1462
|
+
events: import("../core/events.js").DaemonEvent[];
|
|
1463
|
+
}>;
|
|
1464
|
+
/**
|
|
1465
|
+
* Atomically claim an env for teardown UNDER THE POOL LOCK: re-read it, and
|
|
1466
|
+
* (unless force) refuse if it's leased or busy, then flip it to the
|
|
1467
|
+
* 'recycling' guard state so tryClaim/sweep skip it. Returns the row to tear
|
|
1468
|
+
* down, or null if it slipped away. The slow teardown then runs OUTSIDE the
|
|
1469
|
+
* lock, but no claim can touch a 'recycling' env.
|
|
1470
|
+
*/
|
|
1471
|
+
private claimForTeardown;
|
|
1472
|
+
/**
|
|
1473
|
+
* Stop and reap this environment's services mid-bind, and journal the result
|
|
1474
|
+
* at once. The row is what capacity accounting reads: an application charge
|
|
1475
|
+
* retained for a conversion stands on `hot` or recorded pids, so leaving
|
|
1476
|
+
* either in the journal after the processes are gone kept a slot reserved for
|
|
1477
|
+
* nothing if the rest of the bind failed. Survivors stay recorded — and keep
|
|
1478
|
+
* the charge — because a pid nobody can confirm dead may still hold a port.
|
|
1479
|
+
* Re-reads the live row so a concurrent degrade is preserved and a recycled
|
|
1480
|
+
* row is never written back.
|
|
1481
|
+
*/
|
|
1482
|
+
/**
|
|
1483
|
+
* Stop just the services an `up` is about to restart because their build
|
|
1484
|
+
* output changed (decision 0032); the others keep running. Same reap
|
|
1485
|
+
* contract as stopForBind — recorded pids, then a tag scan — but the scan
|
|
1486
|
+
* is limited to THESE services' tags: the env's other services carry the
|
|
1487
|
+
* same env tag and are still serving.
|
|
1488
|
+
*/
|
|
1489
|
+
private stopServicesForRestart;
|
|
1490
|
+
private stopForBind;
|
|
1491
|
+
/**
|
|
1492
|
+
* After a supervisor stopAll(), kill any process that survived or escaped the
|
|
1493
|
+
* group signal so the port is truly free before the port-check below.
|
|
1494
|
+
*
|
|
1495
|
+
* Two failure modes:
|
|
1496
|
+
* (a) Recorded pid survived a previous kill (quiesce/teardown race): reapPids
|
|
1497
|
+
* retries with fresh SIGTERM→SIGKILL on all platforms.
|
|
1498
|
+
* (b) A service called setsid() / spawned a detached grandchild that inherited
|
|
1499
|
+
* the BACKLOT tag but moved to a new process group, escaping the -pgid
|
|
1500
|
+
* SIGKILL: scanTagged (Linux only) finds it by the tag in its environment
|
|
1501
|
+
* and kills the new group.
|
|
1502
|
+
*
|
|
1503
|
+
* Without this, a warm env whose quiesce left survivors hands out a port that
|
|
1504
|
+
* is still bound — the bind then fails with "occupied by a foreign process"
|
|
1505
|
+
* instead of reclaiming the escapee.
|
|
1506
|
+
*
|
|
1507
|
+
* Returns the entries NOT confirmed dead (reapPids contract): callers must
|
|
1508
|
+
* keep those in the journal — a forgotten pid is an orphan nobody can ever
|
|
1509
|
+
* reclaim, and only Linux has the tag scan to fall back on.
|
|
1510
|
+
*/
|
|
1511
|
+
private reapEnvProcesses;
|
|
1512
|
+
private reapDiscoveredProcesses;
|
|
1513
|
+
/** Enforce the kill switch under the env lock before preset validation can fail. */
|
|
1514
|
+
private enforceHolderPreviewForbidden;
|
|
1515
|
+
/**
|
|
1516
|
+
* Tear down a live preview the moment the manifest forbids one.
|
|
1517
|
+
*
|
|
1518
|
+
* Unconditional and early, unlike the rest of the reconcile: this is the
|
|
1519
|
+
* security kill switch README and ADR 0027 present as taking effect, and it
|
|
1520
|
+
* would be worthless if a stack stayed published whenever the bind that read
|
|
1521
|
+
* the flag went on to fail.
|
|
1522
|
+
*/
|
|
1523
|
+
private enforcePreviewForbidden;
|
|
1524
|
+
/**
|
|
1525
|
+
* Reconcile a live preview against the environment this bind just produced.
|
|
1526
|
+
*
|
|
1527
|
+
* The tunnel outlives service restarts, so everything here is about what is
|
|
1528
|
+
* now BEHIND the unchanged public URL. Three outcomes invalidate it outright
|
|
1529
|
+
* and tear it down — the manifest started forbidding preview, the previewed
|
|
1530
|
+
* service is no longer wanted (`runly down`, decision 0034), or it moved to a different local
|
|
1531
|
+
* port (a renamed port key; existing keys are never reassigned, decision
|
|
1532
|
+
* 0004). A data reset does not: the tunnel keeps serving, which is precisely
|
|
1533
|
+
* why it has to be said out loud.
|
|
1534
|
+
*
|
|
1535
|
+
* Runs at the bind's EPILOGUE, once the shape it judges against is committed.
|
|
1536
|
+
* From the top of the bind it was reading a REQUESTED slice that the epilogue
|
|
1537
|
+
* had not written yet, so a bind that then failed its ready probe left the
|
|
1538
|
+
* tunnel torn down against a slice change that never happened — and dropped
|
|
1539
|
+
* the report with the thrown error. `preview.forbidden` is the exception and
|
|
1540
|
+
* is enforced early by `enforcePreviewForbidden`: it depends on the manifest,
|
|
1541
|
+
* not on the bind.
|
|
1542
|
+
*
|
|
1543
|
+
* Never throws. The `down` that stopped the service or the bind that wiped the data is a
|
|
1544
|
+
* legitimate operation and failing it would strand the caller in a loop (and
|
|
1545
|
+
* bump failStreak into a pristine escalation); the classified message is the
|
|
1546
|
+
* report, and it rides back on the bind's own result.
|
|
1547
|
+
*
|
|
1548
|
+
* `active` must be the DURABLE shape this environment is bound to, never the
|
|
1549
|
+
* supervisor's live pid map: a service in restart backoff is missing from
|
|
1550
|
+
* that map for a second and would be read as "left the slice", killing a
|
|
1551
|
+
* tunnel the restart makes correct again. `portsReallocated` is false on the
|
|
1552
|
+
* hot-reload refresh path — only a real bind fills `env.ports` for a renamed port
|
|
1553
|
+
* key, so before one runs the service is still listening where the tunnel
|
|
1554
|
+
* points and a mismatch means nothing yet.
|
|
1555
|
+
*/
|
|
1556
|
+
private reconcilePreviewForBind;
|
|
1557
|
+
/**
|
|
1558
|
+
* Last-resort reap of anything still LIVING IN this env's private directory,
|
|
1559
|
+
* by cwd — at teardown, when that directory is about to be deleted.
|
|
1560
|
+
*
|
|
1561
|
+
* Since decision 0032 services run in the caller's WORKTREE, not here, so
|
|
1562
|
+
* this no longer finds a service that scrubbed its tag: cwd in a worktree is
|
|
1563
|
+
* NOT an ownership signal (the agent's own shells and builds sit there), and
|
|
1564
|
+
* scanning it would kill a stranger's process — the one outcome this whole
|
|
1565
|
+
* module is written to avoid. The tag scan and recorded groups are what
|
|
1566
|
+
* remain for services; this covers anything left in the env directory
|
|
1567
|
+
* itself (a process started under the old projection, for one).
|
|
1568
|
+
*/
|
|
1569
|
+
private reapEnvTree;
|
|
1570
|
+
/** Slow teardown of an already-claimed ('recycling') env. */
|
|
1571
|
+
private teardownClaimed;
|
|
1572
|
+
/**
|
|
1573
|
+
* Drop one namespace the way its row recorded it (decision 0037): run the
|
|
1574
|
+
* templated drop command, or delete a file inside the state root. Throws
|
|
1575
|
+
* when the drop did not confirm.
|
|
1576
|
+
*/
|
|
1577
|
+
private runDropRecipe;
|
|
1578
|
+
/**
|
|
1579
|
+
* Is `env.root` safe to delete? It must sit strictly inside the state root's
|
|
1580
|
+
* envs/ directory, and must not be (or contain) the worktree the
|
|
1581
|
+
* environment runs in — a corrupted row must never turn teardown into
|
|
1582
|
+
* `rm -rf` of someone's work.
|
|
1583
|
+
*/
|
|
1584
|
+
private isPrivateEnvDir;
|
|
1585
|
+
private recycleOne;
|
|
1586
|
+
/**
|
|
1587
|
+
* The clean-slate button, scoped.
|
|
1588
|
+
*
|
|
1589
|
+
* `envId` is an explicit statement of intent — THIS one, not the others — and
|
|
1590
|
+
* it used to be dropped on the floor, so aiming at one cold environment
|
|
1591
|
+
* recycled the whole pool and tore down siblings' live work. A named target
|
|
1592
|
+
* therefore never widens: an unknown id is a usage error, and a leased target
|
|
1593
|
+
* REFUSES rather than being silently skipped, because "nothing happened and
|
|
1594
|
+
* nothing said why" is what sends a caller reaching for force.
|
|
1595
|
+
*
|
|
1596
|
+
* `force` (--force, historically --all) is the only thing that ever reclaims a
|
|
1597
|
+
* leased environment; `busy` is never interrupted, not even by force.
|
|
1598
|
+
*/
|
|
1599
|
+
poolRecycle(opts: {
|
|
1600
|
+
envId?: string;
|
|
1601
|
+
force: boolean;
|
|
1602
|
+
}): Promise<{
|
|
1603
|
+
recycled: string[];
|
|
1604
|
+
skipped: {
|
|
1605
|
+
envId: string;
|
|
1606
|
+
reason: string;
|
|
1607
|
+
}[];
|
|
1608
|
+
}>;
|
|
1609
|
+
/** This worktree's degraded, idle environment(s), reaped so a claim can recreate one. */
|
|
1610
|
+
private reapDegradedOwn;
|
|
1611
|
+
/**
|
|
1612
|
+
* One environment per worktree (decision 0032) holds for every environment
|
|
1613
|
+
* this daemon creates. A journal from an older runly can still hold several
|
|
1614
|
+
* for one stack; the unleased, idle ones beyond the one to keep are recycled
|
|
1615
|
+
* here, so the pool converges. A surplus environment that is still leased is
|
|
1616
|
+
* left to its holder until the lease ends.
|
|
1617
|
+
*/
|
|
1618
|
+
private drainSurplusEnvs;
|
|
1619
|
+
/** Reap the provably-dead (degraded) envs now, instead of waiting for the sweep. */
|
|
1620
|
+
poolReconcile(): Promise<{
|
|
1621
|
+
reaped: string[];
|
|
1622
|
+
}>;
|
|
1623
|
+
/** The proxy saw a client byte: the clock moves in memory, and reaches the journal throttled. */
|
|
1624
|
+
private noteActivity;
|
|
1625
|
+
/** Persist unsaved activity clocks (all, from the sweeper; one env, throttled, from the hook). */
|
|
1626
|
+
private flushActivity;
|
|
1627
|
+
/** When anyone last used this environment: a runly verb or a client byte on any of its ports. */
|
|
1628
|
+
private envActivityAt;
|
|
1629
|
+
/**
|
|
1630
|
+
* One running service's idle clock: the environment's last runly verb, the
|
|
1631
|
+
* last client byte through the service's own public port (any port's, for
|
|
1632
|
+
* a service with none), and its own start.
|
|
1633
|
+
*/
|
|
1634
|
+
private serviceClock;
|
|
1635
|
+
/**
|
|
1636
|
+
* How long a running service of `env` may sit idle: its `idle:` (or the
|
|
1637
|
+
* default), capped for an UNLEASED environment — nobody can wake that one by
|
|
1638
|
+
* traffic — at BACKLOT_IDLE_TTL_MS (the pre-0.16 quiesce). `ps` shows the same.
|
|
1639
|
+
*/
|
|
1640
|
+
private serviceIdleLimit;
|
|
1641
|
+
/** Stop the running services of `env` whose idle clock ran out. */
|
|
1642
|
+
private stopIdleServices;
|
|
1643
|
+
/**
|
|
1644
|
+
* Is a tether gone (decision 0035)? A dead pid is believed only once it has
|
|
1645
|
+
* stayed dead for the grace (BACKLOT_TETHER_GRACE_MS, 1 minute); the first
|
|
1646
|
+
* sighting starts the clock, in memory — a daemon restart starts it again.
|
|
1647
|
+
*/
|
|
1648
|
+
private tetherGone;
|
|
1649
|
+
/**
|
|
1650
|
+
* The proxy's wake hook (decision 0035): a connection reached the public
|
|
1651
|
+
* port of a service that is not running. Starts it when this environment
|
|
1652
|
+
* holds a live lease and the lease wants that service (a `down`ed service
|
|
1653
|
+
* stays down); the proxy holds the connection meanwhile. Synchronous and
|
|
1654
|
+
* cheap: the start runs in the background.
|
|
1655
|
+
*/
|
|
1656
|
+
private requestWake;
|
|
1657
|
+
/**
|
|
1658
|
+
* Start one wanted service — and the wanted, stopped members of its
|
|
1659
|
+
* depends_on closure — on demand. `up`'s start path for a single service:
|
|
1660
|
+
* appliances ensured, datastores kept, the load budget asked (bounded by
|
|
1661
|
+
* how long the proxy holds a connection), no rebuild unless needed (a
|
|
1662
|
+
* `when:` build whose inputs changed, or one that never succeeded here).
|
|
1663
|
+
*/
|
|
1664
|
+
private wakeService;
|
|
1665
|
+
/**
|
|
1666
|
+
* Start `closure` — stopped services the lease wants — the way a wake does.
|
|
1667
|
+
* MUST run under the environment's lock. A wake never restores data: a
|
|
1668
|
+
* datastore that exists is kept as it is, so the trigger listing behind the
|
|
1669
|
+
* template keys is computed only for one that is missing.
|
|
1670
|
+
*/
|
|
1671
|
+
private startStoppedLocked;
|
|
1672
|
+
/**
|
|
1673
|
+
* `exec` and `token` run against the environment as the lease left it. A
|
|
1674
|
+
* service the idle clock stopped (decision 0035) is started first — under
|
|
1675
|
+
* the lock these verbs hold, where a connection-triggered wake would only
|
|
1676
|
+
* queue behind them until the proxy's hold ran out. So is what a daemon
|
|
1677
|
+
* restart stopped (0.19). A failed bind is refused: `up` redoes it.
|
|
1678
|
+
*/
|
|
1679
|
+
private resumeForVerb;
|
|
1680
|
+
/**
|
|
1681
|
+
* A service the supervisor restarts after a crash (decision 0035): its
|
|
1682
|
+
* port HOLDS connections from the crash until the new process accepts on
|
|
1683
|
+
* its (unchanged) internal port, instead of refusing them.
|
|
1684
|
+
*/
|
|
1685
|
+
private serviceCrashed;
|
|
1686
|
+
private serviceRelaunched;
|
|
1687
|
+
private serviceGaveUp;
|
|
1688
|
+
/**
|
|
1689
|
+
* A service of a LEASED environment crash-looped past its restart budget
|
|
1690
|
+
* (decision 0039). It is stopped and reported as failed — `ps` and `ctx`
|
|
1691
|
+
* show `failed` with its last exit, `runly logs <svc>` says why — while the
|
|
1692
|
+
* environment, its data, its other services and its logs stay. Nothing
|
|
1693
|
+
* restarts it on its own (a wake skips it); the next `up` does.
|
|
1694
|
+
*/
|
|
1695
|
+
private serviceFailed;
|
|
1696
|
+
/** Is `name`, or a service it depends on (transitively), failed (decision 0039)? */
|
|
1697
|
+
private blockedByFailure;
|
|
1698
|
+
/** How a failed service ended, or undefined (decision 0039). */
|
|
1699
|
+
private failureOf;
|
|
1700
|
+
sweep(): Promise<void>;
|
|
1701
|
+
private sweepOnce;
|
|
1702
|
+
shutdown(): Promise<void>;
|
|
1703
|
+
}
|