@theford/runly 0.0.0-stage → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +157 -2
  3. package/dist/cli/client.d.ts +80 -0
  4. package/dist/cli/client.js +294 -0
  5. package/dist/cli/client.js.map +1 -0
  6. package/dist/cli/daemon-unit.d.ts +58 -0
  7. package/dist/cli/daemon-unit.js +289 -0
  8. package/dist/cli/daemon-unit.js.map +1 -0
  9. package/dist/cli/index.d.ts +2 -0
  10. package/dist/cli/index.js +1482 -0
  11. package/dist/cli/index.js.map +1 -0
  12. package/dist/cli/install.d.ts +9 -0
  13. package/dist/cli/install.js +53 -0
  14. package/dist/cli/install.js.map +1 -0
  15. package/dist/cli/logs.d.ts +43 -0
  16. package/dist/cli/logs.js +232 -0
  17. package/dist/cli/logs.js.map +1 -0
  18. package/dist/cli/tether.d.ts +3 -0
  19. package/dist/cli/tether.js +65 -0
  20. package/dist/cli/tether.js.map +1 -0
  21. package/dist/cli/watchdog.d.ts +1 -0
  22. package/dist/cli/watchdog.js +81 -0
  23. package/dist/cli/watchdog.js.map +1 -0
  24. package/dist/core/builds.d.ts +20 -0
  25. package/dist/core/builds.js +93 -0
  26. package/dist/core/builds.js.map +1 -0
  27. package/dist/core/caller-env.d.ts +17 -0
  28. package/dist/core/caller-env.js +77 -0
  29. package/dist/core/caller-env.js.map +1 -0
  30. package/dist/core/diagnostics.d.ts +68 -0
  31. package/dist/core/diagnostics.js +34 -0
  32. package/dist/core/diagnostics.js.map +1 -0
  33. package/dist/core/env-vars.d.ts +35 -0
  34. package/dist/core/env-vars.js +32 -0
  35. package/dist/core/env-vars.js.map +1 -0
  36. package/dist/core/events.d.ts +10 -0
  37. package/dist/core/events.js +42 -0
  38. package/dist/core/events.js.map +1 -0
  39. package/dist/core/exec.d.ts +22 -0
  40. package/dist/core/exec.js +84 -0
  41. package/dist/core/exec.js.map +1 -0
  42. package/dist/core/journal.d.ts +227 -0
  43. package/dist/core/journal.js +395 -0
  44. package/dist/core/journal.js.map +1 -0
  45. package/dist/core/logs.d.ts +56 -0
  46. package/dist/core/logs.js +259 -0
  47. package/dist/core/logs.js.map +1 -0
  48. package/dist/core/manifest.d.ts +292 -0
  49. package/dist/core/manifest.js +179 -0
  50. package/dist/core/manifest.js.map +1 -0
  51. package/dist/core/namespaces.d.ts +7 -0
  52. package/dist/core/namespaces.js +59 -0
  53. package/dist/core/namespaces.js.map +1 -0
  54. package/dist/core/paths.d.ts +31 -0
  55. package/dist/core/paths.js +65 -0
  56. package/dist/core/paths.js.map +1 -0
  57. package/dist/core/policy.d.ts +94 -0
  58. package/dist/core/policy.js +122 -0
  59. package/dist/core/policy.js.map +1 -0
  60. package/dist/core/ports.d.ts +52 -0
  61. package/dist/core/ports.js +94 -0
  62. package/dist/core/ports.js.map +1 -0
  63. package/dist/core/presets.d.ts +25 -0
  64. package/dist/core/presets.js +86 -0
  65. package/dist/core/presets.js.map +1 -0
  66. package/dist/core/procscan.d.ts +111 -0
  67. package/dist/core/procscan.js +342 -0
  68. package/dist/core/procscan.js.map +1 -0
  69. package/dist/core/retention.d.ts +97 -0
  70. package/dist/core/retention.js +322 -0
  71. package/dist/core/retention.js.map +1 -0
  72. package/dist/core/sleep.d.ts +51 -0
  73. package/dist/core/sleep.js +99 -0
  74. package/dist/core/sleep.js.map +1 -0
  75. package/dist/core/tree-ledger.d.ts +39 -0
  76. package/dist/core/tree-ledger.js +92 -0
  77. package/dist/core/tree-ledger.js.map +1 -0
  78. package/dist/core/types.d.ts +21 -0
  79. package/dist/core/types.js +6 -0
  80. package/dist/core/types.js.map +1 -0
  81. package/dist/core/units.d.ts +20 -0
  82. package/dist/core/units.js +69 -0
  83. package/dist/core/units.js.map +1 -0
  84. package/dist/core/upkeep.d.ts +90 -0
  85. package/dist/core/upkeep.js +176 -0
  86. package/dist/core/upkeep.js.map +1 -0
  87. package/dist/core/util.d.ts +46 -0
  88. package/dist/core/util.js +168 -0
  89. package/dist/core/util.js.map +1 -0
  90. package/dist/core/version.d.ts +39 -0
  91. package/dist/core/version.js +144 -0
  92. package/dist/core/version.js.map +1 -0
  93. package/dist/core/worktree.d.ts +51 -0
  94. package/dist/core/worktree.js +309 -0
  95. package/dist/core/worktree.js.map +1 -0
  96. package/dist/daemon/budget.d.ts +155 -0
  97. package/dist/daemon/budget.js +288 -0
  98. package/dist/daemon/budget.js.map +1 -0
  99. package/dist/daemon/election.d.ts +9 -0
  100. package/dist/daemon/election.js +140 -0
  101. package/dist/daemon/election.js.map +1 -0
  102. package/dist/daemon/engine.d.ts +1703 -0
  103. package/dist/daemon/engine.js +5305 -0
  104. package/dist/daemon/engine.js.map +1 -0
  105. package/dist/daemon/index.d.ts +2 -0
  106. package/dist/daemon/index.js +412 -0
  107. package/dist/daemon/index.js.map +1 -0
  108. package/dist/daemon/proxy.d.ts +108 -0
  109. package/dist/daemon/proxy.js +615 -0
  110. package/dist/daemon/proxy.js.map +1 -0
  111. package/dist/daemon/supervisor.d.ts +101 -0
  112. package/dist/daemon/supervisor.js +493 -0
  113. package/dist/daemon/supervisor.js.map +1 -0
  114. package/dist/drivers/appliances.d.ts +11 -0
  115. package/dist/drivers/appliances.js +174 -0
  116. package/dist/drivers/appliances.js.map +1 -0
  117. package/dist/drivers/datastores.d.ts +144 -0
  118. package/dist/drivers/datastores.js +775 -0
  119. package/dist/drivers/datastores.js.map +1 -0
  120. package/dist/drivers/preview.d.ts +69 -0
  121. package/dist/drivers/preview.js +605 -0
  122. package/dist/drivers/preview.js.map +1 -0
  123. package/dist/drivers/types.d.ts +70 -0
  124. package/dist/drivers/types.js +13 -0
  125. package/dist/drivers/types.js.map +1 -0
  126. package/package.json +63 -4
  127. package/schema/backlot.schema.json +549 -0
  128. package/schema/runly.schema.json +556 -0
@@ -0,0 +1,1703 @@
1
+ import { Journal, type LeaseRow } from '../core/journal.js';
2
+ import { LoadBudget, type Committed } from './budget.js';
3
+ import { type UpkeepStep } from '../core/upkeep.js';
4
+ import { ProxyHub } from './proxy.js';
5
+ import { killGroupVerified } from './supervisor.js';
6
+ import { type BindDiagnostics } from '../core/diagnostics.js';
7
+ import type { Hygiene, LeaseKind } from '../core/types.js';
8
+ /** Streamed bind phases → human progress on stderr (never on the --json stdout). */
9
+ export type Progress = (phase: string) => void;
10
+ /** The answer to `up --data-only` from any client that still sends it (the CLI exits 64 before the daemon). */
11
+ export declare const DATA_ONLY_REMOVED: string;
12
+ export interface UpOptions {
13
+ /**
14
+ * Datastores to RELOAD from their template, each with the preset to load
15
+ * (decision 0034). A datastore not named here keeps whatever it holds.
16
+ */
17
+ presets?: unknown;
18
+ cwd: string;
19
+ /** Explicit refresh envelope; absent means retain this lease's in-memory inputs. */
20
+ callerEnv?: unknown;
21
+ holder?: string;
22
+ hygiene?: Hygiene;
23
+ kind?: LeaseKind;
24
+ ttlMs?: number;
25
+ /**
26
+ * Services to START, plus their transitive depends_on closure — ADDED to
27
+ * what the environment already runs (decision 0034: `up` is additive and
28
+ * never stops a running service). Empty or omitted is the manifest's default
29
+ * set, which is every service. (Only the internal reset-data rebind adds
30
+ * nothing; it does not come through `up`.)
31
+ */
32
+ services?: string[];
33
+ /** Removed by decision 0034; a request that still sets it is refused with DATA_ONLY_REMOVED. */
34
+ dataOnly?: boolean;
35
+ /** Run every build, also those whose `when:` inputs are unchanged (decision 0038). */
36
+ rebuild?: boolean;
37
+ /**
38
+ * The CALLER's process, so its lease can be released when it dies.
39
+ * The CLI exits per invocation, so this must be the long-lived agent's pid —
40
+ * supplied via --holder-pid or BACKLOT_HOLDER_PID.
41
+ */
42
+ holderPid?: number;
43
+ /** Set by the daemon per-request; emits progress frames back to the client. */
44
+ onProgress?: Progress;
45
+ /**
46
+ * Aborted when the requesting client disconnects (the daemon wires it to
47
+ * the response's close). A queued `up` whose caller is gone gives up
48
+ * instead of binding for nobody.
49
+ */
50
+ signal?: AbortSignal;
51
+ }
52
+ export declare class Engine {
53
+ private readonly reapServiceGroup;
54
+ constructor(reapServiceGroup?: typeof killGroupVerified);
55
+ readonly journal: Journal;
56
+ /** The L4 proxy holding every environment's public ports (decision 0033). */
57
+ readonly proxy: ProxyHub;
58
+ private supervisors;
59
+ private lastSweep;
60
+ private lastRetention;
61
+ private lastGc;
62
+ private lastSweepMono;
63
+ /** waketime (epoch ms) of the last kernel-recorded sleep already pardoned. */
64
+ private lastPardonedWake;
65
+ /** FIFO tickets for capacity waiters, so an early waiter is not starved.
66
+ * PER STACK: capacity is per-stack, so one queue for the whole engine made
67
+ * a waiter on full stack A block stack B's instantly-satisfiable claim. */
68
+ private waitTicket;
69
+ private waiting;
70
+ private poolChain;
71
+ private envChains;
72
+ /**
73
+ * One lock per WORKTREE (keyed by stack id), held around everything runly
74
+ * writes into it: upkeep rules, builds, and a pristine bind clearing the
75
+ * worktree ledger (decision 0032). A worktree has one environment, but
76
+ * `runly warm` runs there with none — and before the first bind creates it —
77
+ * so the environment lock alone cannot serialize them. Ordering rule: an
78
+ * environment lock is always taken BEFORE this one, never after — `warm`
79
+ * takes the environment lock(s) of its stack first, so no cycle exists.
80
+ */
81
+ private treeChains;
82
+ /** Envs with an operation in flight — the sweeper must not expire/quiesce these. */
83
+ readonly busy: Set<string>;
84
+ /** What is in flight on a busy env, so a deferral can name it. */
85
+ private busyOp;
86
+ /** Never persisted or returned. Lease IDs prevent reuse from inheriting another caller's inputs. */
87
+ private leaseInputs;
88
+ /** Opaque in-memory revisions only; no secret values or hashes in the journal. */
89
+ private appliedInputs;
90
+ private appliedInputSpecs;
91
+ private appliedManifests;
92
+ /**
93
+ * Re-publishing the leases' previews after a restart (decision 0045), per
94
+ * lease id: in flight, or failed with the reason `ctx` reports. In memory:
95
+ * it describes this daemon life; the lease row keeps what to re-publish.
96
+ */
97
+ private previewRestores;
98
+ /** Settles once every recorded preview was re-published or reported; a restore that outlives a `shutdown` stops what it started. */
99
+ previewsRestored: Promise<void>;
100
+ private inputRevision;
101
+ /** Unsaved proxy activity per env, by port key; flushed throttled and by the sweeper. */
102
+ private activityDirty;
103
+ private activityFlushedAt;
104
+ /** When each running service process was started — a fresh start is activity. */
105
+ private startedAt;
106
+ /** Services stopped for idleness (ps shows them as `idle`). */
107
+ /** The last failed `up` per environment (0.19), for ctx and ps; cleared by a successful one. */
108
+ private readonly bindFailures;
109
+ private idleStopped;
110
+ /**
111
+ * Services of a LEASED environment that crash-looped and were stopped
112
+ * (decision 0039), by env + service: `ps` and `ctx` show them as `failed`
113
+ * with how they ended; a wake skips them; the next `up` starts them again.
114
+ */
115
+ private failedServices;
116
+ /** In-flight wakes, by env + service, so a burst of connections starts a service once. */
117
+ private waking;
118
+ /** First time a tether was seen dead; it is believed only after the grace. */
119
+ private goneSince;
120
+ /** The manifest each environment's running services were started from (budget accounting). */
121
+ private envManifests;
122
+ /** Appliances runly started in this daemon life, by probe address (budget accounting); pruned when one stops. */
123
+ private startedAppliances;
124
+ /** When each idle-stopped service stopped: a wake soon after skips the CPU gate (decision 0036). */
125
+ private idleStoppedAt;
126
+ /** A sweep is running: the next tick is skipped rather than overlapping it. */
127
+ private sweeping;
128
+ /** shutdown() has begun: no sweep, no wake, nothing new. */
129
+ stopping: boolean;
130
+ /** The server-wide load budget (decision 0036). */
131
+ readonly budget: LoadBudget;
132
+ private poolLocked;
133
+ private envLocked;
134
+ private treeLocked;
135
+ /** Hold several environment locks at once, in a fixed (sorted) order. */
136
+ private envsLocked;
137
+ /**
138
+ * The worktree's upkeep ledger, and a committer that writes back only the
139
+ * worktree's half of a merged fingerprint map. MUST be used inside
140
+ * treeLocked: the read-modify-write is what the lock serializes.
141
+ */
142
+ private treeLedgerSession;
143
+ /** Recovery (decision 0009): reap recorded PIDs from a previous daemon life; hot -> warm. */
144
+ recover(): Promise<void>;
145
+ /**
146
+ * Reclaim runly-spawned processes that no live environment accounts for.
147
+ *
148
+ * A process is an orphan when it carries this state root's tag but its env
149
+ * either no longer exists in the journal, or exists in a state that must have
150
+ * no services running (warm/recycling). Anything belonging to a hot env, or
151
+ * to an env with an operation in flight, is left strictly alone — a bind
152
+ * racing the sweep must not have its dev-server shot out from under it.
153
+ */
154
+ poolGc(): Promise<{
155
+ supported: boolean;
156
+ reclaimed: Array<{
157
+ pid: number;
158
+ envId: string;
159
+ service: string;
160
+ }>;
161
+ skipped: number;
162
+ }>;
163
+ /**
164
+ * An environment's PRIVATE state — data dir, logs. There is no tree here:
165
+ * services run in the caller's worktree (decision 0032).
166
+ */
167
+ private envDirs;
168
+ private createEnv;
169
+ /**
170
+ * A public port nobody holds: inside the public block, recorded on no
171
+ * environment row, not one this daemon listens on, and probed free (which
172
+ * also skips a port a tunnel or another process binds on any address).
173
+ */
174
+ private allocPublicPort;
175
+ /**
176
+ * A fresh internal port for one start of env/key (decision 0033), reserved
177
+ * in the proxy at once. The probe awaits, so another start can be probing
178
+ * the same port meanwhile: the reservation is re-checked after it and taken
179
+ * with no await in between. Released when the target goes down (a stop, a
180
+ * failed start, a settle) or is owned by the target once it is up.
181
+ */
182
+ private allocInternalPort;
183
+ /**
184
+ * Make the proxy hold every public port of `env` (decision 0033). Idempotent.
185
+ *
186
+ * A recorded port is kept whenever it can be: the whole point is that it is
187
+ * stable. It moves only when it cannot be held — it lies outside the public
188
+ * block (a journal from runly 0.13, whose ports came from the OS ephemeral
189
+ * range), or another process took it while no listener held it (the daemon
190
+ * was down). A move is saved to the journal at once and reported; returns
191
+ * one line per moved port, empty when nothing moved.
192
+ */
193
+ private ensureProxies;
194
+ /** One atomic claim attempt — MUST run under the pool lock. */
195
+ private tryClaim;
196
+ /**
197
+ * Does the machine-wide ceiling refuse a NEW environment right now? Named
198
+ * explicitly because a refusal must quote the cap that bound (#47). There
199
+ * is no per-stack cap (one environment per worktree, decision 0032) and no
200
+ * data-only cap (decision 0034: a database without an application is a
201
+ * `runly db` copy, which is not an environment). MUST run under the pool lock.
202
+ */
203
+ private capacityBinding;
204
+ /**
205
+ * Cold, unleased environments that can be given up to free a MACHINE-WIDE
206
+ * slot, least-recently-used first.
207
+ *
208
+ * Why eviction rather than just not counting them (#46's own suggestion): the
209
+ * caps gate environment CREATION only — rebinding an existing environment is
210
+ * never capacity-checked — so the row count is what bounds worst-case
211
+ * concurrent load. Excluding cold rows from the count would leave nothing to
212
+ * stop N of them being rebound hot at once. Giving the slot back for real
213
+ * keeps the ceiling meaningful, and costs the least-recently-used environment
214
+ * one cold provision the next time it is bound.
215
+ *
216
+ * Unleased + not busy + idle past IDLE_TTL qualifies. A recently released
217
+ * environment is the warm pool doing its job, so it is never taken — if
218
+ * nothing is that idle the caller gets a refusal naming the cap that actually
219
+ * bound, which is still strictly better than the lockout.
220
+ *
221
+ * Deliberately NOT restricted to `warm`. Idle past IDLE_TTL is precisely what
222
+ * the sweeper quiesces, but it only looks every BACKLOT_SWEEP_MS (15s by
223
+ * default) — so an environment released and abandoned a moment ago is still
224
+ * `hot` while already being condemned. Requiring `warm` meant that for a whole
225
+ * sweep interval the caller was refused with "waiting will not help", when the
226
+ * next sweep would in fact have made this environment evictable. Taking a
227
+ * condemned `hot` environment is the same trade one step further, and
228
+ * recycleOne stops its services through the ordinary teardown.
229
+ */
230
+ private evictionCandidates;
231
+ /**
232
+ * Free a machine-wide slot by giving up the least-recently-used cold
233
+ * environment. Returns the id evicted, or null if nothing qualified.
234
+ *
235
+ * MUST NOT hold the pool lock: recycleOne claims under it (the chain is not
236
+ * re-entrant), and the teardown that follows deletes a tree, which the pool
237
+ * lock must never wait on. claimForTeardown re-reads and refuses anything
238
+ * leased or busy, so a candidate chosen here and leased in the gap is declined
239
+ * rather than taken — never take an environment somebody else is holding (#40).
240
+ */
241
+ private evictForMachineCapacity;
242
+ /**
243
+ * Is some environment about to become claimable on its own?
244
+ *
245
+ * `busy` is the one that matters here and was missed: the idle quiesce runs
246
+ * under the environment lock, which marks it busy, and decision 0021 keeps its
247
+ * mid-quiesce state as plain `hot` on purpose. So the sweeper reclaiming heat
248
+ * from the only candidate made it briefly unevictable — and the machine-wide
249
+ * refusal, which tells the caller that waiting cannot help, fired on a
250
+ * condition that clears in milliseconds. It reproduced on the macOS runner,
251
+ * where teardown is slower (services get SIGTERM, then a verified SIGKILL).
252
+ *
253
+ * The other three are transient for the reasons the per-stack branch already
254
+ * gives: the sweeper reaps them and frees capacity.
255
+ */
256
+ private transientlyUnclaimable;
257
+ /**
258
+ * Why this environment could not be given up, per environment.
259
+ *
260
+ * It used to say "too recent to evict" for every unleased row, which is the
261
+ * defect #47 was about in miniature: a message that names one cause for four
262
+ * different situations sends the reader after the wrong one. (It cost an hour
263
+ * here, diagnosing a CI failure whose real cause was `busy`.)
264
+ */
265
+ private notEvictableBecause;
266
+ /**
267
+ * Is the pool full of environments whose leases outlast our whole wait?
268
+ *
269
+ * If so, queueing cannot possibly succeed, and reporting "waited 60s" blames a
270
+ * timing problem that does not exist. MUST run under the pool lock.
271
+ */
272
+ private structuralCapacityBlock;
273
+ /**
274
+ * Why this worktree's own environment cannot be claimed, if that is what is
275
+ * in the way (decision 0032: one environment per worktree, so a second
276
+ * holder waits for it instead of getting another).
277
+ *
278
+ * - `undefined`: the worktree has no environment, or it sits unleased and
279
+ * usable — whatever refused the claim is a capacity ceiling, not the
280
+ * worktree.
281
+ * - `null`: it is in the way, but will clear by itself within the wait
282
+ * (a lease that lapses in time, a recycle or reap in progress).
283
+ * - a string: it is leased past the wait window — naming who holds it.
284
+ */
285
+ private worktreeHold;
286
+ /**
287
+ * One refusal text for both throw sites, quoting real counts and the cap that
288
+ * actually bound.
289
+ *
290
+ * The old messages printed `(${POOL_MAX()}/${POOL_MAX()})` — the cap twice,
291
+ * not a count — so a stack with ZERO environments was told "pool at capacity
292
+ * (6/6)" and pointed at the wrong knob (#47).
293
+ */
294
+ private capacityRefusal;
295
+ /** Queue at capacity WITHOUT holding the pool lock while sleeping. */
296
+ private acquireEnv;
297
+ private acquireQueued;
298
+ /**
299
+ * The values `{{…}}` resolves to. Everything is PUBLIC (decision 0033): the
300
+ * ports and URLs users, builds, exec and other services see are the proxy's.
301
+ * The one exception is a service's own command line and env, given
302
+ * `own = { key, port }`: there `{{ports.<own key>}}` is the INTERNAL port the
303
+ * service must listen on. `{{public_ports.<key>}}` is always the public one,
304
+ * for a service that has to advertise its own address.
305
+ */
306
+ private templateCtx;
307
+ private supervisor;
308
+ /**
309
+ * The services a bind must bring up: the named ones plus the transitive
310
+ * `depends_on` closure of each. An unknown name is a manifest work-error,
311
+ * reported like the other service lookups. A depends_on cycle is tolerated
312
+ * here (the visit is closure-guarded) and caught by the start loop's own
313
+ * cycle check.
314
+ */
315
+ private resolveServiceClosure;
316
+ /**
317
+ * The services `env`'s lease wants up (decision 0034): `up` adds to this set
318
+ * and `down` takes away. Undefined on the row means every declared service,
319
+ * so a service added to the manifest later is included.
320
+ */
321
+ private desiredServices;
322
+ /** How a wanted set is recorded: undefined for every declared service, else the list (possibly empty). */
323
+ private recordedShape;
324
+ /**
325
+ * The services that use datastore `ds`, read from the manifest: a service
326
+ * uses it when its `run:`, `build:` or `env:` templates `{{datastores.<ds>.…}}`
327
+ * — that is how a service is handed a connection string. `null` when no
328
+ * service references it at all: then runly cannot tell who reads it (a
329
+ * connection string in a config file, a hard-coded path), and the caller
330
+ * must restart every running service to be safe (decision 0034).
331
+ */
332
+ private datastoreUsers;
333
+ /** The datastores an environment of `stack` provisions: all but the `copies_only` ones (decision 0039). */
334
+ private envDatastores;
335
+ /** `up --preset` and `reset-data --preset` name only datastores an environment has (decision 0039). */
336
+ private refuseCopiesOnlyPresets;
337
+ /**
338
+ * Give back the namespaces of datastores an environment no longer
339
+ * provisions — one made before its datastore became `copies_only` (decision
340
+ * 0039) — by the drop recorded for each. A drop that fails stays on the row
341
+ * for teardown to retry.
342
+ */
343
+ private retireEnvDatastores;
344
+ /**
345
+ * Create, keep or reload each datastore in `plan` — all of them at once
346
+ * (0.19): each bakes and restores under its own datastore lock, so three
347
+ * datastores cost the slowest one, not their sum. Every completed restore is
348
+ * journalled at once, so a failure still reports the others truthfully, and
349
+ * the call settles only when every datastore has (the first error wins). A
350
+ * datastore not forced and already present is KEPT — whatever it holds, even
351
+ * a preset the catalog no longer offers (decision 0034: no preset means keep
352
+ * the data, never a silent reset to the default).
353
+ */
354
+ private prepareDatastores;
355
+ /**
356
+ * What an additive `up` builds and may restart: every service it leaves
357
+ * running for a plain `up`; for `up <service…>` only the named services,
358
+ * their depends_on closure and what it (re)starts because it is not running
359
+ * (0.19). A full rebind still stops and starts everything — its reasons
360
+ * (an upkeep rule ran, the manifest changed, …) concern every service.
361
+ */
362
+ private bindScope;
363
+ private bindAndStart;
364
+ private bindAndStartInner;
365
+ /** The public ports of `services` hold new connections until their service is ready again. */
366
+ private markStarting;
367
+ /**
368
+ * The order the builds of `names` run in: depends_on waves. A build runs
369
+ * after the builds of what its service depends on (transitively), and the
370
+ * builds of one wave run at once — independent services no longer wait for
371
+ * each other. `build: {serial: true}` runs alone; `builds: serial` on the
372
+ * stack runs every build alone, in manifest order (the behaviour before
373
+ * 0.16.1). The load budget holds the costliest wave's sum (needFor).
374
+ *
375
+ * Builds that write one shared output or take one tool lock belong in
376
+ * `serial`. Two Angular projects are fine in parallel: the CLI's persistent
377
+ * cache is keyed per project (`.angular/cache/<version>/<project>`).
378
+ */
379
+ private buildWaves;
380
+ /**
381
+ * Run `fn` for each service of `waves`, wave after wave, the members of one
382
+ * wave at once. A failure fails the call once its whole wave has settled —
383
+ * never with a sibling build still running behind the caller's back.
384
+ */
385
+ private inWaves;
386
+ /** Build `names` (those that declare a build) in waves. MUST run under treeLocked. */
387
+ private runBuilds;
388
+ /**
389
+ * Run one service's build in the worktree. MUST run under treeLocked. Its
390
+ * output goes to `<service>.build.log` in the environment's log directory
391
+ * (decision 0038), replaced by each build; `logDir` is undefined only for a
392
+ * `warm` with no environment.
393
+ */
394
+ private runServiceBuild;
395
+ /**
396
+ * Whether a service's build must run now (decision 0038), and the key a
397
+ * successful run is recorded under. A `when:` build runs only when its
398
+ * inputs changed since its last successful build in this worktree; a
399
+ * string build always runs on a bind, and on a wake only when it never
400
+ * succeeded here (a wake resumes what ran; `up` applies changes).
401
+ */
402
+ private buildDecision;
403
+ private decideBuild;
404
+ /** Build one service if its build is due; record a success. MUST run under treeLocked. */
405
+ private buildService;
406
+ /** Would this service's build run now? (Recorded in `decided` for the build that follows.) */
407
+ private buildWillRun;
408
+ /**
409
+ * A stat print of a service's declared outputs, as the build ledger keeps
410
+ * it: undefined when it declares none, '' when none exist on disk.
411
+ */
412
+ private outputsPrint;
413
+ /** The upkeep rules' output of this bind, in `upkeep.build.log` (decision 0038), begun by the first rule that runs. */
414
+ private upkeepLog;
415
+ /**
416
+ * Ensure the stack's appliances (decision 0018), remembering the ones runly
417
+ * started for the budget. A started one is counted as running from now on,
418
+ * so its share leaves the caller's reservation.
419
+ */
420
+ private ensureAppliances;
421
+ /**
422
+ * Forget appliances that are no longer up (stopped by hand, crashed, a
423
+ * reboot): their cost stayed committed for the rest of the daemon's life.
424
+ * Re-probed every sweep — a TCP connect per appliance runly started.
425
+ */
426
+ private pruneStartedAppliances;
427
+ /** Declared (or default) run resources of everything runly runs now, across every environment. */
428
+ private committedRunning;
429
+ /**
430
+ * What starting `starts` and building `builds` would cost (decision 0036),
431
+ * item by item: run resources per start, build resources per build that
432
+ * will actually run, and run resources per appliance that is not up.
433
+ */
434
+ private needFor;
435
+ /**
436
+ * Start `only` (a subset of `active`) in dependency order, readiness-gated,
437
+ * fatal-log fast-fail. The other members of `active` are already running,
438
+ * so they count as started for depends_on — and so does a dependency the
439
+ * lease took `down` on purpose (it is not wanted; the dependent reaches its
440
+ * stable public port). On a failure a bind stops everything (`stop-all`); a
441
+ * wake stops only what it started (`stop-these`). In a LEASED environment a
442
+ * bind's service that fails to boot on its own account (a work-error: it
443
+ * exited, hit a fatal log marker, daemonized) is instead stopped and marked
444
+ * failed like a runtime crash loop (decision 0039, 0.18.1); its dependents
445
+ * are not started, everything else is. Returns those dependents.
446
+ */
447
+ private startServices;
448
+ /**
449
+ * Who the caller is when it names no holder: its WORKTREE — the directory
450
+ * holding the manifest, found by searching upward — whichever subdirectory
451
+ * the verb runs from. `up` at the root and `ctx --env` from `src/app` are
452
+ * the same caller. A live lease an older runly keyed to a subdirectory is
453
+ * still honoured from that subdirectory until it ends.
454
+ */
455
+ private callerHolder;
456
+ up(opts: UpOptions): Promise<{
457
+ previewNotice: string | undefined;
458
+ bindDiagnostics: BindDiagnostics;
459
+ stack: string;
460
+ envId: string;
461
+ state: import("../core/types.js").EnvState;
462
+ /**
463
+ * `holderPid`: the process the lease is tethered to (decision 0035) —
464
+ * while it lives the lease does not expire, so `expiresAt` is only the
465
+ * renewal horizon. null for a lease that ends at `expiresAt`.
466
+ */
467
+ lease: {
468
+ id: string;
469
+ kind: "session";
470
+ hygiene: Hygiene;
471
+ expiresAt: number;
472
+ holderPid: number | null;
473
+ } | null;
474
+ urls: Record<string, string>;
475
+ /** The environment's PUBLIC ports by manifest key (`runly ctx --env` → RUNLY_PORT_<KEY>). */
476
+ ports: {
477
+ [x: string]: number;
478
+ };
479
+ /**
480
+ * The proxy in front of each public port, by port key (decision 0033):
481
+ * state, client→server bytes and the last client byte's time. runly's
482
+ * own readiness probes bypass it and are never counted.
483
+ */
484
+ proxy: Record<string, import("./proxy.js").ProxyStats>;
485
+ previewUrls: Record<string, string>;
486
+ /**
487
+ * A preview the daemon re-publishes after a restart (decision 0045):
488
+ * `restoring`, or `failed` with the error; null when there is none.
489
+ */
490
+ previewRestore: {
491
+ error?: string | undefined;
492
+ service: string;
493
+ url: string;
494
+ state: "failed" | "restoring";
495
+ } | null;
496
+ /**
497
+ * Every declared service and where it stands (decision 0034): `running`,
498
+ * `stopped` (wanted, but not running — a quiesce or a daemon restart; the
499
+ * next `up` starts it) or `down` (not wanted: never started, or taken
500
+ * down with `runly down`). `urls` lists the wanted ones.
501
+ */
502
+ services: Record<string, "down" | "failed" | "running" | "stopped">;
503
+ /** Services that crash-looped and were stopped (decision 0039), with their last exit; the next `up` retries them. */
504
+ failures: Record<string, {
505
+ reason: string;
506
+ exitCode: number | null;
507
+ signal: string | null;
508
+ at: number;
509
+ detail: string;
510
+ hint: string;
511
+ }>;
512
+ /** The last `up` failed and none succeeded since (0.19): what failed; null otherwise. */
513
+ lastUpFailed: {
514
+ at: number | null;
515
+ class: string | null;
516
+ message: string;
517
+ source: string | null;
518
+ hint: string;
519
+ } | null;
520
+ /**
521
+ * `logins` stays the PRIMARY login even for a stack that declares a list, so
522
+ * a consumer reading `ctx.logins.user` is unaffected by the manifest growing
523
+ * more of them; `allLogins` carries the full set. Both are null/omitted when
524
+ * the stack declares none.
525
+ */
526
+ logins: import("../core/manifest.js").Login | null;
527
+ allLogins: import("../core/manifest.js").Login[];
528
+ /**
529
+ * The manifest's INTERNAL hook, templated and run inside the leased
530
+ * environment — not something to run by hand. It still carries its
531
+ * `{{role}}` placeholder, and run from a worktree it signs with the wrong
532
+ * key, so a token minted that way comes back 401 and reads as a
533
+ * permissions problem. `tokenVia` is the supported path.
534
+ */
535
+ tokenCommand: string | null;
536
+ tokenVia: string | null;
537
+ /**
538
+ * Every datastore exists for the environment's whole life (decision
539
+ * 0034). `preset` is what it holds now — the last preset restored into
540
+ * it — or null before its first restore. `runly ctx --env` exports it as
541
+ * RUNLY_DATASTORE_<NAME>_PRESET.
542
+ */
543
+ datastores: {
544
+ [k: string]: {
545
+ url: string;
546
+ ns: string;
547
+ preset: string | null;
548
+ };
549
+ };
550
+ events: import("./supervisor.js").ServiceEvent[];
551
+ }>;
552
+ /**
553
+ * The environment a lease points to. leaseForHolder JOINs envs, so the row
554
+ * existed when the lease was resolved — but a concurrent forced teardown can
555
+ * delete it before this read (deleteEnv is one transaction now, so a torn
556
+ * write can no longer leave a lease naming a deleted row; journals from
557
+ * before that change still can, and the sweeper prunes those). Every verb
558
+ * that asserted `getEnv(lease.envId)!` used to crash with an unclassified
559
+ * TypeError instead of telling the caller what to do about it.
560
+ */
561
+ private envForLease;
562
+ ctx(cwd: string, holder?: string, envId?: string): {
563
+ stack: string;
564
+ envId: string;
565
+ state: import("../core/types.js").EnvState;
566
+ /**
567
+ * `holderPid`: the process the lease is tethered to (decision 0035) —
568
+ * while it lives the lease does not expire, so `expiresAt` is only the
569
+ * renewal horizon. null for a lease that ends at `expiresAt`.
570
+ */
571
+ lease: {
572
+ id: string;
573
+ kind: "session";
574
+ hygiene: Hygiene;
575
+ expiresAt: number;
576
+ holderPid: number | null;
577
+ } | null;
578
+ urls: Record<string, string>;
579
+ /** The environment's PUBLIC ports by manifest key (`runly ctx --env` → RUNLY_PORT_<KEY>). */
580
+ ports: {
581
+ [x: string]: number;
582
+ };
583
+ /**
584
+ * The proxy in front of each public port, by port key (decision 0033):
585
+ * state, client→server bytes and the last client byte's time. runly's
586
+ * own readiness probes bypass it and are never counted.
587
+ */
588
+ proxy: Record<string, import("./proxy.js").ProxyStats>;
589
+ previewUrls: Record<string, string>;
590
+ /**
591
+ * A preview the daemon re-publishes after a restart (decision 0045):
592
+ * `restoring`, or `failed` with the error; null when there is none.
593
+ */
594
+ previewRestore: {
595
+ error?: string | undefined;
596
+ service: string;
597
+ url: string;
598
+ state: "failed" | "restoring";
599
+ } | null;
600
+ /**
601
+ * Every declared service and where it stands (decision 0034): `running`,
602
+ * `stopped` (wanted, but not running — a quiesce or a daemon restart; the
603
+ * next `up` starts it) or `down` (not wanted: never started, or taken
604
+ * down with `runly down`). `urls` lists the wanted ones.
605
+ */
606
+ services: Record<string, "down" | "failed" | "running" | "stopped">;
607
+ /** Services that crash-looped and were stopped (decision 0039), with their last exit; the next `up` retries them. */
608
+ failures: Record<string, {
609
+ reason: string;
610
+ exitCode: number | null;
611
+ signal: string | null;
612
+ at: number;
613
+ detail: string;
614
+ hint: string;
615
+ }>;
616
+ /** The last `up` failed and none succeeded since (0.19): what failed; null otherwise. */
617
+ lastUpFailed: {
618
+ at: number | null;
619
+ class: string | null;
620
+ message: string;
621
+ source: string | null;
622
+ hint: string;
623
+ } | null;
624
+ /**
625
+ * `logins` stays the PRIMARY login even for a stack that declares a list, so
626
+ * a consumer reading `ctx.logins.user` is unaffected by the manifest growing
627
+ * more of them; `allLogins` carries the full set. Both are null/omitted when
628
+ * the stack declares none.
629
+ */
630
+ logins: import("../core/manifest.js").Login | null;
631
+ allLogins: import("../core/manifest.js").Login[];
632
+ /**
633
+ * The manifest's INTERNAL hook, templated and run inside the leased
634
+ * environment — not something to run by hand. It still carries its
635
+ * `{{role}}` placeholder, and run from a worktree it signs with the wrong
636
+ * key, so a token minted that way comes back 401 and reads as a
637
+ * permissions problem. `tokenVia` is the supported path.
638
+ */
639
+ tokenCommand: string | null;
640
+ tokenVia: string | null;
641
+ /**
642
+ * Every datastore exists for the environment's whole life (decision
643
+ * 0034). `preset` is what it holds now — the last preset restored into
644
+ * it — or null before its first restore. `runly ctx --env` exports it as
645
+ * RUNLY_DATASTORE_<NAME>_PRESET.
646
+ */
647
+ datastores: {
648
+ [k: string]: {
649
+ url: string;
650
+ ns: string;
651
+ preset: string | null;
652
+ };
653
+ };
654
+ events: import("./supervisor.js").ServiceEvent[];
655
+ };
656
+ /**
657
+ * Where each declared service stands: running, failed (wanted, crash-looped
658
+ * and stopped — decision 0039), stopped (wanted, not running) or down (not
659
+ * wanted).
660
+ */
661
+ private serviceStates;
662
+ /**
663
+ * The last `up` of `env`, when it failed and none has succeeded since
664
+ * (0.19): what failed, and that `up` redoes it — neither a connection nor a
665
+ * verb starts its services meanwhile. After a daemon restart only the fact
666
+ * survives (the journal's fail streak), not the message.
667
+ */
668
+ private bindFailureOf;
669
+ /** The failed services of `env` (decision 0039): how each last ended, and the log to read. */
670
+ private serviceFailures;
671
+ /**
672
+ * `runly warm`: run THIS worktree's due upkeep rules and its services'
673
+ * `build:` steps, with no lease and no services (decision 0032). The intended
674
+ * caller is an idle pool slot that was just moved to a new commit
675
+ * (`git checkout <sha> && runly warm`): the installs are done and the build
676
+ * tools' own incremental state is current, so the next agent's bind finds
677
+ * the upkeep fresh and its builds near no-ops.
678
+ *
679
+ * Upkeep rules go through the same worktree ledger a bind reads, so a rule
680
+ * warm ran is a rule the next bind skips. Builds are not recorded anywhere:
681
+ * warm runs every one, and the next bind runs them again — cheaply, because
682
+ * the build tool finds its output current. It holds the environment lock of
683
+ * this worktree (if it has one: a check must not run while its output
684
+ * changes underneath it) and then the worktree lock — the same order every
685
+ * bind uses, so it can wait but never deadlock.
686
+ *
687
+ * What it cannot do without an environment it reports as skipped: a build
688
+ * line that templates an environment's ports or datastores has no values to
689
+ * run with, and `@` built-ins act on an environment's data.
690
+ */
691
+ warm(cwd: string, onProgress?: Progress): Promise<{
692
+ ok: boolean;
693
+ stack: string;
694
+ root: string;
695
+ steps: (({
696
+ kind: "upkeep";
697
+ } & UpkeepStep) | {
698
+ kind: "build";
699
+ service: string;
700
+ status: "ran" | "skipped" | "failed";
701
+ durationMs: number;
702
+ reason?: string;
703
+ })[];
704
+ failure: null;
705
+ durationMs: number;
706
+ }>;
707
+ resetData(cwd: string, holder?: string, onProgress?: Progress, presets?: unknown): Promise<{
708
+ previewNotice: string | undefined;
709
+ bindDiagnostics: BindDiagnostics;
710
+ stack: string;
711
+ envId: string;
712
+ state: import("../core/types.js").EnvState;
713
+ /**
714
+ * `holderPid`: the process the lease is tethered to (decision 0035) —
715
+ * while it lives the lease does not expire, so `expiresAt` is only the
716
+ * renewal horizon. null for a lease that ends at `expiresAt`.
717
+ */
718
+ lease: {
719
+ id: string;
720
+ kind: "session";
721
+ hygiene: Hygiene;
722
+ expiresAt: number;
723
+ holderPid: number | null;
724
+ } | null;
725
+ urls: Record<string, string>;
726
+ /** The environment's PUBLIC ports by manifest key (`runly ctx --env` → RUNLY_PORT_<KEY>). */
727
+ ports: {
728
+ [x: string]: number;
729
+ };
730
+ /**
731
+ * The proxy in front of each public port, by port key (decision 0033):
732
+ * state, client→server bytes and the last client byte's time. runly's
733
+ * own readiness probes bypass it and are never counted.
734
+ */
735
+ proxy: Record<string, import("./proxy.js").ProxyStats>;
736
+ previewUrls: Record<string, string>;
737
+ /**
738
+ * A preview the daemon re-publishes after a restart (decision 0045):
739
+ * `restoring`, or `failed` with the error; null when there is none.
740
+ */
741
+ previewRestore: {
742
+ error?: string | undefined;
743
+ service: string;
744
+ url: string;
745
+ state: "failed" | "restoring";
746
+ } | null;
747
+ /**
748
+ * Every declared service and where it stands (decision 0034): `running`,
749
+ * `stopped` (wanted, but not running — a quiesce or a daemon restart; the
750
+ * next `up` starts it) or `down` (not wanted: never started, or taken
751
+ * down with `runly down`). `urls` lists the wanted ones.
752
+ */
753
+ services: Record<string, "down" | "failed" | "running" | "stopped">;
754
+ /** Services that crash-looped and were stopped (decision 0039), with their last exit; the next `up` retries them. */
755
+ failures: Record<string, {
756
+ reason: string;
757
+ exitCode: number | null;
758
+ signal: string | null;
759
+ at: number;
760
+ detail: string;
761
+ hint: string;
762
+ }>;
763
+ /** The last `up` failed and none succeeded since (0.19): what failed; null otherwise. */
764
+ lastUpFailed: {
765
+ at: number | null;
766
+ class: string | null;
767
+ message: string;
768
+ source: string | null;
769
+ hint: string;
770
+ } | null;
771
+ /**
772
+ * `logins` stays the PRIMARY login even for a stack that declares a list, so
773
+ * a consumer reading `ctx.logins.user` is unaffected by the manifest growing
774
+ * more of them; `allLogins` carries the full set. Both are null/omitted when
775
+ * the stack declares none.
776
+ */
777
+ logins: import("../core/manifest.js").Login | null;
778
+ allLogins: import("../core/manifest.js").Login[];
779
+ /**
780
+ * The manifest's INTERNAL hook, templated and run inside the leased
781
+ * environment — not something to run by hand. It still carries its
782
+ * `{{role}}` placeholder, and run from a worktree it signs with the wrong
783
+ * key, so a token minted that way comes back 401 and reads as a
784
+ * permissions problem. `tokenVia` is the supported path.
785
+ */
786
+ tokenCommand: string | null;
787
+ tokenVia: string | null;
788
+ /**
789
+ * Every datastore exists for the environment's whole life (decision
790
+ * 0034). `preset` is what it holds now — the last preset restored into
791
+ * it — or null before its first restore. `runly ctx --env` exports it as
792
+ * RUNLY_DATASTORE_<NAME>_PRESET.
793
+ */
794
+ datastores: {
795
+ [k: string]: {
796
+ url: string;
797
+ ns: string;
798
+ preset: string | null;
799
+ };
800
+ };
801
+ events: import("./supervisor.js").ServiceEvent[];
802
+ }>;
803
+ /**
804
+ * Re-read an environment INSIDE its lock and refuse work on one that is being
805
+ * torn down.
806
+ *
807
+ * Teardown claims the row under the pool lock and then runs slowly outside the
808
+ * env lock, so a request that resolved its lease before the claim can arrive
809
+ * here afterwards and operate on a tree that is about to be deleted (or
810
+ * already is). bindAndStart already re-checks; exec and token
811
+ * phase did not.
812
+ */
813
+ /**
814
+ * Record that an environment was USED, without extending its lease.
815
+ *
816
+ * Activity is what keeps services awake (decision 0035), so only verbs that
817
+ * USE the environment call this (decision 0039): `up` and `reset-data` (a
818
+ * bind sets the clock itself), `exec`, `token`, `preview` and `down`. The
819
+ * read-only verbs — `ctx`, `ps`, `plan`, `logs`, `status`, `db ls`,
820
+ * `doctor` — do not: an agent polling them is not using a service, and
821
+ * counting them kept idle services running for as long as anyone looked.
822
+ */
823
+ private touch;
824
+ /** `resumable`: the caller starts idle-stopped services itself (resumeForVerb) instead of being refused. */
825
+ private assertUsable;
826
+ /**
827
+ * What `runly exec` runs with (decision 0040): the environment's variables,
828
+ * with idle-stopped services (and, after a daemon restart, every wanted
829
+ * service) started first and the activity clock touched. The CLI spawns the
830
+ * command itself — inherited stdio, the caller's own environment, the real
831
+ * exit code — so nothing here holds the environment's lock while it runs.
832
+ */
833
+ execEnv(cwd: string, holder?: string): Promise<{
834
+ envId: string;
835
+ root: string;
836
+ vars: {
837
+ [x: string]: string;
838
+ };
839
+ }>;
840
+ /** A running `runly exec` is activity (decision 0039): the CLI calls this while the command runs. */
841
+ execTouch(envId: string): {
842
+ touched: boolean;
843
+ };
844
+ /** Resolve auth.token with {{role}} and run it in the worktree the environment runs in. */
845
+ token(cwd: string, role: string, holder?: string): Promise<{
846
+ token: string;
847
+ role: string;
848
+ }>;
849
+ /**
850
+ * Where this worktree's environment keeps the logs of `services` (decision
851
+ * 0038) — all of them when none are named. The CLI reads, filters and
852
+ * follows the files itself; the daemon only resolves which ones. With
853
+ * `build`, the build output of each service's last build, and the upkeep
854
+ * output of the last `up` that ran a rule.
855
+ */
856
+ logsSpec(cwd: string, services: string[], build: boolean, holder?: string): {
857
+ envId: string;
858
+ dir: string;
859
+ build: boolean;
860
+ files: {
861
+ service: string;
862
+ file: string;
863
+ }[];
864
+ };
865
+ /** The pre-0.16 single-service read, kept for RPC clients: the last `lines` lines, without stamps. */
866
+ logs(cwd: string, service: string, lines: number, holder?: string): {
867
+ service: string;
868
+ lines: string;
869
+ };
870
+ /**
871
+ * Hand the environment back. A no-op answer must say WHY.
872
+ *
873
+ * A lease is keyed by holder NAME, which defaults to the worktree path — so
874
+ * releasing from a different directory than the one that bound silently
875
+ * matches nothing. `{released: false}` on its own gave an agent no way to tell
876
+ * that apart from "already released", and the reported behaviour was simply to
877
+ * give up and let the environment linger until its TTL. Name the mismatch, and
878
+ * list who does hold this stack's leases.
879
+ */
880
+ release(cwd: string, holder?: string): Promise<{
881
+ released: boolean;
882
+ holder: string;
883
+ reason: string;
884
+ otherHolders: string[];
885
+ envId?: undefined;
886
+ } | {
887
+ released: boolean;
888
+ envId: string;
889
+ holder?: undefined;
890
+ reason?: undefined;
891
+ otherHolders?: undefined;
892
+ }>;
893
+ private previewPublisherName;
894
+ private assertPreviewAllowed;
895
+ /**
896
+ * Tagged members of a verified leased preview process group. A publisher's
897
+ * wrapper can fork its tunnel child; excluding only the leader lets a scan
898
+ * select the child and kill the whole leased group. All scan consumers share
899
+ * this classification, using the same state-root-scoped process snapshot.
900
+ *
901
+ * Ownership requires the recorded leader identity AND matching env tags.
902
+ * A stale/reused leader PID, an unreadable identity, or a generic preview:
903
+ * service label cannot exempt another process. Descendants that leave the
904
+ * group (setsid) need a separate ownership mechanism and are not protected.
905
+ */
906
+ private leasedPreviewPids;
907
+ /**
908
+ * Stop and clear any lease-scoped preview tunnel recorded on this lease.
909
+ *
910
+ * Teardown may NEVER be blocked by a manifest edit: `preview.publisher` is
911
+ * re-read from disk on every call, so a name that resolved at start can be a
912
+ * typo by the time the sweeper gets here — and an unknown name throws a
913
+ * work-error. Thrown from the sweeper's lease loop that aborted every tick,
914
+ * so TTL expiry, dead-holder release and eviction stopped daemon-wide. The
915
+ * teardown path therefore falls back to the default publisher instead.
916
+ *
917
+ * Returns false when the tunnel could NOT be confirmed dead. The record is
918
+ * kept in that case (reapPids' contract): a forgotten preview pid is a
919
+ * public, unauthenticated URL nobody can ever name again.
920
+ */
921
+ private stopPreviewForLease;
922
+ /**
923
+ * What an operator can actually do about a tunnel that outlived its kill.
924
+ *
925
+ * Pointing at `pool gc` was only ever true on Linux — it is a tag scan, and
926
+ * `procScanSupported()` is Linux-only — which is precisely the platform where
927
+ * the record is NOT recoverable. Name the pid and the public URL instead, so
928
+ * the remedy survives the lease row that is usually about to be deleted.
929
+ */
930
+ private unreapedPreview;
931
+ /**
932
+ * Release a lease and reap its preview tunnel first.
933
+ *
934
+ * Re-read between the stop and the delete. `release` and the sweeper hold no
935
+ * env lock, and the stop blocks in `killGroupVerified` for up to ~4s — long
936
+ * enough for a concurrent `preview` to take the env lock, publish, and write
937
+ * its pid onto this very row. Deleting the row on the strength of the old
938
+ * snapshot then orphaned the NEW tunnel: no lease names it, `pool gc` skips
939
+ * nothing it can see, and `preview stop` has nothing left to stop. Taking the
940
+ * env lock here instead would nest it inside the pool lock (tryClaim calls
941
+ * this) and stall every claim machine-wide for the length of a bind.
942
+ */
943
+ private endLease;
944
+ /**
945
+ * Everything about an environment that decides whether publishing `service`
946
+ * is legal, and the local port to publish. Returns that port.
947
+ *
948
+ * One function because it has to run TWICE — once before queueing for the env
949
+ * lock, and again inside it against a re-read row. Every field it reads is one
950
+ * a concurrent bind rewrites.
951
+ */
952
+ private previewTarget;
953
+ /**
954
+ * Publish every recorded preview again after a restart (decision 0045): same
955
+ * publisher, the settings that pin its address, aimed at the same public
956
+ * port — which the proxy holds again, so the first request wakes the
957
+ * service as usual. A failure is an event and `ctx.previewRestore`, never
958
+ * an error of anyone's verb.
959
+ */
960
+ private restorePreviews;
961
+ private restorePreview;
962
+ /** The lease's preview is over on purpose: a restart must not bring it back (decision 0045). */
963
+ private forgetPreview;
964
+ previewStart(cwd: string, service: string, holder?: string, ttlMs?: number, httpsPort?: number): Promise<{
965
+ service: string;
966
+ url: string;
967
+ }>;
968
+ previewStop(cwd: string, holder?: string): Promise<{
969
+ stopped: boolean;
970
+ service?: string;
971
+ }>;
972
+ /** Live-probed appliance overview for the stack at cwd. */
973
+ applianceLs(cwd: string): Promise<{
974
+ stack: string;
975
+ appliances: Record<string, {
976
+ probe: string;
977
+ up: boolean;
978
+ startable: boolean;
979
+ stoppable: boolean;
980
+ }>;
981
+ }>;
982
+ /** Ensure one appliance (or all of them) — the same path a bind takes. */
983
+ applianceStart(cwd: string, name?: string): Promise<{
984
+ results: Record<string, string>;
985
+ }>;
986
+ /** Explicit stop — the only path that ever stops an appliance. */
987
+ applianceStop(cwd: string, name: string): Promise<{
988
+ stopped: string;
989
+ }>;
990
+ /**
991
+ * `runly down [service…]` (decision 0034): stop just these services — or,
992
+ * with no names, every service — and keep the lease, the data and the public
993
+ * ports. The proxy keeps listening; connections to a downed service are
994
+ * refused until an `up` brings it back. Dependents of a downed service are
995
+ * NOT stopped ("just those"); they are named in the answer, because they now
996
+ * talk to a port with nothing behind it.
997
+ */
998
+ down(opts: {
999
+ cwd: string;
1000
+ holder?: string;
1001
+ services: string[];
1002
+ onProgress?: Progress;
1003
+ }): Promise<{
1004
+ stopped: string[];
1005
+ down: string[];
1006
+ dependentsStillRunning: string[];
1007
+ previewNotice: string | undefined;
1008
+ stack: string;
1009
+ envId: string;
1010
+ state: import("../core/types.js").EnvState;
1011
+ /**
1012
+ * `holderPid`: the process the lease is tethered to (decision 0035) —
1013
+ * while it lives the lease does not expire, so `expiresAt` is only the
1014
+ * renewal horizon. null for a lease that ends at `expiresAt`.
1015
+ */
1016
+ lease: {
1017
+ id: string;
1018
+ kind: "session";
1019
+ hygiene: Hygiene;
1020
+ expiresAt: number;
1021
+ holderPid: number | null;
1022
+ } | null;
1023
+ urls: Record<string, string>;
1024
+ /** The environment's PUBLIC ports by manifest key (`runly ctx --env` → RUNLY_PORT_<KEY>). */
1025
+ ports: {
1026
+ [x: string]: number;
1027
+ };
1028
+ /**
1029
+ * The proxy in front of each public port, by port key (decision 0033):
1030
+ * state, client→server bytes and the last client byte's time. runly's
1031
+ * own readiness probes bypass it and are never counted.
1032
+ */
1033
+ proxy: Record<string, import("./proxy.js").ProxyStats>;
1034
+ previewUrls: Record<string, string>;
1035
+ /**
1036
+ * A preview the daemon re-publishes after a restart (decision 0045):
1037
+ * `restoring`, or `failed` with the error; null when there is none.
1038
+ */
1039
+ previewRestore: {
1040
+ error?: string | undefined;
1041
+ service: string;
1042
+ url: string;
1043
+ state: "failed" | "restoring";
1044
+ } | null;
1045
+ /**
1046
+ * Every declared service and where it stands (decision 0034): `running`,
1047
+ * `stopped` (wanted, but not running — a quiesce or a daemon restart; the
1048
+ * next `up` starts it) or `down` (not wanted: never started, or taken
1049
+ * down with `runly down`). `urls` lists the wanted ones.
1050
+ */
1051
+ services: Record<string, "down" | "failed" | "running" | "stopped">;
1052
+ /** Services that crash-looped and were stopped (decision 0039), with their last exit; the next `up` retries them. */
1053
+ failures: Record<string, {
1054
+ reason: string;
1055
+ exitCode: number | null;
1056
+ signal: string | null;
1057
+ at: number;
1058
+ detail: string;
1059
+ hint: string;
1060
+ }>;
1061
+ /** The last `up` failed and none succeeded since (0.19): what failed; null otherwise. */
1062
+ lastUpFailed: {
1063
+ at: number | null;
1064
+ class: string | null;
1065
+ message: string;
1066
+ source: string | null;
1067
+ hint: string;
1068
+ } | null;
1069
+ /**
1070
+ * `logins` stays the PRIMARY login even for a stack that declares a list, so
1071
+ * a consumer reading `ctx.logins.user` is unaffected by the manifest growing
1072
+ * more of them; `allLogins` carries the full set. Both are null/omitted when
1073
+ * the stack declares none.
1074
+ */
1075
+ logins: import("../core/manifest.js").Login | null;
1076
+ allLogins: import("../core/manifest.js").Login[];
1077
+ /**
1078
+ * The manifest's INTERNAL hook, templated and run inside the leased
1079
+ * environment — not something to run by hand. It still carries its
1080
+ * `{{role}}` placeholder, and run from a worktree it signs with the wrong
1081
+ * key, so a token minted that way comes back 401 and reads as a
1082
+ * permissions problem. `tokenVia` is the supported path.
1083
+ */
1084
+ tokenCommand: string | null;
1085
+ tokenVia: string | null;
1086
+ /**
1087
+ * Every datastore exists for the environment's whole life (decision
1088
+ * 0034). `preset` is what it holds now — the last preset restored into
1089
+ * it — or null before its first restore. `runly ctx --env` exports it as
1090
+ * RUNLY_DATASTORE_<NAME>_PRESET.
1091
+ */
1092
+ datastores: {
1093
+ [k: string]: {
1094
+ url: string;
1095
+ ns: string;
1096
+ preset: string | null;
1097
+ };
1098
+ };
1099
+ events: import("./supervisor.js").ServiceEvent[];
1100
+ }>;
1101
+ /** In-flight creations and drops of database copies; the reaper never touches these. */
1102
+ private dbBusy;
1103
+ /** What `db new` / `db ls` / `ps` show for one copy. */
1104
+ private dbCopyView;
1105
+ /**
1106
+ * Is a recorded holder process gone? One rule for leases and database copies
1107
+ * alike (decision 0034: copies are reaped exactly like environments), so a
1108
+ * grace period added here applies to both.
1109
+ */
1110
+ private holderGone;
1111
+ /**
1112
+ * `runly db new <datastore>` (decision 0034): a fresh copy of the datastore,
1113
+ * restored from the SAME template the environments use (baked first if it is
1114
+ * missing), outside any environment — no lease, no ports, no services. Many
1115
+ * may exist at once. The row is journalled BEFORE the restore, as
1116
+ * `creating`, so a daemon that dies mid-restore leaves a record the next one
1117
+ * reaps.
1118
+ */
1119
+ dbNew(opts: {
1120
+ cwd: string;
1121
+ datastore: string;
1122
+ preset?: string;
1123
+ holder?: string;
1124
+ holderPid?: number;
1125
+ onProgress?: Progress;
1126
+ }): Promise<{
1127
+ name: string;
1128
+ datastore: string;
1129
+ preset: string;
1130
+ url: string;
1131
+ ns: string;
1132
+ state: "ready" | "creating" | "dropping";
1133
+ worktree: string;
1134
+ holder: string;
1135
+ holderPid: number | null;
1136
+ /** null = no agent tether was given: the copy lives until its worktree goes or it is dropped. */
1137
+ holderAlive: boolean | null;
1138
+ createdAt: number;
1139
+ }>;
1140
+ /**
1141
+ * `runly db with` records the command it runs against a copy (decision
1142
+ * 0039), so a drop can take that command down first when its CLI died.
1143
+ */
1144
+ dbAttach(name: string, child: {
1145
+ pid: number;
1146
+ start?: number;
1147
+ pgid?: number;
1148
+ }): {
1149
+ attached: string;
1150
+ };
1151
+ /**
1152
+ * Stop whatever still runs against copy `row` (decision 0039): the command
1153
+ * `runly db with` recorded — its process group when it has its own, else
1154
+ * the verified pid — and, on Linux, every process carrying the copy's tag.
1155
+ * Only a pid whose start time still matches is signalled. Returns how many
1156
+ * processes were stopped.
1157
+ */
1158
+ private stopCopyUsers;
1159
+ /** Is `path` a copy's own directory under the state root (never anything else)? */
1160
+ private isPrivateDbDir;
1161
+ /**
1162
+ * Drop one copy with what its row recorded — the command (run in the
1163
+ * worktree, or the state root once that is gone) or its private directory.
1164
+ * Needs neither the manifest nor the worktree. A failed drop keeps the row
1165
+ * (`dropping`) with a backoff, so the reaper retries instead of forgetting a
1166
+ * database nothing else can name. Callers hold `dbBusy` for the name.
1167
+ */
1168
+ private dropDbCopy;
1169
+ /** `runly db drop <name>`: drop one copy now, whoever made it. */
1170
+ dbDrop(name: string): Promise<{
1171
+ dropped: string;
1172
+ }>;
1173
+ /** `runly db ls`: this worktree's copies, or every copy with `all`. */
1174
+ dbLs(cwd: string, all: boolean): {
1175
+ scope: string;
1176
+ copies: {
1177
+ name: string;
1178
+ datastore: string;
1179
+ preset: string;
1180
+ url: string;
1181
+ ns: string;
1182
+ state: "ready" | "creating" | "dropping";
1183
+ worktree: string;
1184
+ holder: string;
1185
+ holderPid: number | null;
1186
+ /** null = no agent tether was given: the copy lives until its worktree goes or it is dropped. */
1187
+ holderAlive: boolean | null;
1188
+ createdAt: number;
1189
+ }[];
1190
+ };
1191
+ /** The caller's worktree root, for the per-worktree views (`ps`, `db ls`). */
1192
+ private callerWorktree;
1193
+ /** Why a copy must go now, or null (decision 0034: reaped exactly like environments; no TTL). */
1194
+ private dbCopyOrphanReason;
1195
+ /** Drop every copy whose holder or worktree is gone. Runs in every sweep and after recovery. */
1196
+ private reapDbCopies;
1197
+ /** Resident memory per (env, service), summed over its tagged processes. Linux only; empty elsewhere. */
1198
+ private rssByService;
1199
+ /**
1200
+ * `runly ps` (decision 0034): one view of what runs for the caller — the
1201
+ * services of this worktree's environment and its database copies — or,
1202
+ * with `all`, for the whole server.
1203
+ */
1204
+ ps(cwd: string, all: boolean): {
1205
+ scope: string;
1206
+ services: Record<string, unknown>[];
1207
+ databases: {
1208
+ name: string;
1209
+ datastore: string;
1210
+ preset: string;
1211
+ url: string;
1212
+ ns: string;
1213
+ state: "ready" | "creating" | "dropping";
1214
+ worktree: string;
1215
+ holder: string;
1216
+ holderPid: number | null;
1217
+ /** null = no agent tether was given: the copy lives until its worktree goes or it is dropped. */
1218
+ holderAlive: boolean | null;
1219
+ createdAt: number;
1220
+ }[];
1221
+ /** Environments whose last `up` failed (0.19), with what failed. */
1222
+ failedUps: Record<string, unknown>[];
1223
+ /** The server-wide load budget (decision 0036). */
1224
+ budget: {
1225
+ enabled: boolean;
1226
+ memoryBytes: number;
1227
+ cpu: number;
1228
+ committedMemoryBytes: number;
1229
+ committedCpu: number;
1230
+ waiting: number;
1231
+ };
1232
+ };
1233
+ /**
1234
+ * `runly plan [service…]` (decision 0036): what an `up` with these
1235
+ * services would build and start now, what that costs (declared, or the
1236
+ * default, said so), what the box and the budget have free, and whether it
1237
+ * would start now or wait — without doing any of it.
1238
+ */
1239
+ plan(cwd: string, services: string[], rebuild: boolean, holder?: string): Promise<{
1240
+ stack: string;
1241
+ envId: string | null;
1242
+ items: {
1243
+ kind: "start" | "appliance" | "build";
1244
+ name: string;
1245
+ memoryBytes: number;
1246
+ cpu: number;
1247
+ declared: boolean;
1248
+ skipped: string | null;
1249
+ }[];
1250
+ need: {
1251
+ memoryBytes: number;
1252
+ cpu: number;
1253
+ start: {
1254
+ memoryBytes: number;
1255
+ cpu: number;
1256
+ };
1257
+ build: {
1258
+ memoryBytes: number;
1259
+ cpu: number;
1260
+ };
1261
+ };
1262
+ machine: import("./budget.js").Machine;
1263
+ budget: {
1264
+ enabled: boolean;
1265
+ memoryBytes: number;
1266
+ cpu: number;
1267
+ reserveBytes: number;
1268
+ loadPerCore: number;
1269
+ waitMs: number;
1270
+ };
1271
+ committed: Committed;
1272
+ queue: number;
1273
+ startsNow: boolean;
1274
+ verdict: string;
1275
+ defaultsAssumed: string[];
1276
+ }>;
1277
+ /**
1278
+ * `runly destroy` (decision 0035): everything runly holds for this worktree
1279
+ * goes now — its environment (services, data, preview, ports, lease) and its
1280
+ * database copies. What a worktree pool calls when it takes a worktree back;
1281
+ * the sweeper does the same by itself when the worktree is removed or its
1282
+ * agent's tether dies. The baked templates and the build records stay
1283
+ * (decision 0039; a build's `outputs:` are checked before it is skipped).
1284
+ * Of the upkeep ledger only the rules that declare `outputs:` stay: the pool
1285
+ * usually resets the worktree next (`git clean -fdx`), which removes what
1286
+ * the other rules produced, and nothing else could tell.
1287
+ */
1288
+ destroy(cwd: string): Promise<{
1289
+ worktree: string;
1290
+ environments: string[];
1291
+ copies: string[];
1292
+ copiesNotDropped: string[];
1293
+ upkeepForgotten: number;
1294
+ }>;
1295
+ /**
1296
+ * `runly pool doctor [--fix]` (decision 0037): what this state root owns
1297
+ * that nothing references any more. Dry run by default; `--fix` removes it.
1298
+ * Only runly's own: files under the state root in runly's naming,
1299
+ * processes carrying this state root's tag, listeners of this daemon, and
1300
+ * server-side namespaces a datastore's `list:` hook reports that carry
1301
+ * runly's naming AND a stack id this state root has a record of.
1302
+ */
1303
+ poolDoctor(cwd: string, fix: boolean): Promise<{
1304
+ fix: boolean;
1305
+ clean: boolean;
1306
+ findings: {
1307
+ kind: string;
1308
+ what: string;
1309
+ detail: string;
1310
+ fixed?: boolean;
1311
+ error?: string;
1312
+ }[];
1313
+ }>;
1314
+ status(): {
1315
+ pid: number;
1316
+ /**
1317
+ * Who restarts this daemon when it crashes (decision 0039): `systemd` or
1318
+ * `launchd` after `runly daemon install` (the unit sets
1319
+ * RUNLY_SUPERVISOR), `autospawn` (the next CLI command) otherwise. Not
1320
+ * INVOCATION_ID: a daemon autospawned from anything under ANY systemd
1321
+ * service inherits that.
1322
+ */
1323
+ supervisor: string;
1324
+ envs: {
1325
+ id: string;
1326
+ stack: string;
1327
+ state: import("../core/types.js").EnvState;
1328
+ ports: Record<string, number>;
1329
+ bindCount: number;
1330
+ /** Proxy state and traffic per public port (decision 0033). */
1331
+ proxy: Record<string, import("./proxy.js").ProxyStats>;
1332
+ /** Where its services run: the caller's worktree (decision 0032). */
1333
+ worktree: string;
1334
+ lease: LeaseRow | null;
1335
+ /** null = every service (the default set); otherwise the services the lease wants up — [] after `runly down` (decision 0034). */
1336
+ activeServices: string[] | null;
1337
+ idleMs: number;
1338
+ /** null = the holder never identified itself, so liveness is unknowable. */
1339
+ holderAlive: boolean | null;
1340
+ /** Why this environment is still holding its services, in one word. */
1341
+ heat: string;
1342
+ /**
1343
+ * Will the next bind take this environment as-is?
1344
+ *
1345
+ * `heat: 'cold'` means quiesced — services stopped, everything else
1346
+ * intact — which is a HEALTHY free pool entry, not a stuck one. Read as
1347
+ * "dead", it sent an operator to `pool recycle` to unstick a pool that
1348
+ * was already fine, and that recycle took other people's live leases
1349
+ * with it. So state the conclusion instead of leaving it to be inferred.
1350
+ */
1351
+ available: boolean;
1352
+ /** Plain language for the two fields above, so no one has to infer it. */
1353
+ summary: string;
1354
+ }[];
1355
+ poolMaxTotal: number;
1356
+ ports: {
1357
+ public: import("../core/ports.js").PortBlock;
1358
+ internal: import("../core/ports.js").PortBlock;
1359
+ tunnel: import("../core/ports.js").PortBlock;
1360
+ ephemeral: import("../core/ports.js").PortBlock;
1361
+ conflicts: string[];
1362
+ };
1363
+ /** The server-wide load budget (decision 0036). */
1364
+ budget: {
1365
+ committedMemoryBytes: number;
1366
+ committedCpu: number;
1367
+ committed: {
1368
+ what: string;
1369
+ memoryBytes: number;
1370
+ cpu: number;
1371
+ }[];
1372
+ waiting: number;
1373
+ enabled: boolean;
1374
+ memoryBytes: number;
1375
+ cpu: number;
1376
+ reserveBytes: number;
1377
+ cpuPressure: number;
1378
+ loadPerCore: number;
1379
+ waitMs: number;
1380
+ maxQueue: number;
1381
+ defaultRun: {
1382
+ memoryBytes: number;
1383
+ cpu: number;
1384
+ };
1385
+ defaultBuild: {
1386
+ memoryBytes: number;
1387
+ cpu: number;
1388
+ };
1389
+ };
1390
+ /** Client bytes every proxied connection holds for a replay right now; capped by BACKLOT_PROXY_REPLAY_CAP_BYTES (0.19, default 64 MiB). */
1391
+ proxyReplayCarriedBytes: number;
1392
+ events: import("../core/events.js").DaemonEvent[];
1393
+ };
1394
+ /** Who a restart would make rebind: every live lease, named. */
1395
+ leaseHolders(): Array<{
1396
+ envId: string;
1397
+ holder: string;
1398
+ kind: string;
1399
+ }>;
1400
+ /**
1401
+ * What a `runly update` restart would cost, without doing it.
1402
+ *
1403
+ * Reported rather than inferred by the CLI because the two facts that decide
1404
+ * whether a restart is safe — which environments are BUSY, and who holds a
1405
+ * lease — only exist in the daemon's memory and journal.
1406
+ */
1407
+ updatePlan(cliVersion?: string): {
1408
+ daemon: string;
1409
+ daemonBuild: string;
1410
+ cli: string | null;
1411
+ daemonPid: number;
1412
+ journalSchema: number;
1413
+ skew: import("../core/version.js").VersionSkew | null;
1414
+ busy: string[];
1415
+ leases: {
1416
+ envId: string;
1417
+ holder: string;
1418
+ kind: string;
1419
+ }[];
1420
+ };
1421
+ /**
1422
+ * Refuse a restart that would destroy work, and allow the one that costs only
1423
+ * a rebind.
1424
+ *
1425
+ * BUSY is the refusal. An `exec` runs detached so the command itself
1426
+ * survives the daemon, but the CALLER is blocked on this socket waiting for
1427
+ * its result — restarting hands it a dead connection and no result. Every other reclaim path already treats busy as inviolable
1428
+ * (claimForTeardown, both sweeper branches, pool gc, and shutdown's own reap
1429
+ * since 0.8.0), so this one does too.
1430
+ *
1431
+ * A LIVE LEASE is deliberately NOT a refusal. A restart stops services, keeps
1432
+ * the lease, and the holder's next verb rebinds — which is exactly what the
1433
+ * idle quiesce already does to leased environments without anyone's consent,
1434
+ * and what assertUsable already tells the holder to do ("run 'runly up' to
1435
+ * rebind"). Refusing here would mean an update on any busy shared box always
1436
+ * needs --force, and a flag you always pass is a flag that stops meaning
1437
+ * anything: that habituation is what made issue #40 destructive. Holders are
1438
+ * named in the plan instead.
1439
+ *
1440
+ * A DOWNGRADE is refused separately. Restarting from an older CLI replaces a
1441
+ * newer daemon with older code, which is the direction that strands state —
1442
+ * the journal stamp exists because of exactly that (JOURNAL_SCHEMA_VERSION).
1443
+ */
1444
+ assertRestartable(opts: {
1445
+ force: boolean;
1446
+ cliVersion?: string;
1447
+ }): void;
1448
+ /**
1449
+ * doctor: actively check for the failure shapes the review surfaced —
1450
+ * orphaned ports, journal/reality pid divergence, envs stuck recycling, and a
1451
+ * CLI talking to a daemon that is not the installed build.
1452
+ */
1453
+ doctor(cliVersion?: string): Promise<{
1454
+ ok: boolean;
1455
+ issues: {
1456
+ level: string;
1457
+ envId?: string;
1458
+ issue: string;
1459
+ }[];
1460
+ version: string;
1461
+ journalSchema: number;
1462
+ events: import("../core/events.js").DaemonEvent[];
1463
+ }>;
1464
+ /**
1465
+ * Atomically claim an env for teardown UNDER THE POOL LOCK: re-read it, and
1466
+ * (unless force) refuse if it's leased or busy, then flip it to the
1467
+ * 'recycling' guard state so tryClaim/sweep skip it. Returns the row to tear
1468
+ * down, or null if it slipped away. The slow teardown then runs OUTSIDE the
1469
+ * lock, but no claim can touch a 'recycling' env.
1470
+ */
1471
+ private claimForTeardown;
1472
+ /**
1473
+ * Stop and reap this environment's services mid-bind, and journal the result
1474
+ * at once. The row is what capacity accounting reads: an application charge
1475
+ * retained for a conversion stands on `hot` or recorded pids, so leaving
1476
+ * either in the journal after the processes are gone kept a slot reserved for
1477
+ * nothing if the rest of the bind failed. Survivors stay recorded — and keep
1478
+ * the charge — because a pid nobody can confirm dead may still hold a port.
1479
+ * Re-reads the live row so a concurrent degrade is preserved and a recycled
1480
+ * row is never written back.
1481
+ */
1482
+ /**
1483
+ * Stop just the services an `up` is about to restart because their build
1484
+ * output changed (decision 0032); the others keep running. Same reap
1485
+ * contract as stopForBind — recorded pids, then a tag scan — but the scan
1486
+ * is limited to THESE services' tags: the env's other services carry the
1487
+ * same env tag and are still serving.
1488
+ */
1489
+ private stopServicesForRestart;
1490
+ private stopForBind;
1491
+ /**
1492
+ * After a supervisor stopAll(), kill any process that survived or escaped the
1493
+ * group signal so the port is truly free before the port-check below.
1494
+ *
1495
+ * Two failure modes:
1496
+ * (a) Recorded pid survived a previous kill (quiesce/teardown race): reapPids
1497
+ * retries with fresh SIGTERM→SIGKILL on all platforms.
1498
+ * (b) A service called setsid() / spawned a detached grandchild that inherited
1499
+ * the BACKLOT tag but moved to a new process group, escaping the -pgid
1500
+ * SIGKILL: scanTagged (Linux only) finds it by the tag in its environment
1501
+ * and kills the new group.
1502
+ *
1503
+ * Without this, a warm env whose quiesce left survivors hands out a port that
1504
+ * is still bound — the bind then fails with "occupied by a foreign process"
1505
+ * instead of reclaiming the escapee.
1506
+ *
1507
+ * Returns the entries NOT confirmed dead (reapPids contract): callers must
1508
+ * keep those in the journal — a forgotten pid is an orphan nobody can ever
1509
+ * reclaim, and only Linux has the tag scan to fall back on.
1510
+ */
1511
+ private reapEnvProcesses;
1512
+ private reapDiscoveredProcesses;
1513
+ /** Enforce the kill switch under the env lock before preset validation can fail. */
1514
+ private enforceHolderPreviewForbidden;
1515
+ /**
1516
+ * Tear down a live preview the moment the manifest forbids one.
1517
+ *
1518
+ * Unconditional and early, unlike the rest of the reconcile: this is the
1519
+ * security kill switch README and ADR 0027 present as taking effect, and it
1520
+ * would be worthless if a stack stayed published whenever the bind that read
1521
+ * the flag went on to fail.
1522
+ */
1523
+ private enforcePreviewForbidden;
1524
+ /**
1525
+ * Reconcile a live preview against the environment this bind just produced.
1526
+ *
1527
+ * The tunnel outlives service restarts, so everything here is about what is
1528
+ * now BEHIND the unchanged public URL. Three outcomes invalidate it outright
1529
+ * and tear it down — the manifest started forbidding preview, the previewed
1530
+ * service is no longer wanted (`runly down`, decision 0034), or it moved to a different local
1531
+ * port (a renamed port key; existing keys are never reassigned, decision
1532
+ * 0004). A data reset does not: the tunnel keeps serving, which is precisely
1533
+ * why it has to be said out loud.
1534
+ *
1535
+ * Runs at the bind's EPILOGUE, once the shape it judges against is committed.
1536
+ * From the top of the bind it was reading a REQUESTED slice that the epilogue
1537
+ * had not written yet, so a bind that then failed its ready probe left the
1538
+ * tunnel torn down against a slice change that never happened — and dropped
1539
+ * the report with the thrown error. `preview.forbidden` is the exception and
1540
+ * is enforced early by `enforcePreviewForbidden`: it depends on the manifest,
1541
+ * not on the bind.
1542
+ *
1543
+ * Never throws. The `down` that stopped the service or the bind that wiped the data is a
1544
+ * legitimate operation and failing it would strand the caller in a loop (and
1545
+ * bump failStreak into a pristine escalation); the classified message is the
1546
+ * report, and it rides back on the bind's own result.
1547
+ *
1548
+ * `active` must be the DURABLE shape this environment is bound to, never the
1549
+ * supervisor's live pid map: a service in restart backoff is missing from
1550
+ * that map for a second and would be read as "left the slice", killing a
1551
+ * tunnel the restart makes correct again. `portsReallocated` is false on the
1552
+ * hot-reload refresh path — only a real bind fills `env.ports` for a renamed port
1553
+ * key, so before one runs the service is still listening where the tunnel
1554
+ * points and a mismatch means nothing yet.
1555
+ */
1556
+ private reconcilePreviewForBind;
1557
+ /**
1558
+ * Last-resort reap of anything still LIVING IN this env's private directory,
1559
+ * by cwd — at teardown, when that directory is about to be deleted.
1560
+ *
1561
+ * Since decision 0032 services run in the caller's WORKTREE, not here, so
1562
+ * this no longer finds a service that scrubbed its tag: cwd in a worktree is
1563
+ * NOT an ownership signal (the agent's own shells and builds sit there), and
1564
+ * scanning it would kill a stranger's process — the one outcome this whole
1565
+ * module is written to avoid. The tag scan and recorded groups are what
1566
+ * remain for services; this covers anything left in the env directory
1567
+ * itself (a process started under the old projection, for one).
1568
+ */
1569
+ private reapEnvTree;
1570
+ /** Slow teardown of an already-claimed ('recycling') env. */
1571
+ private teardownClaimed;
1572
+ /**
1573
+ * Drop one namespace the way its row recorded it (decision 0037): run the
1574
+ * templated drop command, or delete a file inside the state root. Throws
1575
+ * when the drop did not confirm.
1576
+ */
1577
+ private runDropRecipe;
1578
+ /**
1579
+ * Is `env.root` safe to delete? It must sit strictly inside the state root's
1580
+ * envs/ directory, and must not be (or contain) the worktree the
1581
+ * environment runs in — a corrupted row must never turn teardown into
1582
+ * `rm -rf` of someone's work.
1583
+ */
1584
+ private isPrivateEnvDir;
1585
+ private recycleOne;
1586
+ /**
1587
+ * The clean-slate button, scoped.
1588
+ *
1589
+ * `envId` is an explicit statement of intent — THIS one, not the others — and
1590
+ * it used to be dropped on the floor, so aiming at one cold environment
1591
+ * recycled the whole pool and tore down siblings' live work. A named target
1592
+ * therefore never widens: an unknown id is a usage error, and a leased target
1593
+ * REFUSES rather than being silently skipped, because "nothing happened and
1594
+ * nothing said why" is what sends a caller reaching for force.
1595
+ *
1596
+ * `force` (--force, historically --all) is the only thing that ever reclaims a
1597
+ * leased environment; `busy` is never interrupted, not even by force.
1598
+ */
1599
+ poolRecycle(opts: {
1600
+ envId?: string;
1601
+ force: boolean;
1602
+ }): Promise<{
1603
+ recycled: string[];
1604
+ skipped: {
1605
+ envId: string;
1606
+ reason: string;
1607
+ }[];
1608
+ }>;
1609
+ /** This worktree's degraded, idle environment(s), reaped so a claim can recreate one. */
1610
+ private reapDegradedOwn;
1611
+ /**
1612
+ * One environment per worktree (decision 0032) holds for every environment
1613
+ * this daemon creates. A journal from an older runly can still hold several
1614
+ * for one stack; the unleased, idle ones beyond the one to keep are recycled
1615
+ * here, so the pool converges. A surplus environment that is still leased is
1616
+ * left to its holder until the lease ends.
1617
+ */
1618
+ private drainSurplusEnvs;
1619
+ /** Reap the provably-dead (degraded) envs now, instead of waiting for the sweep. */
1620
+ poolReconcile(): Promise<{
1621
+ reaped: string[];
1622
+ }>;
1623
+ /** The proxy saw a client byte: the clock moves in memory, and reaches the journal throttled. */
1624
+ private noteActivity;
1625
+ /** Persist unsaved activity clocks (all, from the sweeper; one env, throttled, from the hook). */
1626
+ private flushActivity;
1627
+ /** When anyone last used this environment: a runly verb or a client byte on any of its ports. */
1628
+ private envActivityAt;
1629
+ /**
1630
+ * One running service's idle clock: the environment's last runly verb, the
1631
+ * last client byte through the service's own public port (any port's, for
1632
+ * a service with none), and its own start.
1633
+ */
1634
+ private serviceClock;
1635
+ /**
1636
+ * How long a running service of `env` may sit idle: its `idle:` (or the
1637
+ * default), capped for an UNLEASED environment — nobody can wake that one by
1638
+ * traffic — at BACKLOT_IDLE_TTL_MS (the pre-0.16 quiesce). `ps` shows the same.
1639
+ */
1640
+ private serviceIdleLimit;
1641
+ /** Stop the running services of `env` whose idle clock ran out. */
1642
+ private stopIdleServices;
1643
+ /**
1644
+ * Is a tether gone (decision 0035)? A dead pid is believed only once it has
1645
+ * stayed dead for the grace (BACKLOT_TETHER_GRACE_MS, 1 minute); the first
1646
+ * sighting starts the clock, in memory — a daemon restart starts it again.
1647
+ */
1648
+ private tetherGone;
1649
+ /**
1650
+ * The proxy's wake hook (decision 0035): a connection reached the public
1651
+ * port of a service that is not running. Starts it when this environment
1652
+ * holds a live lease and the lease wants that service (a `down`ed service
1653
+ * stays down); the proxy holds the connection meanwhile. Synchronous and
1654
+ * cheap: the start runs in the background.
1655
+ */
1656
+ private requestWake;
1657
+ /**
1658
+ * Start one wanted service — and the wanted, stopped members of its
1659
+ * depends_on closure — on demand. `up`'s start path for a single service:
1660
+ * appliances ensured, datastores kept, the load budget asked (bounded by
1661
+ * how long the proxy holds a connection), no rebuild unless needed (a
1662
+ * `when:` build whose inputs changed, or one that never succeeded here).
1663
+ */
1664
+ private wakeService;
1665
+ /**
1666
+ * Start `closure` — stopped services the lease wants — the way a wake does.
1667
+ * MUST run under the environment's lock. A wake never restores data: a
1668
+ * datastore that exists is kept as it is, so the trigger listing behind the
1669
+ * template keys is computed only for one that is missing.
1670
+ */
1671
+ private startStoppedLocked;
1672
+ /**
1673
+ * `exec` and `token` run against the environment as the lease left it. A
1674
+ * service the idle clock stopped (decision 0035) is started first — under
1675
+ * the lock these verbs hold, where a connection-triggered wake would only
1676
+ * queue behind them until the proxy's hold ran out. So is what a daemon
1677
+ * restart stopped (0.19). A failed bind is refused: `up` redoes it.
1678
+ */
1679
+ private resumeForVerb;
1680
+ /**
1681
+ * A service the supervisor restarts after a crash (decision 0035): its
1682
+ * port HOLDS connections from the crash until the new process accepts on
1683
+ * its (unchanged) internal port, instead of refusing them.
1684
+ */
1685
+ private serviceCrashed;
1686
+ private serviceRelaunched;
1687
+ private serviceGaveUp;
1688
+ /**
1689
+ * A service of a LEASED environment crash-looped past its restart budget
1690
+ * (decision 0039). It is stopped and reported as failed — `ps` and `ctx`
1691
+ * show `failed` with its last exit, `runly logs <svc>` says why — while the
1692
+ * environment, its data, its other services and its logs stay. Nothing
1693
+ * restarts it on its own (a wake skips it); the next `up` does.
1694
+ */
1695
+ private serviceFailed;
1696
+ /** Is `name`, or a service it depends on (transitively), failed (decision 0039)? */
1697
+ private blockedByFailure;
1698
+ /** How a failed service ended, or undefined (decision 0039). */
1699
+ private failureOf;
1700
+ sweep(): Promise<void>;
1701
+ private sweepOnce;
1702
+ shutdown(): Promise<void>;
1703
+ }