agentlane 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/dist/agents/claude-plugin.js +69 -8
- package/dist/agents/cli.js +11 -4
- package/dist/agents/hook-check.js +290 -0
- package/dist/agents/launcher.js +34 -6
- package/dist/agents/wiring-check.js +10 -4
- package/dist/agents/wiring.js +6 -33
- package/dist/browser/env.js +27 -7
- package/dist/browser/install.js +144 -6
- package/dist/browser/lane-browser.js +6 -3
- package/dist/browser/stop.js +2 -17
- package/dist/cache/key.js +23 -11
- package/dist/cli.js +63 -23
- package/dist/devices/backend.js +6 -0
- package/dist/devices/emulator.js +15 -5
- package/dist/devices/hygiene.js +20 -4
- package/dist/devices/simctl.js +15 -3
- package/dist/disk-limits.js +134 -0
- package/dist/driver/caller.js +21 -6
- package/dist/driver/client.js +9 -5
- package/dist/driver/engine.js +29 -14
- package/dist/driver/install.js +88 -36
- package/dist/driver/paths.js +15 -11
- package/dist/driver/session.js +4 -1
- package/dist/driver/superseded.js +378 -0
- package/dist/driver-cli.js +32 -23
- package/dist/evidence/device.js +4 -0
- package/dist/evidence/evidence.js +4 -2
- package/dist/evidence/files.js +68 -0
- package/dist/evidence/strays.js +107 -0
- package/dist/hooks/env.js +2 -1
- package/dist/mcp/server.js +15 -0
- package/dist/mcp/tools.js +9 -1
- package/dist/plugin/guard.js +30 -0
- package/dist/plugin/hook-cli.js +101 -6
- package/dist/plugin/hook-mark.js +73 -0
- package/dist/plugin/hook-once.js +78 -0
- package/dist/plugin/shell.js +109 -8
- package/dist/processes.js +52 -2
- package/dist/registry/config.js +12 -8
- package/dist/registry/holds.js +37 -0
- package/dist/registry/leases.js +109 -0
- package/dist/registry/owner-checks.js +55 -0
- package/dist/registry/owner.js +18 -0
- package/dist/registry/parks.js +60 -0
- package/dist/registry/paths.js +28 -4
- package/dist/registry/ports.js +5 -1
- package/dist/registry/prune.js +643 -24
- package/dist/registry/refusal.js +1 -0
- package/dist/registry/registry.js +1261 -236
- package/dist/registry/rows.js +18 -4
- package/dist/registry/status.js +51 -8
- package/dist/registry/store.js +65 -2
- package/dist/registry/touch.js +180 -0
- package/dist/registry-cli.js +197 -36
- package/dist/services/services.js +227 -56
- package/dist/services/system.js +2 -1
- package/dist/up/answers.js +124 -0
- package/dist/up/doctor.js +29 -4
- package/dist/up/exec.js +29 -4
- package/dist/up/lane-files.js +26 -8
- package/dist/up/lane-services.js +8 -3
- package/dist/up/park.js +146 -0
- package/dist/up/preflight.js +2 -10
- package/dist/up/release.js +20 -24
- package/dist/up/restart.js +9 -2
- package/dist/up/sweep.js +437 -0
- package/dist/up/up.js +38 -15
- package/dist/up/wake.js +124 -0
- package/dist/up/waker.js +49 -0
- package/dist/up-cli.js +6 -1
- package/package.json +2 -2
- package/skills/dev-loop/SKILL.md +7 -5
- package/skills/drive/SKILL.md +4 -0
- package/skills/setup/SKILL.md +2 -2
|
@@ -5,24 +5,29 @@ import { CLI, TOOL } from "../name.js";
|
|
|
5
5
|
import { identityOf } from "../processes.js";
|
|
6
6
|
import { DEVICE_MEMORY_MB, EMULATOR_PORTS, EMULATOR_SERVICE, emulatorSerial, loadConfig, poolCap } from "./config.js";
|
|
7
7
|
import { formatAge, formatTime } from "./format.js";
|
|
8
|
-
import {
|
|
8
|
+
import { allLeases, endLease, handOver, heldBy, KEEPER_PID, leaseDevice, leasedConsolePorts, leaseReservations, sameProcess, } from "./leases.js";
|
|
9
|
+
import { describeOwner, describeState, hostSession, isProven, judgeProcess, ownerKey, ownerState, worktreeState, } from "./owner.js";
|
|
10
|
+
import { dropCheck, noteChecked, noteGone, readCheck, RETURN_WINDOW_MS, startReap, stopReap } from "./owner-checks.js";
|
|
11
|
+
import { dropHold, readHold, writeHold } from "./holds.js";
|
|
12
|
+
import { dropPark, NOTHING_TO_RESTART, readPark, writePark } from "./parks.js";
|
|
9
13
|
import { configPath, diskNames } from "./paths.js";
|
|
10
14
|
import { deletePoolDevice, listPrunable } from "./prune.js";
|
|
11
15
|
import { NotClaimed, Refusal, UsageError } from "./refusal.js";
|
|
12
16
|
import { endTeardown, failResource, failTeardown, finishResource, holdersBeforeLock, pendingTeardowns, startResource, takeTeardowns, } from "./resources.js";
|
|
13
|
-
import { loadLane, ownerFromRow } from "./rows.js";
|
|
17
|
+
import { loadLane, ownerFromRow, processesOf } from "./rows.js";
|
|
14
18
|
import { readStatus } from "./status.js";
|
|
15
|
-
import { int, optInt, optText, Store, text } from "./store.js";
|
|
19
|
+
import { int, optInt, optText, Store, text, UNIDENTIFIED } from "./store.js";
|
|
16
20
|
export const STEPS = [
|
|
17
21
|
'claim:inspected',
|
|
18
22
|
'claim:locked',
|
|
19
23
|
'claim:swept',
|
|
24
|
+
'claim:lease-committed',
|
|
25
|
+
'claim:stopped-device',
|
|
20
26
|
'claim:ticketed',
|
|
21
27
|
'claim:reserved',
|
|
22
28
|
'claim:reserve-committed',
|
|
23
29
|
'claim:retired-device',
|
|
24
30
|
'claim:created-device',
|
|
25
|
-
'claim:took-over',
|
|
26
31
|
'claim:inserted',
|
|
27
32
|
'claim:committed',
|
|
28
33
|
'claim:waiting',
|
|
@@ -30,6 +35,9 @@ export const STEPS = [
|
|
|
30
35
|
'release:locked',
|
|
31
36
|
'release:ended',
|
|
32
37
|
'release:committed',
|
|
38
|
+
'release:stopped-device',
|
|
39
|
+
'park:committed',
|
|
40
|
+
'park:stopped-device',
|
|
33
41
|
'touch:locked',
|
|
34
42
|
];
|
|
35
43
|
export const HISTORY_KEEP_MS = 7 * 24 * 3_600_000;
|
|
@@ -43,15 +51,25 @@ function emulatorConsolePorts() {
|
|
|
43
51
|
function gigabytes(mb) {
|
|
44
52
|
return `${String(Math.round(mb / 100) / 10)} GB`;
|
|
45
53
|
}
|
|
46
|
-
function
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
54
|
+
function byId(devices) {
|
|
55
|
+
return new Map(devices.map(d => [d.id, d]));
|
|
56
|
+
}
|
|
57
|
+
function asError(error) {
|
|
58
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
59
|
+
}
|
|
60
|
+
function consolePortOf(serial) {
|
|
61
|
+
const port = /^emulator-(\d+)$/.exec(serial ?? '')?.[1];
|
|
62
|
+
return port === undefined ? null : Number(port);
|
|
63
|
+
}
|
|
64
|
+
function sameIdentity(a, b) {
|
|
65
|
+
return a === null || b === null ? a === b : a.pid === b.pid && a.start === b.start;
|
|
66
|
+
}
|
|
67
|
+
function settlementKey(s) {
|
|
68
|
+
return s.kind === 'lease' ? s.lease.reservation : s.name;
|
|
53
69
|
}
|
|
54
70
|
const DEFAULT_LOCK_WAIT_MS = 120_000;
|
|
71
|
+
const unsettled = new Map();
|
|
72
|
+
const unreadableKeepers = new WeakSet();
|
|
55
73
|
export class Registry {
|
|
56
74
|
dir;
|
|
57
75
|
config;
|
|
@@ -62,7 +80,9 @@ export class Registry {
|
|
|
62
80
|
clock;
|
|
63
81
|
pollMs;
|
|
64
82
|
onStep;
|
|
83
|
+
sweepSoon;
|
|
65
84
|
selfIdentity;
|
|
85
|
+
keeperIdentity;
|
|
66
86
|
claiming = false;
|
|
67
87
|
constructor(options, config, store) {
|
|
68
88
|
this.dir = options.dir;
|
|
@@ -74,6 +94,7 @@ export class Registry {
|
|
|
74
94
|
this.clock = options.clock ?? systemClock;
|
|
75
95
|
this.pollMs = options.pollMs ?? 500;
|
|
76
96
|
this.onStep = options.onStep ?? (() => undefined);
|
|
97
|
+
this.sweepSoon = options.sweepSoon ?? (() => undefined);
|
|
77
98
|
this.selfIdentity = options.self;
|
|
78
99
|
}
|
|
79
100
|
static open(options) {
|
|
@@ -86,12 +107,48 @@ export class Registry {
|
|
|
86
107
|
step(step) {
|
|
87
108
|
this.onStep(step);
|
|
88
109
|
}
|
|
110
|
+
identities() {
|
|
111
|
+
if (this.keeperIdentity === undefined && unreadableKeepers.has(this.processes))
|
|
112
|
+
this.keeperIdentity = null;
|
|
113
|
+
if (this.selfIdentity !== undefined && this.keeperIdentity !== undefined)
|
|
114
|
+
return;
|
|
115
|
+
const pids = [...(this.selfIdentity ? [] : [process.pid]), ...(this.keeperIdentity === undefined ? [KEEPER_PID] : [])];
|
|
116
|
+
const table = this.processes.startTimes(pids);
|
|
117
|
+
const own = table?.get(process.pid);
|
|
118
|
+
if (!this.selfIdentity && own !== undefined)
|
|
119
|
+
this.selfIdentity = { pid: process.pid, start: own };
|
|
120
|
+
if (this.keeperIdentity !== undefined)
|
|
121
|
+
return;
|
|
122
|
+
const start = table?.get(KEEPER_PID);
|
|
123
|
+
this.keeperIdentity = start === undefined ? null : { pid: KEEPER_PID, start };
|
|
124
|
+
if (start === undefined)
|
|
125
|
+
unreadableKeepers.add(this.processes);
|
|
126
|
+
}
|
|
89
127
|
self() {
|
|
128
|
+
this.identities();
|
|
90
129
|
this.selfIdentity ??= identityOf(this.processes, process.pid);
|
|
91
130
|
if (!this.selfIdentity)
|
|
92
131
|
throw new Error(`ps could not read this process's start time (pid ${String(process.pid)})`);
|
|
93
132
|
return this.selfIdentity;
|
|
94
133
|
}
|
|
134
|
+
keeper() {
|
|
135
|
+
this.identities();
|
|
136
|
+
return this.keeperIdentity ?? null;
|
|
137
|
+
}
|
|
138
|
+
stash(me, settlements) {
|
|
139
|
+
if (settlements.length === 0)
|
|
140
|
+
return;
|
|
141
|
+
const key = `${this.dir}\0${String(me.pid)}\0${me.start}`;
|
|
142
|
+
const kept = unsettled.get(key) ?? [];
|
|
143
|
+
const known = new Set(kept.map(settlementKey));
|
|
144
|
+
unsettled.set(key, [...kept, ...settlements.filter(s => !known.has(settlementKey(s)))]);
|
|
145
|
+
}
|
|
146
|
+
unstash(me) {
|
|
147
|
+
const key = `${this.dir}\0${String(me.pid)}\0${me.start}`;
|
|
148
|
+
const kept = unsettled.get(key) ?? [];
|
|
149
|
+
unsettled.delete(key);
|
|
150
|
+
return kept;
|
|
151
|
+
}
|
|
95
152
|
backend(platform) {
|
|
96
153
|
const backend = this.backends[platform];
|
|
97
154
|
if (backend)
|
|
@@ -106,27 +163,116 @@ export class Registry {
|
|
|
106
163
|
snapshotBeforeLock() {
|
|
107
164
|
const db = this.store.db;
|
|
108
165
|
const pids = [];
|
|
109
|
-
const lanes = new
|
|
110
|
-
for (const row of db.prepare('SELECT
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
if (
|
|
114
|
-
pids.push(pid);
|
|
115
|
-
}
|
|
116
|
-
const reservations = new
|
|
117
|
-
|
|
118
|
-
|
|
166
|
+
const lanes = new Map();
|
|
167
|
+
for (const row of db.prepare('SELECT * FROM claims').all()) {
|
|
168
|
+
const owner = ownerFromRow(row).process;
|
|
169
|
+
lanes.set(text(row, 'id'), owner);
|
|
170
|
+
if (owner)
|
|
171
|
+
pids.push(owner.pid);
|
|
172
|
+
}
|
|
173
|
+
const reservations = new Map();
|
|
174
|
+
const known = new Map();
|
|
175
|
+
const free = new Map();
|
|
176
|
+
const devices = db.prepare(`SELECT name, device_id, claim_id, creator_pid, creator_start, EXISTS (SELECT 1 FROM device_leases l WHERE l.reservation = devices.name) AS leased
|
|
177
|
+
FROM devices`);
|
|
178
|
+
for (const row of devices.all()) {
|
|
179
|
+
const id = optText(row, 'device_id');
|
|
180
|
+
if (id !== null) {
|
|
181
|
+
known.set(text(row, 'name'), id);
|
|
182
|
+
if (optText(row, 'claim_id') === null)
|
|
183
|
+
free.set(text(row, 'name'), id);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (int(row, 'leased') === 1)
|
|
187
|
+
continue;
|
|
188
|
+
reservations.set(text(row, 'name'), { pid: int(row, 'creator_pid'), start: text(row, 'creator_start') });
|
|
119
189
|
pids.push(int(row, 'creator_pid'));
|
|
120
190
|
}
|
|
191
|
+
const leases = new Map();
|
|
192
|
+
for (const row of db.prepare('SELECT reservation, holder_pid, holder_start FROM device_leases').all()) {
|
|
193
|
+
leases.set(text(row, 'reservation'), { pid: int(row, 'holder_pid'), start: text(row, 'holder_start') });
|
|
194
|
+
pids.push(int(row, 'holder_pid'));
|
|
195
|
+
}
|
|
196
|
+
const ended = db.prepare('SELECT MAX(seq) AS seq FROM history').get();
|
|
121
197
|
const tickets = new Set();
|
|
122
198
|
for (const row of db.prepare('SELECT ticket, waiter_pid FROM queue').all()) {
|
|
123
199
|
tickets.add(int(row, 'ticket'));
|
|
124
200
|
pids.push(int(row, 'waiter_pid'));
|
|
125
201
|
}
|
|
126
|
-
return {
|
|
202
|
+
return {
|
|
203
|
+
table: pids.length === 0 ? new Map() : this.processes.startTimes(pids),
|
|
204
|
+
lanes,
|
|
205
|
+
reservations,
|
|
206
|
+
leases,
|
|
207
|
+
tickets,
|
|
208
|
+
known,
|
|
209
|
+
free,
|
|
210
|
+
ended: (ended ? optInt(ended, 'seq') : null) ?? 0,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
hasDeviceRecords(platform) {
|
|
214
|
+
const db = this.store.db;
|
|
215
|
+
return (db.prepare('SELECT 1 FROM devices WHERE platform = ? LIMIT 1').get(platform) !== undefined ||
|
|
216
|
+
db.prepare('SELECT 1 FROM device_leases WHERE platform = ? LIMIT 1').get(platform) !== undefined);
|
|
217
|
+
}
|
|
218
|
+
listBeforeLock(wanted) {
|
|
219
|
+
const listings = new Map();
|
|
220
|
+
for (const backend of Object.values(this.backends)) {
|
|
221
|
+
if (backend === wanted) {
|
|
222
|
+
listings.set(backend.platform, backend.list());
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (!this.hasDeviceRecords(backend.platform))
|
|
226
|
+
continue;
|
|
227
|
+
try {
|
|
228
|
+
listings.set(backend.platform, backend.list());
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
listings.set(backend.platform, asError(error));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
return listings;
|
|
235
|
+
}
|
|
236
|
+
recoveryBeforeLock(verdicts) {
|
|
237
|
+
const db = this.store.db;
|
|
238
|
+
const holders = [
|
|
239
|
+
...db
|
|
240
|
+
.prepare(`SELECT platform, creator_pid AS pid, creator_start AS start FROM devices
|
|
241
|
+
WHERE device_id IS NULL AND name NOT IN (SELECT reservation FROM device_leases)`)
|
|
242
|
+
.all(),
|
|
243
|
+
...db.prepare('SELECT platform, holder_pid AS pid, holder_start AS start FROM device_leases').all(),
|
|
244
|
+
];
|
|
245
|
+
if (holders.length === 0)
|
|
246
|
+
return null;
|
|
247
|
+
const snapshot = verdicts ?? this.snapshotBeforeLock();
|
|
248
|
+
const platforms = new Set();
|
|
249
|
+
for (const row of holders) {
|
|
250
|
+
if (isProven(judgeProcess({ pid: int(row, 'pid'), start: text(row, 'start') }, snapshot.table))) {
|
|
251
|
+
platforms.add(text(row, 'platform'));
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (platforms.size === 0)
|
|
255
|
+
return null;
|
|
256
|
+
const listings = new Map();
|
|
257
|
+
for (const platform of platforms) {
|
|
258
|
+
const backend = this.backends[platform];
|
|
259
|
+
if (!backend)
|
|
260
|
+
continue;
|
|
261
|
+
try {
|
|
262
|
+
listings.set(platform, backend.list());
|
|
263
|
+
}
|
|
264
|
+
catch (error) {
|
|
265
|
+
listings.set(platform, asError(error));
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return { verdicts: snapshot, listings };
|
|
127
269
|
}
|
|
128
270
|
verdictFor(row, verdicts) {
|
|
129
|
-
|
|
271
|
+
const judged = verdicts.lanes.get(text(row, 'id'));
|
|
272
|
+
const owner = ownerFromRow(row).process;
|
|
273
|
+
if (judged === undefined || !sameIdentity(judged, owner))
|
|
274
|
+
return 'alive';
|
|
275
|
+
return judgeProcess(owner, verdicts.table);
|
|
130
276
|
}
|
|
131
277
|
candidatePorts(services, platform) {
|
|
132
278
|
const ports = new Set();
|
|
@@ -176,18 +322,46 @@ export class Registry {
|
|
|
176
322
|
const spec = backend && request.device ? backend.resolveSpec(request.device) : null;
|
|
177
323
|
const caller = this.caller(request.owner, request.worktree);
|
|
178
324
|
const me = this.self();
|
|
179
|
-
const
|
|
180
|
-
const
|
|
325
|
+
const keeper = this.keeper();
|
|
326
|
+
const started = this.clock.now();
|
|
327
|
+
const deadline = started + request.waitMs;
|
|
328
|
+
const progress = { notes: [], found: false, replacedDriver: null };
|
|
329
|
+
const settling = this.unstash(me);
|
|
330
|
+
const stopped = new Set();
|
|
331
|
+
let relisted = false;
|
|
181
332
|
try {
|
|
182
333
|
for (;;) {
|
|
183
334
|
const verdicts = this.snapshotBeforeLock();
|
|
335
|
+
const listings = this.listBeforeLock(backend);
|
|
184
336
|
const listening = await this.probe.listening(this.candidatePorts([...request.services, ...browserServices(request)], request.device?.platform ?? null));
|
|
185
337
|
this.step('claim:inspected');
|
|
186
|
-
const result = this.store.write(() =>
|
|
338
|
+
const result = this.store.write(() => {
|
|
339
|
+
const attempted = this.attempt(request, caller, spec, backend, me, keeper, verdicts, listings, listening, settling, stopped, progress);
|
|
340
|
+
return keeper === null ? this.finishedInWrite(attempted, me, progress.notes) : attempted;
|
|
341
|
+
});
|
|
342
|
+
settling.length = 0;
|
|
343
|
+
if (progress.found) {
|
|
344
|
+
progress.found = false;
|
|
345
|
+
this.sweepSoon();
|
|
346
|
+
}
|
|
187
347
|
if (result.kind === 'refused')
|
|
188
348
|
throw result.refusal;
|
|
189
|
-
if (result.kind === '
|
|
349
|
+
if (result.kind === 'relist') {
|
|
350
|
+
if (relisted && this.clock.now() >= deadline)
|
|
351
|
+
throw this.poolChanging(request, verdicts.ended, started);
|
|
352
|
+
relisted = true;
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
if (result.kind === 'leased') {
|
|
356
|
+
if (result.leases.length === 0)
|
|
357
|
+
continue;
|
|
358
|
+
this.step('claim:lease-committed');
|
|
359
|
+
settling.push(...this.workOn(result.leases, progress.notes, 'claim:stopped-device', true));
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
if (result.kind === 'create') {
|
|
190
363
|
this.step('claim:reserve-committed');
|
|
364
|
+
this.create(result, settling, progress.notes);
|
|
191
365
|
continue;
|
|
192
366
|
}
|
|
193
367
|
if (result.kind === 'done') {
|
|
@@ -195,7 +369,7 @@ export class Registry {
|
|
|
195
369
|
const device = result.lane.device;
|
|
196
370
|
const held = device ? this.backends[device.platform] : undefined;
|
|
197
371
|
if (request.device && held && (request.boot || device?.platform === 'android')) {
|
|
198
|
-
await this.boot(held, result.lane, result.created, caller, progress);
|
|
372
|
+
await this.boot(held, result.lane, result.created, caller, me, keeper, progress);
|
|
199
373
|
}
|
|
200
374
|
return { lane: result.lane, created: result.created, ...progress };
|
|
201
375
|
}
|
|
@@ -203,15 +377,211 @@ export class Registry {
|
|
|
203
377
|
const left = deadline - this.clock.now();
|
|
204
378
|
if (left <= 0)
|
|
205
379
|
throw this.poolBusy(result, request);
|
|
380
|
+
this.sweepSoon();
|
|
206
381
|
await this.clock.sleep(Math.min(this.pollMs, left));
|
|
207
382
|
}
|
|
208
383
|
}
|
|
209
384
|
catch (error) {
|
|
210
|
-
this.abandon(me,
|
|
385
|
+
this.abandon(me, keeper, settling);
|
|
211
386
|
throw error;
|
|
212
387
|
}
|
|
213
388
|
}
|
|
214
|
-
|
|
389
|
+
create(plan, settling, notes) {
|
|
390
|
+
const { backend, retire, spec } = plan;
|
|
391
|
+
const noun = NOUN[backend.platform];
|
|
392
|
+
if (retire) {
|
|
393
|
+
try {
|
|
394
|
+
backend.delete(retire.id);
|
|
395
|
+
}
|
|
396
|
+
catch (error) {
|
|
397
|
+
settling.push({ kind: 'lease', lease: retire, outcome: 'present' });
|
|
398
|
+
throw error;
|
|
399
|
+
}
|
|
400
|
+
settling.push({ kind: 'lease', lease: retire, outcome: 'gone' });
|
|
401
|
+
this.replaced(plan, retire, notes);
|
|
402
|
+
}
|
|
403
|
+
const id = backend.create(plan.name, spec);
|
|
404
|
+
settling.push({ kind: 'created', name: plan.name, id });
|
|
405
|
+
this.step('claim:created-device');
|
|
406
|
+
notes.push(`created pool ${noun} ${plan.name} (${spec.deviceTypeName}, ${spec.runtimeName}, ${id})`);
|
|
407
|
+
}
|
|
408
|
+
replaced(plan, retire, notes) {
|
|
409
|
+
notes.push(`replaced unclaimed pool ${NOUN[retire.platform]} ${retire.name} (${retire.deviceTypeName}, ${retire.runtimeName}): the lane is strict and needs ${plan.spec.deviceTypeName}, ${plan.spec.runtimeName}`);
|
|
410
|
+
this.step('claim:retired-device');
|
|
411
|
+
}
|
|
412
|
+
finishedInWrite(result, me, notes) {
|
|
413
|
+
if (result.kind === 'leased') {
|
|
414
|
+
this.inWrite({ own: [], others: result.leases }, me, null, notes, 'claim:stopped-device');
|
|
415
|
+
return { kind: 'leased', leases: [] };
|
|
416
|
+
}
|
|
417
|
+
if (result.kind !== 'create' || !result.retire)
|
|
418
|
+
return result;
|
|
419
|
+
result.backend.delete(result.retire.id);
|
|
420
|
+
this.settle([{ kind: 'lease', lease: result.retire, outcome: 'gone' }], me, null, notes);
|
|
421
|
+
this.replaced(result, result.retire, notes);
|
|
422
|
+
return { ...result, retire: null };
|
|
423
|
+
}
|
|
424
|
+
workOn(leases, notes, step, unclaimed) {
|
|
425
|
+
return leases.map(lease => {
|
|
426
|
+
const outcome = lease.work === 'delete' ? this.retire(lease, notes) : this.stop(lease, notes, unclaimed);
|
|
427
|
+
this.step(step);
|
|
428
|
+
return { kind: 'lease', lease, outcome };
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
stop(lease, notes, unclaimed) {
|
|
432
|
+
const what = unclaimed ? `unclaimed pool ${NOUN[lease.platform]} ${lease.name}` : lease.name;
|
|
433
|
+
try {
|
|
434
|
+
this.backends[lease.platform]?.shutdown(lease.id);
|
|
435
|
+
if (unclaimed)
|
|
436
|
+
notes.push(`shut down ${what}`);
|
|
437
|
+
}
|
|
438
|
+
catch (error) {
|
|
439
|
+
notes.push(`could not shut down ${what}: ${asError(error).message}`);
|
|
440
|
+
}
|
|
441
|
+
return 'present';
|
|
442
|
+
}
|
|
443
|
+
retire(lease, notes) {
|
|
444
|
+
const noun = NOUN[lease.platform];
|
|
445
|
+
const size = `${lease.platform === 'ios' ? 'poolSize' : 'emulatorPoolSize'} of ${String(poolCap(this.config, lease.platform))}`;
|
|
446
|
+
const backend = this.backends[lease.platform];
|
|
447
|
+
try {
|
|
448
|
+
if (!backend)
|
|
449
|
+
throw new Error(`this process has no ${noun} tool`);
|
|
450
|
+
backend.delete(lease.id);
|
|
451
|
+
}
|
|
452
|
+
catch (error) {
|
|
453
|
+
notes.push(`could not delete pool ${noun} ${lease.name}, which the pool has no room for (its ${size}): ${asError(error).message}; it stays in the pool, one past its size, until ${CLI} prune deletes it`);
|
|
454
|
+
return 'undeletable';
|
|
455
|
+
}
|
|
456
|
+
notes.push(`deleted pool ${noun} ${lease.name}: the pool holds its ${size} without it`);
|
|
457
|
+
return 'gone';
|
|
458
|
+
}
|
|
459
|
+
settleCreated(settling, me, notes) {
|
|
460
|
+
const db = this.store.db;
|
|
461
|
+
for (const s of settling) {
|
|
462
|
+
if (s.kind !== 'created')
|
|
463
|
+
continue;
|
|
464
|
+
const settled = db
|
|
465
|
+
.prepare('UPDATE devices SET device_id = ?, creator_pid = NULL, creator_start = NULL WHERE name = ? AND device_id IS NULL AND creator_pid = ? AND creator_start = ?')
|
|
466
|
+
.run(s.id, s.name, me.pid, me.start);
|
|
467
|
+
if (settled.changes === 0 && !db.prepare('SELECT 1 FROM devices WHERE device_id = ?').get(s.id)) {
|
|
468
|
+
notes.push(`created ${s.name} (${s.id}), but its reservation had been dropped, so no pool record holds it`);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
settleLeases(settling, me, keeper, notes) {
|
|
473
|
+
const next = [];
|
|
474
|
+
for (const s of settling) {
|
|
475
|
+
if (s.kind !== 'lease' || !heldBy(this.store.db, s.lease, me))
|
|
476
|
+
continue;
|
|
477
|
+
const retiring = this.endOrRetire(s.lease, s.outcome, me, keeper, notes, null);
|
|
478
|
+
if (retiring)
|
|
479
|
+
next.push(retiring);
|
|
480
|
+
}
|
|
481
|
+
return next;
|
|
482
|
+
}
|
|
483
|
+
settle(settling, me, keeper, notes) {
|
|
484
|
+
this.settleCreated(settling, me, notes);
|
|
485
|
+
return this.settleLeases(settling, me, keeper, notes);
|
|
486
|
+
}
|
|
487
|
+
endOrRetire(lease, outcome, me, keeper, notes, why) {
|
|
488
|
+
const db = this.store.db;
|
|
489
|
+
const noun = NOUN[lease.platform];
|
|
490
|
+
const end = endLease(db, lease, outcome, poolCap(this.config, lease.platform));
|
|
491
|
+
if (end === 'no-room') {
|
|
492
|
+
return handOver(db, lease, {
|
|
493
|
+
reservation: this.leaseName(lease.name, 'delete', []),
|
|
494
|
+
work: 'delete',
|
|
495
|
+
holder: me,
|
|
496
|
+
keeper,
|
|
497
|
+
consolePort: lease.consolePort,
|
|
498
|
+
now: this.clock.now(),
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
if (why !== null && end === 'restored')
|
|
502
|
+
notes.push(`returned pool ${noun} ${lease.name} to the pool: ${why}`);
|
|
503
|
+
if (why !== null && end === 'recorded-elsewhere') {
|
|
504
|
+
notes.push(`forgot the lease on pool ${noun} ${lease.name}: the registry holds ${lease.id} under another record`);
|
|
505
|
+
}
|
|
506
|
+
return null;
|
|
507
|
+
}
|
|
508
|
+
settleNow(settling, me, keeper, notes) {
|
|
509
|
+
if (settling.length === 0)
|
|
510
|
+
return [];
|
|
511
|
+
try {
|
|
512
|
+
return this.store.write(() => this.settle(settling, me, keeper, notes));
|
|
513
|
+
}
|
|
514
|
+
catch (error) {
|
|
515
|
+
this.stash(me, settling);
|
|
516
|
+
const names = settling.map(s => (s.kind === 'lease' ? s.lease.name : s.name)).join(', ');
|
|
517
|
+
notes.push(`could not return ${names} to the pool yet: ${asError(error).message}; this process's next claim or release returns it, or any process's once this one exits`);
|
|
518
|
+
return [];
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
inWrite(work, me, keeper, notes, step) {
|
|
522
|
+
if (keeper !== null)
|
|
523
|
+
return work;
|
|
524
|
+
this.finish(work, notes, step, done => this.settle(done, me, null, notes));
|
|
525
|
+
return { own: [], others: [] };
|
|
526
|
+
}
|
|
527
|
+
afterWrite(work, me, keeper, notes, step) {
|
|
528
|
+
this.finish(work, notes, step, done => this.settleNow(done, me, keeper, notes));
|
|
529
|
+
}
|
|
530
|
+
finish(work, notes, step, settle) {
|
|
531
|
+
if (work.own.length + work.others.length === 0)
|
|
532
|
+
return;
|
|
533
|
+
let next = settle([...this.workOn(work.own, notes, step, false), ...this.workOn(work.others, notes, step, true)]);
|
|
534
|
+
while (next.length > 0)
|
|
535
|
+
next = settle(this.workOn(next, notes, step, true));
|
|
536
|
+
}
|
|
537
|
+
ownRows(me, covered) {
|
|
538
|
+
const db = this.store.db;
|
|
539
|
+
const leased = leaseReservations(db);
|
|
540
|
+
return {
|
|
541
|
+
reservations: db
|
|
542
|
+
.prepare('SELECT * FROM devices WHERE device_id IS NULL AND creator_pid = ? AND creator_start = ?')
|
|
543
|
+
.all(me.pid, me.start)
|
|
544
|
+
.filter(row => !leased.has(text(row, 'name')) && !covered.has(text(row, 'name'))),
|
|
545
|
+
leases: allLeases(db).filter(l => sameProcess(l.holder, me) && !covered.has(l.reservation)),
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
listOwn(me, pending, have) {
|
|
549
|
+
const { reservations, leases } = this.ownRows(me, new Set(pending.map(settlementKey)));
|
|
550
|
+
const listings = new Map(have);
|
|
551
|
+
for (const platform of new Set([...reservations.map(r => text(r, 'platform')), ...leases.map(l => l.platform)])) {
|
|
552
|
+
const backend = this.backends[platform];
|
|
553
|
+
if (!backend || listings.has(platform))
|
|
554
|
+
continue;
|
|
555
|
+
try {
|
|
556
|
+
listings.set(platform, backend.list());
|
|
557
|
+
}
|
|
558
|
+
catch (error) {
|
|
559
|
+
listings.set(platform, asError(error));
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
return listings;
|
|
563
|
+
}
|
|
564
|
+
landLeftovers(pending, listings, me, keeper, notes) {
|
|
565
|
+
const { reservations, leases } = this.ownRows(me, new Set(pending.map(settlementKey)));
|
|
566
|
+
this.settleCreated(pending, me, notes);
|
|
567
|
+
for (const row of reservations) {
|
|
568
|
+
const listed = listings.get(text(row, 'platform'));
|
|
569
|
+
if (listed && !(listed instanceof Error))
|
|
570
|
+
notes.push(this.settleReservation(row, listed));
|
|
571
|
+
}
|
|
572
|
+
const next = this.settleLeases(pending, me, keeper, notes);
|
|
573
|
+
for (const lease of leases) {
|
|
574
|
+
const listed = listings.get(lease.platform);
|
|
575
|
+
if (!listed || listed instanceof Error)
|
|
576
|
+
continue;
|
|
577
|
+
const outcome = listed.some(d => d.id === lease.id) ? 'present' : 'gone';
|
|
578
|
+
const retire = this.endOrRetire(lease, outcome, me, keeper, notes, 'this process finished its work on it earlier');
|
|
579
|
+
if (retire)
|
|
580
|
+
next.push(retire);
|
|
581
|
+
}
|
|
582
|
+
return next;
|
|
583
|
+
}
|
|
584
|
+
async boot(backend, lane, created, caller, me, keeper, progress) {
|
|
215
585
|
if (!lane.device)
|
|
216
586
|
return;
|
|
217
587
|
try {
|
|
@@ -220,53 +590,65 @@ export class Registry {
|
|
|
220
590
|
}
|
|
221
591
|
catch (error) {
|
|
222
592
|
if (created) {
|
|
223
|
-
this.store.write(() => {
|
|
593
|
+
const work = this.store.write(() => {
|
|
224
594
|
const row = this.store.db.prepare('SELECT * FROM claims WHERE id = ? AND token = ?').get(lane.id, lane.token);
|
|
225
|
-
|
|
226
|
-
|
|
595
|
+
const own = row ? this.endAndLease([row], 'released', caller, me, keeper).leases : [];
|
|
596
|
+
return this.inWrite({ own, others: [] }, me, keeper, progress.notes, 'release:stopped-device');
|
|
227
597
|
});
|
|
598
|
+
this.afterWrite(work, me, keeper, progress.notes, 'release:stopped-device');
|
|
228
599
|
}
|
|
229
600
|
throw error;
|
|
230
601
|
}
|
|
231
602
|
}
|
|
232
603
|
cancelPending() {
|
|
233
|
-
|
|
604
|
+
const me = this.self();
|
|
605
|
+
return this.abandon(me, this.keeper(), []);
|
|
234
606
|
}
|
|
235
|
-
abandon(me,
|
|
607
|
+
abandon(me, keeper, settling) {
|
|
608
|
+
this.stash(me, settling);
|
|
609
|
+
const pending = this.unstash(me);
|
|
610
|
+
let work;
|
|
236
611
|
try {
|
|
237
|
-
this.
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
const
|
|
241
|
-
|
|
242
|
-
.all(me.pid, me.start);
|
|
243
|
-
if (rows.length === 0)
|
|
244
|
-
return;
|
|
245
|
-
for (const row of rows) {
|
|
246
|
-
this.settleReservation(row, listOrNothing(backend ?? this.backends[text(row, 'platform')]));
|
|
247
|
-
}
|
|
612
|
+
const listings = this.listOwn(me, pending);
|
|
613
|
+
work = this.store.write(() => {
|
|
614
|
+
this.dropTicket(me);
|
|
615
|
+
const others = this.landLeftovers(pending, listings, me, keeper, []);
|
|
616
|
+
return this.inWrite({ own: [], others }, me, keeper, [], 'claim:stopped-device');
|
|
248
617
|
});
|
|
249
|
-
return null;
|
|
250
618
|
}
|
|
251
619
|
catch (cleanupError) {
|
|
252
|
-
|
|
620
|
+
this.stash(me, pending);
|
|
621
|
+
return asError(cleanupError);
|
|
253
622
|
}
|
|
623
|
+
this.afterWrite(work, me, keeper, [], 'claim:stopped-device');
|
|
624
|
+
return null;
|
|
254
625
|
}
|
|
255
|
-
attempt(request, caller, spec, backend, me, verdicts, listening, progress) {
|
|
626
|
+
attempt(request, caller, spec, backend, me, keeper, verdicts, listings, listening, settling, stopped, progress) {
|
|
256
627
|
this.step('claim:locked');
|
|
257
628
|
const db = this.store.db;
|
|
258
629
|
const now = this.clock.now();
|
|
259
|
-
|
|
260
|
-
|
|
630
|
+
const retiring = this.landLeftovers(settling, listings, me, keeper, progress.notes);
|
|
631
|
+
if (this.sweep(verdicts, now))
|
|
632
|
+
progress.found = true;
|
|
633
|
+
const { listed, leases, returnedSinceSnapshot } = this.reconcileAll(verdicts, backend, listings, me, keeper, now, stopped, progress.notes);
|
|
261
634
|
this.step('claim:swept');
|
|
262
|
-
|
|
635
|
+
if (retiring.length + leases.length > 0)
|
|
636
|
+
return { kind: 'leased', leases: [...retiring, ...leases] };
|
|
637
|
+
if (request.device !== null && (returnedSinceSnapshot || this.freedSinceListing(verdicts.ended, listings))) {
|
|
638
|
+
return { kind: 'relist' };
|
|
639
|
+
}
|
|
640
|
+
const mine = db.prepare('SELECT * FROM claims WHERE owner_key = ? AND worktree = ?').get(caller.key, caller.worktree);
|
|
641
|
+
if (mine)
|
|
642
|
+
this.recognise(mine, request.owner);
|
|
643
|
+
const existing = mine ?? this.returning(request.owner, caller, now, row => isProven(this.verdictFor(row, verdicts)));
|
|
263
644
|
const existingId = existing ? text(existing, 'id') : null;
|
|
264
645
|
let needDevice = request.device !== null;
|
|
265
646
|
let held;
|
|
266
647
|
if (existingId !== null) {
|
|
267
648
|
held = db.prepare('SELECT * FROM devices WHERE claim_id = ?').get(existingId);
|
|
268
649
|
const platform = held ? text(held, 'platform') : null;
|
|
269
|
-
|
|
650
|
+
const vanished = held && verdicts.known.get(text(held, 'name')) === text(held, 'device_id') && !listed?.has(text(held, 'device_id'));
|
|
651
|
+
if (held && platform === request.device?.platform && listed && vanished) {
|
|
270
652
|
db.prepare('DELETE FROM devices WHERE name = ?').run(text(held, 'name'));
|
|
271
653
|
progress.notes.push(`your ${NOUN[platform]} ${text(held, 'name')} no longer exists; claiming another`);
|
|
272
654
|
held = undefined;
|
|
@@ -287,7 +669,7 @@ export class Registry {
|
|
|
287
669
|
this.dropReservation(me);
|
|
288
670
|
return { kind: 'refused', refusal };
|
|
289
671
|
};
|
|
290
|
-
const portPlan = this.planPorts(wanted, listening, now
|
|
672
|
+
const portPlan = this.planPorts(wanted, listening, now);
|
|
291
673
|
if (portPlan instanceof Refusal)
|
|
292
674
|
return refuse(portPlan);
|
|
293
675
|
const emulator = this.planEmulatorPort(request, existingId, held, needDevice, listed, listening, now);
|
|
@@ -296,21 +678,11 @@ export class Registry {
|
|
|
296
678
|
let deviceName = null;
|
|
297
679
|
if (needDevice && spec && backend && listed) {
|
|
298
680
|
const strict = request.device?.strict === true;
|
|
299
|
-
const plan = this.planDevice(spec, strict, backend, listed, me,
|
|
681
|
+
const plan = this.planDevice(spec, strict, request.device?.prefer, backend, listed, me, keeper, now, caller, progress);
|
|
300
682
|
if (plan.kind !== 'assign')
|
|
301
683
|
return plan;
|
|
302
684
|
deviceName = plan.name;
|
|
303
685
|
}
|
|
304
|
-
let tookOverPorts = false;
|
|
305
|
-
for (const id of portPlan.takeovers) {
|
|
306
|
-
const row = db.prepare('SELECT * FROM claims WHERE id = ?').get(id);
|
|
307
|
-
if (!row)
|
|
308
|
-
continue;
|
|
309
|
-
progress.tookOver.push(this.end(row, 'taken-over', caller, now));
|
|
310
|
-
tookOverPorts = true;
|
|
311
|
-
}
|
|
312
|
-
if (tookOverPorts)
|
|
313
|
-
this.step('claim:took-over');
|
|
314
686
|
const action = request.action ?? 'claim';
|
|
315
687
|
let laneId;
|
|
316
688
|
if (existingId !== null) {
|
|
@@ -367,16 +739,17 @@ export class Registry {
|
|
|
367
739
|
}
|
|
368
740
|
dropReservation(me) {
|
|
369
741
|
this.store.db
|
|
370
|
-
.prepare(
|
|
742
|
+
.prepare(`DELETE FROM devices WHERE device_id IS NULL AND creator_pid = ? AND creator_start = ?
|
|
743
|
+
AND NOT EXISTS (SELECT 1 FROM device_leases l WHERE l.reservation = devices.name)`)
|
|
371
744
|
.run(me.pid, me.start);
|
|
372
745
|
}
|
|
373
|
-
sweep(verdicts, now
|
|
746
|
+
sweep(verdicts, now) {
|
|
374
747
|
const db = this.store.db;
|
|
375
|
-
|
|
748
|
+
let found = false;
|
|
376
749
|
for (const row of db.prepare('SELECT * FROM claims').all()) {
|
|
377
750
|
const state = ownerState(this.verdictFor(row, verdicts), worktreeState(text(row, 'worktree')));
|
|
378
|
-
if (isProven(state))
|
|
379
|
-
|
|
751
|
+
if (isProven(state) && this.foundGone(row, state, now))
|
|
752
|
+
found = true;
|
|
380
753
|
}
|
|
381
754
|
for (const row of db.prepare('SELECT * FROM queue').all()) {
|
|
382
755
|
const ticket = int(row, 'ticket');
|
|
@@ -387,62 +760,199 @@ export class Registry {
|
|
|
387
760
|
db.prepare('DELETE FROM queue WHERE ticket = ?').run(ticket);
|
|
388
761
|
}
|
|
389
762
|
db.prepare('DELETE FROM history WHERE ended_at < ?').run(now - HISTORY_KEEP_MS);
|
|
390
|
-
return
|
|
763
|
+
return found;
|
|
391
764
|
}
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
765
|
+
/** Records the lane's owner as found gone, which ends its hold; the sweep parks the lane, and ends it once it stays gone. */
|
|
766
|
+
foundGone(row, state, now) {
|
|
767
|
+
const db = this.store.db;
|
|
768
|
+
const id = text(row, 'id');
|
|
769
|
+
const key = text(row, 'owner_key');
|
|
770
|
+
if (!noteGone(db, id, key, state, now))
|
|
771
|
+
return false;
|
|
772
|
+
dropHold(db, id, key);
|
|
773
|
+
return true;
|
|
774
|
+
}
|
|
775
|
+
beingEnded(row, gone) {
|
|
776
|
+
const pid = String(gone.reaper?.pid ?? 'unknown');
|
|
777
|
+
return new Refusal('lane-busy', `lane ${text(row, 'id')} is being ended: its owner was found ${describeState(gone.state)} at ${formatTime(gone.foundAt)} and did not come back within ${formatAge(RETURN_WINDOW_MS)}`, [
|
|
778
|
+
{
|
|
779
|
+
who: `the sweep ending it (pid ${pid})`,
|
|
780
|
+
folder: text(row, 'worktree'),
|
|
781
|
+
since: formatTime(gone.reapingSince ?? gone.foundAt),
|
|
782
|
+
},
|
|
783
|
+
], [`${CLI} up (again once it has ended; it claims a new lane)`, `${CLI} status (the lane, and the process ending it)`]);
|
|
784
|
+
}
|
|
785
|
+
/**
|
|
786
|
+
* Takes `owner`, which presents the lane's own owner key, as the process to judge the lane by: a Codex thread's
|
|
787
|
+
* process that came back, or an owner a check took for gone. Either way the lane is its owner's again.
|
|
788
|
+
*/
|
|
789
|
+
recognise(row, owner) {
|
|
790
|
+
const db = this.store.db;
|
|
791
|
+
const id = text(row, 'id');
|
|
792
|
+
const gone = readCheck(db, id, text(row, 'owner_key'))?.gone ?? null;
|
|
793
|
+
if (gone?.reaper)
|
|
794
|
+
throw this.beingEnded(row, gone);
|
|
795
|
+
const moved = owner.process !== null && !sameIdentity(ownerFromRow(row).process, owner.process);
|
|
796
|
+
if (!moved && gone === null)
|
|
797
|
+
return;
|
|
798
|
+
if (owner.process && moved) {
|
|
799
|
+
db.prepare('UPDATE claims SET owner_pid = ?, owner_start = ? WHERE id = ?').run(owner.process.pid, owner.process.start, id);
|
|
800
|
+
}
|
|
801
|
+
dropCheck(db, id);
|
|
802
|
+
}
|
|
803
|
+
/**
|
|
804
|
+
* The lane a gone process of the caller's own Claude Code session held in the caller's folder, taken over by the
|
|
805
|
+
* caller with its ports, resources and record: when the lane's owner was found gone less than RETURN_WINDOW_MS ago
|
|
806
|
+
* and nothing has begun to end the lane, or when `gone` judges that owner gone now.
|
|
807
|
+
*/
|
|
808
|
+
returning(owner, caller, now, gone) {
|
|
809
|
+
const db = this.store.db;
|
|
810
|
+
const session = hostSession(owner);
|
|
811
|
+
if (owner.kind !== 'claude' || session === null)
|
|
812
|
+
return undefined;
|
|
813
|
+
const rows = db
|
|
814
|
+
.prepare(`SELECT * FROM claims WHERE worktree = ? AND owner_kind = 'claude' AND owner_session = ? AND owner_key != ?
|
|
815
|
+
ORDER BY last_used_at DESC, id`)
|
|
816
|
+
.all(caller.worktree, session, caller.key);
|
|
817
|
+
for (const row of rows) {
|
|
818
|
+
const found = readCheck(db, text(row, 'id'), text(row, 'owner_key'))?.gone ?? null;
|
|
819
|
+
const back = found ? found.reaper === null && now < found.foundAt + RETURN_WINDOW_MS : gone(row);
|
|
820
|
+
if (!back)
|
|
397
821
|
continue;
|
|
398
|
-
|
|
399
|
-
|
|
822
|
+
const id = text(row, 'id');
|
|
823
|
+
db.prepare(`UPDATE claims SET owner_key = ?, owner_kind = ?, owner_pid = ?, owner_start = ?, owner_session = ?, owner_thread = ?,
|
|
824
|
+
last_used_at = ?, last_action = ? WHERE id = ?`).run(caller.key, owner.kind, owner.process?.pid ?? null, owner.process?.start ?? null, owner.sessionId, owner.threadId, now, 'returned', id);
|
|
825
|
+
db.prepare('DELETE FROM holds WHERE claim_id = ?').run(id);
|
|
826
|
+
dropCheck(db, id);
|
|
827
|
+
return db.prepare('SELECT * FROM claims WHERE id = ?').get(id);
|
|
828
|
+
}
|
|
829
|
+
return undefined;
|
|
830
|
+
}
|
|
831
|
+
reconcileAll(verdicts, wanted, listings, me, keeper, now, stopped, notes) {
|
|
832
|
+
let listed = null;
|
|
833
|
+
const leases = [];
|
|
834
|
+
let returnedSinceSnapshot = false;
|
|
835
|
+
for (const [platform, devices] of listings) {
|
|
836
|
+
if (devices instanceof Error) {
|
|
837
|
+
notes.push(`could not check the ${platform} pool: ${devices.message}`);
|
|
400
838
|
continue;
|
|
401
|
-
try {
|
|
402
|
-
this.reconcile(backend, verdicts, notes);
|
|
403
839
|
}
|
|
404
|
-
|
|
405
|
-
|
|
840
|
+
const found = byId(devices);
|
|
841
|
+
for (const lease of this.recoverDead(platform, devices, found, verdicts, me, keeper, now, notes)) {
|
|
842
|
+
stopped.add(lease.name);
|
|
843
|
+
leases.push(lease);
|
|
844
|
+
}
|
|
845
|
+
this.forgetMissing(platform, found, verdicts, notes);
|
|
846
|
+
if (platform === wanted?.platform)
|
|
847
|
+
listed = found;
|
|
848
|
+
for (const row of this.store.db
|
|
849
|
+
.prepare('SELECT * FROM devices WHERE platform = ? AND device_id IS NOT NULL AND claim_id IS NULL ORDER BY name')
|
|
850
|
+
.all(platform)) {
|
|
851
|
+
const name = text(row, 'name');
|
|
852
|
+
const device = found.get(text(row, 'device_id'));
|
|
853
|
+
if (stopped.has(name) || !device?.booted)
|
|
854
|
+
continue;
|
|
855
|
+
if (verdicts.free.get(name) !== text(row, 'device_id')) {
|
|
856
|
+
returnedSinceSnapshot = true;
|
|
857
|
+
continue;
|
|
858
|
+
}
|
|
859
|
+
const lease = leaseDevice(this.store.db, name, {
|
|
860
|
+
reservation: this.leaseName(name, 'shutdown', devices),
|
|
861
|
+
work: 'shutdown',
|
|
862
|
+
holder: me,
|
|
863
|
+
keeper,
|
|
864
|
+
consolePort: consolePortOf(device.serial),
|
|
865
|
+
now,
|
|
866
|
+
});
|
|
867
|
+
if (!lease)
|
|
868
|
+
continue;
|
|
869
|
+
stopped.add(name);
|
|
870
|
+
leases.push(lease);
|
|
406
871
|
}
|
|
407
872
|
}
|
|
408
|
-
return
|
|
873
|
+
return { listed, leases, returnedSinceSnapshot };
|
|
409
874
|
}
|
|
410
|
-
|
|
875
|
+
recoverDead(platform, devices, found, verdicts, me, keeper, now, notes) {
|
|
411
876
|
const db = this.store.db;
|
|
412
|
-
const
|
|
413
|
-
const
|
|
414
|
-
for (const row of db.prepare('SELECT * FROM devices WHERE platform = ?').all(backend.platform)) {
|
|
877
|
+
const leased = leaseReservations(db);
|
|
878
|
+
for (const row of db.prepare('SELECT * FROM devices WHERE platform = ? AND device_id IS NULL').all(platform)) {
|
|
415
879
|
const name = text(row, 'name');
|
|
416
|
-
const
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
continue;
|
|
420
|
-
const verdict = judgeProcess({ pid: int(row, 'creator_pid'), start: text(row, 'creator_start') }, verdicts.table);
|
|
421
|
-
if (isProven(verdict))
|
|
422
|
-
notes.push(this.settleReservation(row, listed));
|
|
880
|
+
const creator = { pid: int(row, 'creator_pid'), start: text(row, 'creator_start') };
|
|
881
|
+
const seen = verdicts.reservations.get(name);
|
|
882
|
+
if (leased.has(name) || !seen || !sameProcess(seen, creator))
|
|
423
883
|
continue;
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
}
|
|
884
|
+
if (isProven(judgeProcess(creator, verdicts.table)))
|
|
885
|
+
notes.push(this.settleReservation(row, devices));
|
|
886
|
+
}
|
|
887
|
+
const taken = [];
|
|
888
|
+
for (const lease of allLeases(db)) {
|
|
889
|
+
const seen = verdicts.leases.get(lease.reservation);
|
|
890
|
+
if (lease.platform !== platform || !seen || !sameProcess(seen, lease.holder))
|
|
432
891
|
continue;
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
892
|
+
if (!isProven(judgeProcess(lease.holder, verdicts.table)))
|
|
893
|
+
continue;
|
|
894
|
+
const next = this.recoverLease(lease, found, me, keeper, now, notes);
|
|
895
|
+
if (next)
|
|
896
|
+
taken.push(next);
|
|
897
|
+
}
|
|
898
|
+
return taken;
|
|
899
|
+
}
|
|
900
|
+
recoverLease(lease, found, me, keeper, now, notes) {
|
|
901
|
+
const db = this.store.db;
|
|
902
|
+
const noun = NOUN[lease.platform];
|
|
903
|
+
const doing = { shutdown: 'shutting it down', delete: 'deleting it' };
|
|
904
|
+
const stopped = `the process ${doing[lease.work]} stopped before it finished`;
|
|
905
|
+
const device = found.get(lease.id);
|
|
906
|
+
if (!device) {
|
|
907
|
+
endLease(db, lease, 'gone', poolCap(this.config, lease.platform));
|
|
908
|
+
notes.push(`forgot pool ${noun} ${lease.name}: it no longer exists, and the process ${doing[lease.work]} had stopped`);
|
|
909
|
+
return null;
|
|
910
|
+
}
|
|
911
|
+
if (!device.booted)
|
|
912
|
+
return this.endOrRetire(lease, 'present', me, keeper, notes, stopped);
|
|
913
|
+
notes.push(`took over shutting down pool ${noun} ${lease.name}: ${stopped}`);
|
|
914
|
+
return handOver(db, lease, {
|
|
915
|
+
reservation: this.leaseName(lease.name, 'shutdown', [...found.values()]),
|
|
916
|
+
work: 'shutdown',
|
|
917
|
+
holder: me,
|
|
918
|
+
keeper,
|
|
919
|
+
consolePort: lease.consolePort ?? consolePortOf(device.serial),
|
|
920
|
+
now,
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
forgetMissing(platform, found, verdicts, notes) {
|
|
924
|
+
const db = this.store.db;
|
|
925
|
+
for (const row of db.prepare('SELECT * FROM devices WHERE platform = ? AND device_id IS NOT NULL').all(platform)) {
|
|
926
|
+
const name = text(row, 'name');
|
|
927
|
+
const id = text(row, 'device_id');
|
|
928
|
+
if (found.has(id) || optText(row, 'claim_id') !== null || verdicts.known.get(name) !== id)
|
|
929
|
+
continue;
|
|
930
|
+
db.prepare('DELETE FROM devices WHERE name = ?').run(name);
|
|
931
|
+
notes.push(`forgot pool ${NOUN[platform]} ${name}: it no longer exists`);
|
|
444
932
|
}
|
|
445
|
-
|
|
933
|
+
}
|
|
934
|
+
recover(recovery, me, keeper, notes) {
|
|
935
|
+
if (!recovery)
|
|
936
|
+
return [];
|
|
937
|
+
const now = this.clock.now();
|
|
938
|
+
const taken = [];
|
|
939
|
+
for (const [platform, devices] of recovery.listings) {
|
|
940
|
+
if (devices instanceof Error)
|
|
941
|
+
continue;
|
|
942
|
+
taken.push(...this.recoverDead(platform, devices, byId(devices), recovery.verdicts, me, keeper, now, notes));
|
|
943
|
+
}
|
|
944
|
+
return taken;
|
|
945
|
+
}
|
|
946
|
+
leaseName(name, work, listed) {
|
|
947
|
+
const db = this.store.db;
|
|
948
|
+
const taken = new Set(listed.map(d => d.name));
|
|
949
|
+
let reservation;
|
|
950
|
+
do
|
|
951
|
+
reservation = `${name}.${work}.${randomBytes(2).toString('hex')}`;
|
|
952
|
+
while (taken.has(reservation) ||
|
|
953
|
+
db.prepare('SELECT 1 FROM devices WHERE name = ?').get(reservation) ||
|
|
954
|
+
db.prepare('SELECT 1 FROM device_leases WHERE reservation = ?').get(reservation));
|
|
955
|
+
return reservation;
|
|
446
956
|
}
|
|
447
957
|
settleReservation(row, listed) {
|
|
448
958
|
const db = this.store.db;
|
|
@@ -457,47 +967,25 @@ export class Registry {
|
|
|
457
967
|
db.prepare('DELETE FROM devices WHERE name = ?').run(name);
|
|
458
968
|
return `dropped the reservation for ${name}, left when claiming was interrupted`;
|
|
459
969
|
}
|
|
460
|
-
|
|
461
|
-
return this.store.db
|
|
462
|
-
.prepare('SELECT * FROM claims WHERE last_used_at < ? AND owner_key != ? ORDER BY last_used_at ASC')
|
|
463
|
-
.all(now - this.config.idleMs, caller.key)
|
|
464
|
-
.filter(row => !isProven(this.verdictFor(row, verdicts)));
|
|
465
|
-
}
|
|
466
|
-
planPorts(wanted, listening, now, caller, verdicts) {
|
|
970
|
+
planPorts(wanted, listening, now) {
|
|
467
971
|
const held = new Map();
|
|
468
972
|
for (const row of this.store.db.prepare('SELECT port, claim_id FROM ports').all())
|
|
469
973
|
held.set(int(row, 'port'), text(row, 'claim_id'));
|
|
470
|
-
const idle = wanted.length > 0 ? this.idleLanes(now, caller, verdicts) : [];
|
|
471
974
|
const picks = [];
|
|
472
|
-
const takeovers = [];
|
|
473
975
|
for (const service of wanted) {
|
|
474
976
|
const block = this.config.blocks[service.type];
|
|
475
977
|
if (!block)
|
|
476
978
|
continue;
|
|
477
|
-
const usable = (p) => !listening.has(p) && !picks.some(x => x.port === p);
|
|
478
979
|
let port;
|
|
479
980
|
for (let p = block.first; p <= block.last && port === undefined; p++) {
|
|
480
|
-
|
|
481
|
-
if (usable(p) && (holder === undefined || takeovers.includes(holder)))
|
|
981
|
+
if (!held.has(p) && !listening.has(p) && !picks.some(x => x.port === p))
|
|
482
982
|
port = p;
|
|
483
983
|
}
|
|
484
|
-
for (const row of idle) {
|
|
485
|
-
if (port !== undefined)
|
|
486
|
-
break;
|
|
487
|
-
const id = text(row, 'id');
|
|
488
|
-
if (takeovers.includes(id))
|
|
489
|
-
continue;
|
|
490
|
-
const theirs = [...held].find(([p, c]) => c === id && p >= block.first && p <= block.last && usable(p));
|
|
491
|
-
if (theirs) {
|
|
492
|
-
takeovers.push(id);
|
|
493
|
-
port = theirs[0];
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
984
|
if (port === undefined)
|
|
497
985
|
return this.portsExhausted(service.type, held, listening, now);
|
|
498
986
|
picks.push({ service: service.name, type: service.type, port });
|
|
499
987
|
}
|
|
500
|
-
return { picks
|
|
988
|
+
return { picks };
|
|
501
989
|
}
|
|
502
990
|
planEmulatorPort(request, laneId, held, needDevice, listed, listening, now) {
|
|
503
991
|
const none = { add: null, drop: null };
|
|
@@ -515,10 +1003,13 @@ export class Registry {
|
|
|
515
1003
|
if (!occupied)
|
|
516
1004
|
return none;
|
|
517
1005
|
}
|
|
518
|
-
const taken = new Set(
|
|
519
|
-
.
|
|
520
|
-
|
|
521
|
-
|
|
1006
|
+
const taken = new Set([
|
|
1007
|
+
...this.store.db
|
|
1008
|
+
.prepare('SELECT port FROM ports WHERE port BETWEEN ? AND ?')
|
|
1009
|
+
.all(EMULATOR_PORTS.first, EMULATOR_PORTS.last)
|
|
1010
|
+
.map(r => int(r, 'port')),
|
|
1011
|
+
...leasedConsolePorts(this.store.db),
|
|
1012
|
+
]);
|
|
522
1013
|
const running = new Set([...listed.values()].flatMap(d => (d.serial ? [d.serial] : [])));
|
|
523
1014
|
const free = (p) => !taken.has(p) && !taken.has(p + 1) && !listening.has(p) && !listening.has(p + 1) && !running.has(emulatorSerial(p));
|
|
524
1015
|
const pick = emulatorConsolePorts().find(free);
|
|
@@ -526,6 +1017,20 @@ export class Registry {
|
|
|
526
1017
|
return this.emulatorPortsExhausted(listening, now);
|
|
527
1018
|
return { add: pick, drop: current };
|
|
528
1019
|
}
|
|
1020
|
+
freedSinceListing(seq, listings) {
|
|
1021
|
+
const db = this.store.db;
|
|
1022
|
+
for (const end of db.prepare('SELECT DISTINCT device FROM history WHERE seq > ? AND device IS NOT NULL').all(seq)) {
|
|
1023
|
+
const row = db
|
|
1024
|
+
.prepare('SELECT platform, device_id FROM devices WHERE name = ? AND claim_id IS NULL AND device_id IS NOT NULL')
|
|
1025
|
+
.get(text(end, 'device'));
|
|
1026
|
+
if (!row || !this.backends[text(row, 'platform')])
|
|
1027
|
+
continue;
|
|
1028
|
+
const listed = listings.get(text(row, 'platform'));
|
|
1029
|
+
if (!listed || listed instanceof Error || !listed.some(d => d.id === text(row, 'device_id') && d.booted))
|
|
1030
|
+
return true;
|
|
1031
|
+
}
|
|
1032
|
+
return false;
|
|
1033
|
+
}
|
|
529
1034
|
emulatorPortsExhausted(listening, now) {
|
|
530
1035
|
const db = this.store.db;
|
|
531
1036
|
const rows = db
|
|
@@ -551,10 +1056,12 @@ export class Registry {
|
|
|
551
1056
|
]);
|
|
552
1057
|
}
|
|
553
1058
|
memoryInUse(pool, isMine) {
|
|
1059
|
+
const leased = leaseReservations(this.store.db);
|
|
554
1060
|
let used = 0;
|
|
555
1061
|
for (const row of pool) {
|
|
556
1062
|
const busy = optText(row, 'claim_id') !== null || optText(row, 'device_id') === null;
|
|
557
|
-
|
|
1063
|
+
const deleting = leased.get(text(row, 'name')) === 'delete';
|
|
1064
|
+
if (busy && !deleting && !isMine(row))
|
|
558
1065
|
used += DEVICE_MEMORY_MB[text(row, 'platform')];
|
|
559
1066
|
}
|
|
560
1067
|
return used;
|
|
@@ -568,7 +1075,7 @@ export class Registry {
|
|
|
568
1075
|
const used = this.memoryInUse(pool, () => false);
|
|
569
1076
|
return `${counts('ios')}, ${counts('android')}; ${gigabytes(used)} of the ${gigabytes(this.config.memoryBudgetMb)} memory budget in use`;
|
|
570
1077
|
}
|
|
571
|
-
planDevice(spec, strict, backend, listed, me,
|
|
1078
|
+
planDevice(spec, strict, prefer, backend, listed, me, keeper, now, caller, progress) {
|
|
572
1079
|
const db = this.store.db;
|
|
573
1080
|
const platform = backend.platform;
|
|
574
1081
|
const noun = NOUN[platform];
|
|
@@ -582,11 +1089,16 @@ export class Registry {
|
|
|
582
1089
|
}
|
|
583
1090
|
const aheadRow = db
|
|
584
1091
|
.prepare(`SELECT COUNT(*) AS ahead FROM queue q WHERE q.ticket < ? AND NOT EXISTS (
|
|
585
|
-
SELECT 1 FROM devices d WHERE d.device_id IS NULL AND d.creator_pid = q.waiter_pid AND d.creator_start = q.waiter_start
|
|
1092
|
+
SELECT 1 FROM devices d WHERE d.device_id IS NULL AND d.creator_pid = q.waiter_pid AND d.creator_start = q.waiter_start
|
|
1093
|
+
AND NOT EXISTS (SELECT 1 FROM device_leases l WHERE l.reservation = d.name))`)
|
|
586
1094
|
.get(ticket);
|
|
587
1095
|
const ahead = aheadRow ? int(aheadRow, 'ahead') : 0;
|
|
588
1096
|
const pool = db.prepare('SELECT * FROM devices ORDER BY created_at, name').all();
|
|
589
|
-
const
|
|
1097
|
+
const leased = leaseReservations(db);
|
|
1098
|
+
const isMine = (r) => optText(r, 'device_id') === null &&
|
|
1099
|
+
optInt(r, 'creator_pid') === me.pid &&
|
|
1100
|
+
optText(r, 'creator_start') === me.start &&
|
|
1101
|
+
!leased.has(text(r, 'name'));
|
|
590
1102
|
const rows = pool.filter(r => text(r, 'platform') === platform);
|
|
591
1103
|
const mine = rows.find(isMine);
|
|
592
1104
|
const free = rows.filter(r => {
|
|
@@ -604,7 +1116,8 @@ export class Registry {
|
|
|
604
1116
|
if (ahead > 0 && Math.min(free.length + room, fits) <= ahead)
|
|
605
1117
|
return wait();
|
|
606
1118
|
if (fits > 0) {
|
|
607
|
-
const
|
|
1119
|
+
const matches = (r) => text(r, 'device_type') === spec.deviceType && text(r, 'runtime') === spec.runtime;
|
|
1120
|
+
const match = free.find(r => matches(r) && text(r, 'name') === prefer) ?? free.find(matches);
|
|
608
1121
|
if (match) {
|
|
609
1122
|
this.dropReservation(me);
|
|
610
1123
|
return { kind: 'assign', name: text(match, 'name') };
|
|
@@ -616,49 +1129,30 @@ export class Registry {
|
|
|
616
1129
|
progress.notes.push(`took free pool ${noun} ${text(spare, 'name')} (${text(spare, 'device_type_name')}, ${text(spare, 'runtime_name')}) though ${spec.deviceTypeName}, ${spec.runtimeName} was asked for: the pool is full, and a device type is a preference unless the lane is strict`);
|
|
617
1130
|
return { kind: 'assign', name: text(spare, 'name') };
|
|
618
1131
|
}
|
|
619
|
-
if (room > 0
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
const name = text(mine, 'name');
|
|
633
|
-
const id = backend.create(name, spec);
|
|
634
|
-
this.step('claim:created-device');
|
|
635
|
-
db.prepare('UPDATE devices SET device_id = ?, creator_pid = NULL, creator_start = NULL WHERE name = ?').run(id, name);
|
|
636
|
-
listed.set(id, { id, name, deviceType: spec.deviceType, runtime: spec.runtime, booted: false });
|
|
637
|
-
progress.notes.push(`created pool ${noun} ${name} (${spec.deviceTypeName}, ${spec.runtimeName}, ${id})`);
|
|
638
|
-
return { kind: 'assign', name };
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
if (depth === 0) {
|
|
642
|
-
const deviceSlot = free.length + room > 0;
|
|
643
|
-
const idle = this.idleLanes(now, caller, verdicts).find(row => {
|
|
644
|
-
const device = db.prepare('SELECT platform FROM devices WHERE claim_id = ?').get(text(row, 'id'));
|
|
645
|
-
if (!device)
|
|
646
|
-
return false;
|
|
647
|
-
const theirs = text(device, 'platform');
|
|
648
|
-
if (theirs === platform)
|
|
649
|
-
return true;
|
|
650
|
-
return deviceSlot && spareMemory + DEVICE_MEMORY_MB[theirs] >= DEVICE_MEMORY_MB[platform];
|
|
651
|
-
});
|
|
652
|
-
if (idle) {
|
|
653
|
-
progress.tookOver.push(this.end(idle, 'taken-over', caller, now));
|
|
654
|
-
this.step('claim:took-over');
|
|
655
|
-
return this.planDevice(spec, strict, backend, listed, me, verdicts, now, caller, progress, depth + 1);
|
|
1132
|
+
if (room > 0)
|
|
1133
|
+
return this.reserve(spec, backend, listed, me, now, mine, null);
|
|
1134
|
+
if (spare) {
|
|
1135
|
+
const retire = leaseDevice(db, text(spare, 'name'), {
|
|
1136
|
+
reservation: this.leaseName(text(spare, 'name'), 'delete', [...listed.values()]),
|
|
1137
|
+
work: 'delete',
|
|
1138
|
+
holder: me,
|
|
1139
|
+
keeper,
|
|
1140
|
+
consolePort: null,
|
|
1141
|
+
now,
|
|
1142
|
+
});
|
|
1143
|
+
if (retire)
|
|
1144
|
+
return this.reserve(spec, backend, listed, me, now, mine, retire);
|
|
656
1145
|
}
|
|
657
1146
|
}
|
|
658
1147
|
return wait();
|
|
659
1148
|
}
|
|
660
|
-
reserve(spec, backend, listed, me, now) {
|
|
1149
|
+
reserve(spec, backend, listed, me, now, mine, retire) {
|
|
661
1150
|
const db = this.store.db;
|
|
1151
|
+
if (mine) {
|
|
1152
|
+
const name = text(mine, 'name');
|
|
1153
|
+
db.prepare('UPDATE devices SET device_type = ?, device_type_name = ?, runtime = ?, runtime_name = ? WHERE name = ?').run(spec.deviceType, spec.deviceTypeName, spec.runtime, spec.runtimeName, name);
|
|
1154
|
+
return { kind: 'create', backend, name, spec, retire };
|
|
1155
|
+
}
|
|
662
1156
|
const taken = new Set([...listed.values()].map(d => d.name));
|
|
663
1157
|
let name;
|
|
664
1158
|
do {
|
|
@@ -668,16 +1162,13 @@ export class Registry {
|
|
|
668
1162
|
creator_pid, creator_start, claim_id, created_at)
|
|
669
1163
|
VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, NULL, ?)`).run(name, backend.platform, spec.deviceType, spec.deviceTypeName, spec.runtime, spec.runtimeName, me.pid, me.start, now);
|
|
670
1164
|
this.step('claim:reserved');
|
|
671
|
-
return { kind: '
|
|
1165
|
+
return { kind: 'create', backend, name, spec, retire };
|
|
672
1166
|
}
|
|
673
1167
|
end(row, reason, by, now) {
|
|
674
1168
|
const db = this.store.db;
|
|
675
1169
|
const id = text(row, 'id');
|
|
676
1170
|
const device = db.prepare('SELECT name FROM devices WHERE claim_id = ?').get(id);
|
|
677
|
-
const processes = db
|
|
678
|
-
.prepare('SELECT * FROM processes WHERE claim_id = ? ORDER BY service')
|
|
679
|
-
.all(id)
|
|
680
|
-
.map(p => ({ service: text(p, 'service'), pid: int(p, 'pid'), start: text(p, 'start'), cwd: optText(p, 'cwd') }));
|
|
1171
|
+
const processes = processesOf(db, id);
|
|
681
1172
|
const ended = {
|
|
682
1173
|
id,
|
|
683
1174
|
owner: describeOwner(ownerFromRow(row)),
|
|
@@ -697,24 +1188,32 @@ export class Registry {
|
|
|
697
1188
|
db.prepare('DELETE FROM claims WHERE id = ?').run(id);
|
|
698
1189
|
return ended;
|
|
699
1190
|
}
|
|
700
|
-
|
|
1191
|
+
endAndLease(rows, reason, by, me, keeper) {
|
|
701
1192
|
const db = this.store.db;
|
|
702
1193
|
const now = this.clock.now();
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
const
|
|
1194
|
+
const leases = [];
|
|
1195
|
+
const ended = rows.map(row => {
|
|
1196
|
+
const id = text(row, 'id');
|
|
1197
|
+
const device = db.prepare('SELECT * FROM devices WHERE claim_id = ?').get(id);
|
|
1198
|
+
const console = db.prepare('SELECT port FROM ports WHERE claim_id = ? AND service = ?').get(id, EMULATOR_SERVICE);
|
|
1199
|
+
const lane = this.end(row, reason, by, now);
|
|
706
1200
|
this.step('release:ended');
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
1201
|
+
if (device && this.backends[text(device, 'platform')]) {
|
|
1202
|
+
const name = text(device, 'name');
|
|
1203
|
+
const lease = leaseDevice(db, name, {
|
|
1204
|
+
reservation: this.leaseName(name, 'shutdown', []),
|
|
1205
|
+
work: 'shutdown',
|
|
1206
|
+
holder: me,
|
|
1207
|
+
keeper,
|
|
1208
|
+
consolePort: console ? int(console, 'port') : null,
|
|
1209
|
+
now,
|
|
1210
|
+
});
|
|
1211
|
+
if (lease)
|
|
1212
|
+
leases.push(lease);
|
|
715
1213
|
}
|
|
716
|
-
return
|
|
1214
|
+
return lane;
|
|
717
1215
|
});
|
|
1216
|
+
return { ended, leases };
|
|
718
1217
|
}
|
|
719
1218
|
holderOf(row, now) {
|
|
720
1219
|
return {
|
|
@@ -734,13 +1233,23 @@ export class Registry {
|
|
|
734
1233
|
const holder = this.holderOf(row, now);
|
|
735
1234
|
holders.push({ ...holder, detail: `${text(row, 'device_name')}, ${holder.detail ?? ''}` });
|
|
736
1235
|
}
|
|
1236
|
+
const leases = new Map(allLeases(db).map(l => [l.reservation, l]));
|
|
737
1237
|
for (const row of db.prepare('SELECT * FROM devices WHERE device_id IS NULL').all()) {
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
1238
|
+
const article = text(row, 'platform') === 'android' ? 'an emulator' : 'a simulator';
|
|
1239
|
+
const lease = leases.get(text(row, 'name'));
|
|
1240
|
+
holders.push(lease
|
|
1241
|
+
? {
|
|
1242
|
+
who: `a process ${lease.work === 'shutdown' ? 'shutting down' : 'deleting'} ${lease.name} (pid ${String(lease.holder.pid)})`,
|
|
1243
|
+
folder: `(freeing ${article})`,
|
|
1244
|
+
since: formatTime(int(row, 'created_at')),
|
|
1245
|
+
detail: lease.reservation,
|
|
1246
|
+
}
|
|
1247
|
+
: {
|
|
1248
|
+
who: `a process claiming a lane (pid ${String(int(row, 'creator_pid'))})`,
|
|
1249
|
+
folder: `(creating ${article})`,
|
|
1250
|
+
since: formatTime(int(row, 'created_at')),
|
|
1251
|
+
detail: text(row, 'name'),
|
|
1252
|
+
});
|
|
744
1253
|
}
|
|
745
1254
|
for (const row of db.prepare('SELECT * FROM queue WHERE ticket < ? ORDER BY ticket').all(ticket)) {
|
|
746
1255
|
holders.push({
|
|
@@ -763,6 +1272,25 @@ export class Registry {
|
|
|
763
1272
|
`set "${cap}": ${String(poolCap(this.config, wait.platform) + 1)} or "memoryBudgetGB": ${String(Math.ceil((this.config.memoryBudgetMb + DEVICE_MEMORY_MB[wait.platform]) / 1000))} in ${configPath(this.dir)} (allow one more ${NOUN[wait.platform]})`,
|
|
764
1273
|
]);
|
|
765
1274
|
}
|
|
1275
|
+
poolChanging(request, since, started) {
|
|
1276
|
+
const now = this.clock.now();
|
|
1277
|
+
const ends = this.store.db
|
|
1278
|
+
.prepare('SELECT * FROM history WHERE seq > ? AND device IS NOT NULL ORDER BY seq')
|
|
1279
|
+
.all(since)
|
|
1280
|
+
.map(row => ({
|
|
1281
|
+
who: `${text(row, 'owner_label')}, whose lane ${text(row, 'claim_id')} held ${text(row, 'device')}`,
|
|
1282
|
+
folder: text(row, 'worktree'),
|
|
1283
|
+
since: formatTime(int(row, 'created_at')),
|
|
1284
|
+
detail: `ended ${formatTime(int(row, 'ended_at'))} (${text(row, 'reason')}) by ${text(row, 'by_label')}`,
|
|
1285
|
+
}));
|
|
1286
|
+
const waited = request.waitMs > 0
|
|
1287
|
+
? `its --timeout has passed, ${formatAge(now - started)} after it started`
|
|
1288
|
+
: 'it did not wait (--timeout 0)';
|
|
1289
|
+
return new Refusal('pool-busy', `the device pool kept changing while this claim looked at it: during each of its last two listings a device went back to the pool, and that listing may show it in an old state; ${waited}`, ends.length > 0 ? ends : [{ who: 'no lane the registry can see', folder: this.dir, since: formatTime(now) }], [
|
|
1290
|
+
`${CLI} claim --timeout 1800 (keep looking for up to 30 minutes)`,
|
|
1291
|
+
`${CLI} status (the lanes on this Mac and their devices)`,
|
|
1292
|
+
]);
|
|
1293
|
+
}
|
|
766
1294
|
portsExhausted(type, held, listening, now) {
|
|
767
1295
|
const db = this.store.db;
|
|
768
1296
|
const block = this.config.blocks[type];
|
|
@@ -834,48 +1362,250 @@ export class Registry {
|
|
|
834
1362
|
.get(ownerKey(owner), worktree);
|
|
835
1363
|
return row ? loadLane(this.store.db, text(row, 'id')) : null;
|
|
836
1364
|
}
|
|
1365
|
+
/**
|
|
1366
|
+
* As `find`, for a call by `owner` itself: the lane is its owner's again if a check took the owner for gone, and a
|
|
1367
|
+
* gone process of the same Claude Code session gives `owner` its lane in this folder while the lane's window lasts.
|
|
1368
|
+
* Refused while the lane is being ended.
|
|
1369
|
+
*/
|
|
1370
|
+
takeBack(owner, worktree) {
|
|
1371
|
+
const db = this.store.db;
|
|
1372
|
+
const key = ownerKey(owner);
|
|
1373
|
+
const mine = db.prepare('SELECT * FROM claims WHERE owner_key = ? AND worktree = ?').get(key, worktree);
|
|
1374
|
+
const session = hostSession(owner);
|
|
1375
|
+
if (mine) {
|
|
1376
|
+
const gone = readCheck(db, text(mine, 'id'), key)?.gone ?? null;
|
|
1377
|
+
const moved = owner.process !== null && !sameIdentity(ownerFromRow(mine).process, owner.process);
|
|
1378
|
+
if (gone === null && !moved)
|
|
1379
|
+
return loadLane(db, text(mine, 'id'));
|
|
1380
|
+
}
|
|
1381
|
+
else if (owner.kind !== 'claude' || session === null)
|
|
1382
|
+
return null;
|
|
1383
|
+
const others = mine
|
|
1384
|
+
? []
|
|
1385
|
+
: db
|
|
1386
|
+
.prepare(`SELECT * FROM claims WHERE worktree = ? AND owner_kind = 'claude' AND owner_session = ? AND owner_key != ?`)
|
|
1387
|
+
.all(worktree, session, key);
|
|
1388
|
+
const pids = others.flatMap(row => {
|
|
1389
|
+
const identity = ownerFromRow(row).process;
|
|
1390
|
+
return identity ? [identity.pid] : [];
|
|
1391
|
+
});
|
|
1392
|
+
if (!mine && others.length === 0)
|
|
1393
|
+
return null;
|
|
1394
|
+
const table = pids.length > 0 ? this.processes.startTimes(pids) : new Map();
|
|
1395
|
+
return this.store.write(() => {
|
|
1396
|
+
const caller = this.caller(owner, worktree);
|
|
1397
|
+
const now = this.clock.now();
|
|
1398
|
+
const own = db.prepare('SELECT * FROM claims WHERE owner_key = ? AND worktree = ?').get(key, worktree);
|
|
1399
|
+
if (own) {
|
|
1400
|
+
this.recognise(own, owner);
|
|
1401
|
+
return loadLane(db, text(own, 'id'));
|
|
1402
|
+
}
|
|
1403
|
+
const back = this.returning(owner, caller, now, row => isProven(judgeProcess(ownerFromRow(row).process, table)));
|
|
1404
|
+
return back ? loadLane(db, text(back, 'id')) : null;
|
|
1405
|
+
});
|
|
1406
|
+
}
|
|
1407
|
+
/** Every lane, with what the sweep needs to judge it, in one read of the registry. */
|
|
1408
|
+
sweepList() {
|
|
1409
|
+
const db = this.store.db;
|
|
1410
|
+
const now = this.clock.now();
|
|
1411
|
+
return db
|
|
1412
|
+
.prepare('SELECT * FROM claims ORDER BY last_used_at, id')
|
|
1413
|
+
.all()
|
|
1414
|
+
.map(row => {
|
|
1415
|
+
const id = text(row, 'id');
|
|
1416
|
+
const key = text(row, 'owner_key');
|
|
1417
|
+
const check = readCheck(db, id, key);
|
|
1418
|
+
return {
|
|
1419
|
+
ref: { id, token: text(row, 'token') },
|
|
1420
|
+
owner: ownerFromRow(row),
|
|
1421
|
+
ownerKey: key,
|
|
1422
|
+
worktree: text(row, 'worktree'),
|
|
1423
|
+
lastUsedAt: int(row, 'last_used_at'),
|
|
1424
|
+
checkedAt: check?.checkedAt ?? null,
|
|
1425
|
+
gone: check?.gone ?? null,
|
|
1426
|
+
park: readPark(db, id),
|
|
1427
|
+
hold: readHold(db, id, key, now),
|
|
1428
|
+
running: db.prepare('SELECT 1 FROM processes WHERE claim_id = ? LIMIT 1').get(id) !== undefined,
|
|
1429
|
+
};
|
|
1430
|
+
});
|
|
1431
|
+
}
|
|
1432
|
+
/**
|
|
1433
|
+
* Records what a check of the lanes' owners found, in one write: `seen` alive or past telling, `gone` with what the
|
|
1434
|
+
* check found. Each counts only while the lane's owner is still the one the check judged. Returns the lanes whose
|
|
1435
|
+
* owner this check is the first to find gone; their holds end.
|
|
1436
|
+
*/
|
|
1437
|
+
noteOwners(seen, gone) {
|
|
1438
|
+
if (seen.length + gone.length === 0)
|
|
1439
|
+
return [];
|
|
1440
|
+
return this.store.write(() => {
|
|
1441
|
+
const db = this.store.db;
|
|
1442
|
+
const now = this.clock.now();
|
|
1443
|
+
const current = (judged) => {
|
|
1444
|
+
const row = db.prepare('SELECT * FROM claims WHERE id = ? AND owner_key = ?').get(judged.id, judged.ownerKey);
|
|
1445
|
+
return row && sameIdentity(ownerFromRow(row).process, judged.process) ? row : undefined;
|
|
1446
|
+
};
|
|
1447
|
+
for (const judged of seen)
|
|
1448
|
+
if (current(judged))
|
|
1449
|
+
noteChecked(db, judged.id, judged.ownerKey, now);
|
|
1450
|
+
const found = [];
|
|
1451
|
+
for (const judged of gone) {
|
|
1452
|
+
const row = current(judged);
|
|
1453
|
+
if (row && this.foundGone(row, judged.state, now))
|
|
1454
|
+
found.push(judged.id);
|
|
1455
|
+
}
|
|
1456
|
+
return found;
|
|
1457
|
+
});
|
|
1458
|
+
}
|
|
1459
|
+
/**
|
|
1460
|
+
* Takes the lane for ending as attempt `attempt` of this process, once its owner has stayed gone for RETURN_WINDOW_MS:
|
|
1461
|
+
* `dead` names processes proven gone before this call, whose end or park of the lane may be taken over. Left while a
|
|
1462
|
+
* service it records may still run, since its ports would go to another lane.
|
|
1463
|
+
*/
|
|
1464
|
+
beginReap(ref, attempt, dead) {
|
|
1465
|
+
const me = this.self();
|
|
1466
|
+
return this.store.write(() => {
|
|
1467
|
+
const db = this.store.db;
|
|
1468
|
+
const row = db.prepare('SELECT * FROM claims WHERE id = ?').get(ref.id);
|
|
1469
|
+
if (!row || text(row, 'token') !== ref.token)
|
|
1470
|
+
return { kind: 'left', reason: 'it has ended' };
|
|
1471
|
+
this.store.actor = { who: `${TOOL} ending lane ${ref.id}`, folder: text(row, 'worktree') };
|
|
1472
|
+
const gone = readCheck(db, ref.id, text(row, 'owner_key'))?.gone ?? null;
|
|
1473
|
+
const now = this.clock.now();
|
|
1474
|
+
const isDead = (p) => p !== null && dead.some(d => sameProcess(d, p));
|
|
1475
|
+
if (!gone)
|
|
1476
|
+
return { kind: 'left', reason: 'its owner came back' };
|
|
1477
|
+
if (now < gone.foundAt + RETURN_WINDOW_MS)
|
|
1478
|
+
return { kind: 'left', reason: `its owner may come back until ${formatTime(gone.foundAt + RETURN_WINDOW_MS)}` };
|
|
1479
|
+
if (gone.reaper && !isDead(gone.reaper))
|
|
1480
|
+
return { kind: 'left', reason: `pid ${String(gone.reaper.pid)} is ending it` };
|
|
1481
|
+
const park = readPark(db, ref.id);
|
|
1482
|
+
if (park && park.state !== 'parked' && !isDead(park.holder)) {
|
|
1483
|
+
return {
|
|
1484
|
+
kind: 'left',
|
|
1485
|
+
reason: `pid ${String(park.holder?.pid ?? 'unknown')} is ${park.state === 'waking' ? 'waking' : 'parking'} it`,
|
|
1486
|
+
};
|
|
1487
|
+
}
|
|
1488
|
+
const running = processesOf(db, ref.id).map(p => p.service);
|
|
1489
|
+
if (running.length > 0)
|
|
1490
|
+
return { kind: 'left', reason: `it still records ${running.join(', ')}, which no stop proved down` };
|
|
1491
|
+
startReap(db, ref.id, me, attempt, now);
|
|
1492
|
+
const lane = loadLane(db, ref.id);
|
|
1493
|
+
if (!lane)
|
|
1494
|
+
throw new Error(`lane ${ref.id} vanished inside its own transaction`);
|
|
1495
|
+
return { kind: 'taken', lane, gone: { ...gone, reaper: me, reapingSince: now, attempt } };
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1498
|
+
/** Gives up attempt `attempt` at ending the lane, so a later sweep tries again. */
|
|
1499
|
+
abortReap(ref, attempt) {
|
|
1500
|
+
this.store.write(() => {
|
|
1501
|
+
stopReap(this.store.db, ref.id, attempt);
|
|
1502
|
+
});
|
|
1503
|
+
}
|
|
1504
|
+
/**
|
|
1505
|
+
* Ends the lane attempt `attempt` of this process took for ending, as a release does: its device goes back to the
|
|
1506
|
+
* pool once it is shut down, and the resources it made wait for their teardown. Null when the attempt no longer holds it.
|
|
1507
|
+
*/
|
|
1508
|
+
endReaped(ref, attempt, notes = []) {
|
|
1509
|
+
const me = this.self();
|
|
1510
|
+
const by = { key: `reap:${String(me.pid)}:${me.start}`, label: `${TOOL} reap (pid ${String(me.pid)})`, worktree: '' };
|
|
1511
|
+
const [ended] = this.releasing(notes, () => undefined, (mine, keeper) => {
|
|
1512
|
+
const db = this.store.db;
|
|
1513
|
+
const row = db.prepare('SELECT * FROM claims WHERE id = ? AND token = ?').get(ref.id, ref.token);
|
|
1514
|
+
const gone = row ? (readCheck(db, ref.id, text(row, 'owner_key'))?.gone ?? null) : null;
|
|
1515
|
+
if (!row || gone?.attempt !== attempt || !gone.reaper || !sameProcess(gone.reaper, mine))
|
|
1516
|
+
return { ended: [], leases: [] };
|
|
1517
|
+
by.worktree = text(row, 'worktree');
|
|
1518
|
+
this.store.actor = { who: by.label, folder: by.worktree };
|
|
1519
|
+
return this.endAndLease([row], gone.state, by, mine, keeper);
|
|
1520
|
+
});
|
|
1521
|
+
return ended ?? null;
|
|
1522
|
+
}
|
|
1523
|
+
/** Whether this process holds device work it could not record as done in `dir`'s registry, for `settleOwn` to land. */
|
|
1524
|
+
static holdsLeftovers(dir) {
|
|
1525
|
+
const prefix = `${dir}\0${String(process.pid)}\0`;
|
|
1526
|
+
return [...unsettled.keys()].some(key => key.startsWith(prefix));
|
|
1527
|
+
}
|
|
1528
|
+
/** Lands the device work this process could not record as done earlier, as every claim and release does first. */
|
|
1529
|
+
settleOwn(notes = []) {
|
|
1530
|
+
const me = this.self();
|
|
1531
|
+
const keeper = this.keeper();
|
|
1532
|
+
const pending = this.unstash(me);
|
|
1533
|
+
let work;
|
|
1534
|
+
try {
|
|
1535
|
+
const listings = this.listOwn(me, pending);
|
|
1536
|
+
work = this.store.write(() => {
|
|
1537
|
+
const others = this.landLeftovers(pending, listings, me, keeper, notes);
|
|
1538
|
+
return this.inWrite({ own: [], others }, me, keeper, notes, 'release:stopped-device');
|
|
1539
|
+
});
|
|
1540
|
+
}
|
|
1541
|
+
catch (error) {
|
|
1542
|
+
this.stash(me, pending);
|
|
1543
|
+
throw error;
|
|
1544
|
+
}
|
|
1545
|
+
this.afterWrite(work, me, keeper, notes, 'release:stopped-device');
|
|
1546
|
+
}
|
|
1547
|
+
releasing(notes, judge, end) {
|
|
1548
|
+
const me = this.self();
|
|
1549
|
+
const keeper = this.keeper();
|
|
1550
|
+
const pending = this.unstash(me);
|
|
1551
|
+
let released;
|
|
1552
|
+
try {
|
|
1553
|
+
const verdicts = judge();
|
|
1554
|
+
const recovery = this.recoveryBeforeLock(verdicts);
|
|
1555
|
+
const listings = this.listOwn(me, pending, recovery?.listings);
|
|
1556
|
+
released = this.store.write(() => {
|
|
1557
|
+
this.step('release:locked');
|
|
1558
|
+
const others = [
|
|
1559
|
+
...this.landLeftovers(pending, listings, me, keeper, notes),
|
|
1560
|
+
...this.recover(recovery, me, keeper, notes),
|
|
1561
|
+
];
|
|
1562
|
+
const { ended, leases } = end(me, keeper, verdicts);
|
|
1563
|
+
return { ended, work: this.inWrite({ own: leases, others }, me, keeper, notes, 'release:stopped-device') };
|
|
1564
|
+
});
|
|
1565
|
+
}
|
|
1566
|
+
catch (error) {
|
|
1567
|
+
this.stash(me, pending);
|
|
1568
|
+
throw error;
|
|
1569
|
+
}
|
|
1570
|
+
this.step('release:committed');
|
|
1571
|
+
this.afterWrite(released.work, me, keeper, notes, 'release:stopped-device');
|
|
1572
|
+
return released.ended;
|
|
1573
|
+
}
|
|
837
1574
|
release(ref, by, notes = []) {
|
|
838
1575
|
const caller = this.caller(by, '');
|
|
839
|
-
const [ended] = this.
|
|
840
|
-
this.step('release:locked');
|
|
1576
|
+
const [ended] = this.releasing(notes, () => undefined, (me, keeper) => {
|
|
841
1577
|
const row = this.requireLane(ref);
|
|
842
1578
|
caller.worktree = text(row, 'worktree');
|
|
843
|
-
return this.
|
|
1579
|
+
return this.endAndLease([row], 'released', caller, me, keeper);
|
|
844
1580
|
});
|
|
845
|
-
this.step('release:committed');
|
|
846
1581
|
if (!ended)
|
|
847
1582
|
throw new Error(`releasing lane ${ref.id} ended nothing`);
|
|
848
1583
|
return ended;
|
|
849
1584
|
}
|
|
850
1585
|
releaseOwn(owner, worktree, notes = []) {
|
|
851
1586
|
const caller = this.caller(owner, worktree ?? '');
|
|
852
|
-
|
|
853
|
-
this.step('release:locked');
|
|
1587
|
+
return this.releasing(notes, () => undefined, (me, keeper) => {
|
|
854
1588
|
const db = this.store.db;
|
|
855
1589
|
const rows = worktree === null
|
|
856
1590
|
? db.prepare('SELECT * FROM claims WHERE owner_key = ? ORDER BY created_at').all(caller.key)
|
|
857
1591
|
: db.prepare('SELECT * FROM claims WHERE owner_key = ? AND worktree = ?').all(caller.key, worktree);
|
|
858
|
-
return this.
|
|
1592
|
+
return this.endAndLease(rows, 'released', caller, me, keeper);
|
|
859
1593
|
});
|
|
860
|
-
this.step('release:committed');
|
|
861
|
-
return ended;
|
|
862
1594
|
}
|
|
863
1595
|
releaseById(id, by, force, notes = []) {
|
|
864
1596
|
const caller = this.caller(by, '');
|
|
865
|
-
const
|
|
866
|
-
const [ended] = this.store.write(() => {
|
|
867
|
-
this.step('release:locked');
|
|
1597
|
+
const [ended] = this.releasing(notes, () => this.snapshotBeforeLock(), (me, keeper, verdicts) => {
|
|
868
1598
|
const row = this.store.db.prepare('SELECT * FROM claims WHERE id = ?').get(id);
|
|
869
1599
|
if (!row)
|
|
870
1600
|
throw this.laneGone(id);
|
|
871
1601
|
caller.worktree = text(row, 'worktree');
|
|
872
1602
|
if (text(row, 'owner_key') === caller.key)
|
|
873
|
-
return this.
|
|
1603
|
+
return this.endAndLease([row], 'released', caller, me, keeper);
|
|
874
1604
|
const state = ownerState(this.verdictFor(row, verdicts), worktreeState(text(row, 'worktree')));
|
|
875
1605
|
if (isProven(state))
|
|
876
|
-
return this.
|
|
1606
|
+
return this.endAndLease([row], state, caller, me, keeper);
|
|
877
1607
|
if (force)
|
|
878
|
-
return this.
|
|
1608
|
+
return this.endAndLease([row], 'forced', caller, me, keeper);
|
|
879
1609
|
const now = this.clock.now();
|
|
880
1610
|
const forced = `${CLI} release --lane ${id} --force`;
|
|
881
1611
|
if (state === 'cant-tell') {
|
|
@@ -886,12 +1616,13 @@ export class Registry {
|
|
|
886
1616
|
`${forced} (only if you know that session no longer needs it)`,
|
|
887
1617
|
]);
|
|
888
1618
|
});
|
|
889
|
-
this.step('release:committed');
|
|
890
1619
|
if (!ended)
|
|
891
1620
|
throw new Error(`releasing lane ${id} ended nothing`);
|
|
892
1621
|
return ended;
|
|
893
1622
|
}
|
|
894
1623
|
touch(ref, action) {
|
|
1624
|
+
if (this.store.actor.who === UNIDENTIFIED)
|
|
1625
|
+
this.store.actor = { ...this.store.actor, who: `${TOOL} ${action}, on lane ${ref.id}` };
|
|
895
1626
|
return this.store.write(() => {
|
|
896
1627
|
this.step('touch:locked');
|
|
897
1628
|
this.requireLane(ref);
|
|
@@ -904,9 +1635,248 @@ export class Registry {
|
|
|
904
1635
|
return lane;
|
|
905
1636
|
});
|
|
906
1637
|
}
|
|
1638
|
+
/** How the lane is parked, or being parked or woken, and what its next call restarts; null while it is awake. */
|
|
1639
|
+
parkOf(id) {
|
|
1640
|
+
return readPark(this.store.db, id);
|
|
1641
|
+
}
|
|
1642
|
+
/**
|
|
1643
|
+
* The lanes the sweep looks at: last used, by the registry's record, at or before `idleSince`, and either awake, or
|
|
1644
|
+
* last parked or woken at or before `retrySince` and left holding a device or a service, or left mid-park or mid-wake.
|
|
1645
|
+
* A held lane is listed with its hold, so the sweep can judge its owner: a hold keeps only a live owner's lane up.
|
|
1646
|
+
*/
|
|
1647
|
+
parkCandidates(idleSince, retrySince) {
|
|
1648
|
+
const db = this.store.db;
|
|
1649
|
+
const now = this.clock.now();
|
|
1650
|
+
return db
|
|
1651
|
+
.prepare(`SELECT c.* FROM claims c LEFT JOIN parks p ON p.claim_id = c.id
|
|
1652
|
+
WHERE c.last_used_at <= ?
|
|
1653
|
+
AND (p.claim_id IS NULL OR (p.since <= ? AND (p.state != 'parked'
|
|
1654
|
+
OR EXISTS (SELECT 1 FROM devices d WHERE d.claim_id = c.id)
|
|
1655
|
+
OR EXISTS (SELECT 1 FROM processes r WHERE r.claim_id = c.id))))
|
|
1656
|
+
ORDER BY c.last_used_at, c.id`)
|
|
1657
|
+
.all(idleSince, retrySince)
|
|
1658
|
+
.map(row => ({
|
|
1659
|
+
ref: { id: text(row, 'id'), token: text(row, 'token') },
|
|
1660
|
+
lastUsedAt: int(row, 'last_used_at'),
|
|
1661
|
+
owner: ownerFromRow(row),
|
|
1662
|
+
ownerKey: text(row, 'owner_key'),
|
|
1663
|
+
worktree: text(row, 'worktree'),
|
|
1664
|
+
park: readPark(db, text(row, 'id')),
|
|
1665
|
+
hold: readHold(db, text(row, 'id'), text(row, 'owner_key'), now),
|
|
1666
|
+
}));
|
|
1667
|
+
}
|
|
1668
|
+
/**
|
|
1669
|
+
* Holds the lane up past the park window for `forMs`, in place of any hold before, and counts as a call on it. A
|
|
1670
|
+
* parked lane stays parked: its next call brings it back, and the hold keeps it up from then.
|
|
1671
|
+
*/
|
|
1672
|
+
keep(ref, reason, forMs) {
|
|
1673
|
+
return this.store.write(() => {
|
|
1674
|
+
const row = this.requireLane(ref);
|
|
1675
|
+
const db = this.store.db;
|
|
1676
|
+
const now = this.clock.now();
|
|
1677
|
+
const owner = text(row, 'owner_key');
|
|
1678
|
+
const replaced = readHold(db, ref.id, owner, now);
|
|
1679
|
+
const hold = { reason, since: now, until: now + forMs };
|
|
1680
|
+
writeHold(db, ref.id, owner, hold);
|
|
1681
|
+
db.prepare('UPDATE claims SET last_used_at = ?, last_action = ? WHERE id = ?').run(now, 'keep', ref.id);
|
|
1682
|
+
const lane = loadLane(db, ref.id);
|
|
1683
|
+
if (!lane)
|
|
1684
|
+
throw new Error(`lane ${ref.id} vanished inside its own transaction`);
|
|
1685
|
+
return { lane, hold, replaced, park: readPark(db, ref.id) };
|
|
1686
|
+
});
|
|
1687
|
+
}
|
|
1688
|
+
/** Ends the hold the owner keyed `ownerKey` placed on the lane, as the sweep does once it finds that owner dead. */
|
|
1689
|
+
endHold(laneId, ownerKey) {
|
|
1690
|
+
return this.store.write(() => dropHold(this.store.db, laneId, ownerKey));
|
|
1691
|
+
}
|
|
1692
|
+
parkActor(row, doing) {
|
|
1693
|
+
this.store.actor = { who: `${TOOL} ${doing} lane ${text(row, 'id')}`, folder: text(row, 'worktree') };
|
|
1694
|
+
}
|
|
1695
|
+
heldByDead(park, dead) {
|
|
1696
|
+
return dead !== null && park.holder !== null && sameProcess(dead, park.holder);
|
|
1697
|
+
}
|
|
1698
|
+
/**
|
|
1699
|
+
* Starts parking the lane as attempt `attempt` of this process, if the registry's record still has it unused since
|
|
1700
|
+
* `idleSince`, no hold keeps it up, and nobody else is parking or waking it; `dead` is the holder of an earlier park
|
|
1701
|
+
* or wake that was proven gone before this call. With `owner-gone` instead of a time, the lane is parked while a check
|
|
1702
|
+
* has its owner as gone and nothing has begun to end it, however lately it was used. Null when it is not to be parked now.
|
|
1703
|
+
*/
|
|
1704
|
+
beginPark(ref, idleSince, attempt, dead) {
|
|
1705
|
+
const me = this.self();
|
|
1706
|
+
return this.store.write(() => {
|
|
1707
|
+
const row = this.requireLane(ref);
|
|
1708
|
+
this.parkActor(row, 'parking');
|
|
1709
|
+
const db = this.store.db;
|
|
1710
|
+
if (idleSince === 'owner-gone') {
|
|
1711
|
+
const gone = readCheck(db, ref.id, text(row, 'owner_key'))?.gone;
|
|
1712
|
+
if (!gone || gone.reaper)
|
|
1713
|
+
return null;
|
|
1714
|
+
}
|
|
1715
|
+
else {
|
|
1716
|
+
if (int(row, 'last_used_at') > idleSince)
|
|
1717
|
+
return null;
|
|
1718
|
+
if (readHold(db, ref.id, text(row, 'owner_key'), this.clock.now()))
|
|
1719
|
+
return null;
|
|
1720
|
+
}
|
|
1721
|
+
const park = readPark(db, ref.id);
|
|
1722
|
+
if (park && park.state !== 'parked' && !this.heldByDead(park, dead))
|
|
1723
|
+
return null;
|
|
1724
|
+
writePark(db, ref.id, {
|
|
1725
|
+
state: 'parking',
|
|
1726
|
+
since: this.clock.now(),
|
|
1727
|
+
parkedAt: park?.parkedAt ?? null,
|
|
1728
|
+
holder: me,
|
|
1729
|
+
attempt,
|
|
1730
|
+
restarts: park?.restarts ?? NOTHING_TO_RESTART,
|
|
1731
|
+
problems: park?.problems ?? [],
|
|
1732
|
+
});
|
|
1733
|
+
return loadLane(db, ref.id);
|
|
1734
|
+
});
|
|
1735
|
+
}
|
|
1736
|
+
/** Gives up attempt `attempt` at parking the lane, because it was used meanwhile; a lane parked before stays parked. */
|
|
1737
|
+
abortPark(ref, attempt) {
|
|
1738
|
+
this.store.write(() => {
|
|
1739
|
+
const db = this.store.db;
|
|
1740
|
+
const park = readPark(db, ref.id);
|
|
1741
|
+
if (park?.state !== 'parking' || park.attempt !== attempt)
|
|
1742
|
+
return;
|
|
1743
|
+
if (park.parkedAt === null)
|
|
1744
|
+
dropPark(db, ref.id);
|
|
1745
|
+
else
|
|
1746
|
+
writePark(db, ref.id, { ...park, state: 'parked', since: this.clock.now(), holder: null, attempt: null });
|
|
1747
|
+
});
|
|
1748
|
+
}
|
|
1749
|
+
/**
|
|
1750
|
+
* Ends attempt `attempt` at parking the lane: frees its device, which goes back to the pool once it is shut down, and
|
|
1751
|
+
* records the lane as parked with what its next call restarts. False when the attempt no longer holds the lane.
|
|
1752
|
+
*/
|
|
1753
|
+
finishPark(ref, attempt, restarts, problems, notes) {
|
|
1754
|
+
const me = this.self();
|
|
1755
|
+
const keeper = this.keeper();
|
|
1756
|
+
const pending = this.unstash(me);
|
|
1757
|
+
let done;
|
|
1758
|
+
try {
|
|
1759
|
+
const listings = this.listOwn(me, pending);
|
|
1760
|
+
done = this.store.write(() => {
|
|
1761
|
+
const db = this.store.db;
|
|
1762
|
+
const others = this.landLeftovers(pending, listings, me, keeper, notes);
|
|
1763
|
+
const row = this.requireLane(ref);
|
|
1764
|
+
this.parkActor(row, 'parking');
|
|
1765
|
+
const park = readPark(db, ref.id);
|
|
1766
|
+
if (park?.state !== 'parking' || park.attempt !== attempt) {
|
|
1767
|
+
return { parked: false, work: this.inWrite({ own: [], others }, me, keeper, notes, 'park:stopped-device') };
|
|
1768
|
+
}
|
|
1769
|
+
const now = this.clock.now();
|
|
1770
|
+
const kept = [...problems];
|
|
1771
|
+
const own = [];
|
|
1772
|
+
const device = db.prepare('SELECT * FROM devices WHERE claim_id = ?').get(ref.id);
|
|
1773
|
+
if (device) {
|
|
1774
|
+
const name = text(device, 'name');
|
|
1775
|
+
const platform = text(device, 'platform');
|
|
1776
|
+
if (this.backends[platform]) {
|
|
1777
|
+
const console = db.prepare('SELECT port FROM ports WHERE claim_id = ? AND service = ?').get(ref.id, EMULATOR_SERVICE);
|
|
1778
|
+
db.prepare('UPDATE devices SET claim_id = NULL WHERE name = ?').run(name);
|
|
1779
|
+
const lease = leaseDevice(db, name, {
|
|
1780
|
+
reservation: this.leaseName(name, 'shutdown', []),
|
|
1781
|
+
work: 'shutdown',
|
|
1782
|
+
holder: me,
|
|
1783
|
+
keeper,
|
|
1784
|
+
consolePort: console ? int(console, 'port') : null,
|
|
1785
|
+
now,
|
|
1786
|
+
});
|
|
1787
|
+
if (lease)
|
|
1788
|
+
own.push(lease);
|
|
1789
|
+
}
|
|
1790
|
+
else {
|
|
1791
|
+
kept.push(`kept ${NOUN[platform]} ${name}: this process has no ${NOUN[platform]} tool to shut it down`);
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
writePark(db, ref.id, {
|
|
1795
|
+
state: 'parked',
|
|
1796
|
+
since: now,
|
|
1797
|
+
parkedAt: now,
|
|
1798
|
+
holder: null,
|
|
1799
|
+
attempt: null,
|
|
1800
|
+
restarts,
|
|
1801
|
+
problems: kept,
|
|
1802
|
+
});
|
|
1803
|
+
return { parked: true, work: this.inWrite({ own, others }, me, keeper, notes, 'park:stopped-device') };
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
catch (error) {
|
|
1807
|
+
this.stash(me, pending);
|
|
1808
|
+
throw error;
|
|
1809
|
+
}
|
|
1810
|
+
if (done.parked)
|
|
1811
|
+
this.step('park:committed');
|
|
1812
|
+
this.afterWrite(done.work, me, keeper, notes, 'park:stopped-device');
|
|
1813
|
+
return done.parked;
|
|
1814
|
+
}
|
|
1815
|
+
/** Adds problems to a parked lane's record, such as a device that would not shut down after the park was recorded. */
|
|
1816
|
+
noteParkProblems(ref, problems) {
|
|
1817
|
+
if (problems.length === 0)
|
|
1818
|
+
return;
|
|
1819
|
+
this.store.write(() => {
|
|
1820
|
+
const db = this.store.db;
|
|
1821
|
+
const park = readPark(db, ref.id);
|
|
1822
|
+
if (park?.state !== 'parked')
|
|
1823
|
+
return;
|
|
1824
|
+
writePark(db, ref.id, { ...park, problems: [...park.problems, ...problems] });
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
/**
|
|
1828
|
+
* Takes a parked lane for waking as attempt `attempt` of this process: `taken` with the park it ends, `busy` while
|
|
1829
|
+
* another park or wake runs, `awake` when there is nothing to wake. `dead` is the holder of an earlier park or wake
|
|
1830
|
+
* that was proven gone before this call.
|
|
1831
|
+
*/
|
|
1832
|
+
beginWake(ref, attempt, dead) {
|
|
1833
|
+
const me = this.self();
|
|
1834
|
+
return this.store.write(() => {
|
|
1835
|
+
const row = this.requireLane(ref);
|
|
1836
|
+
this.parkActor(row, 'waking');
|
|
1837
|
+
const db = this.store.db;
|
|
1838
|
+
const park = readPark(db, ref.id);
|
|
1839
|
+
if (!park)
|
|
1840
|
+
return { kind: 'awake' };
|
|
1841
|
+
if (park.state !== 'parked' && !this.heldByDead(park, dead))
|
|
1842
|
+
return { kind: 'busy', park, worktree: text(row, 'worktree') };
|
|
1843
|
+
writePark(db, ref.id, { ...park, state: 'waking', since: this.clock.now(), holder: me, attempt });
|
|
1844
|
+
return { kind: 'taken', park };
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
/** Records the lane as awake once attempt `attempt` has brought everything back. */
|
|
1848
|
+
finishWake(ref, attempt) {
|
|
1849
|
+
this.store.write(() => {
|
|
1850
|
+
const db = this.store.db;
|
|
1851
|
+
if (readPark(db, ref.id)?.attempt === attempt)
|
|
1852
|
+
dropPark(db, ref.id);
|
|
1853
|
+
});
|
|
1854
|
+
}
|
|
1855
|
+
/**
|
|
1856
|
+
* Records the lane as parked, with `problems`, after attempt `attempt` at parking or waking it stopped partway, so the
|
|
1857
|
+
* next sweep or call finishes the job; whatever the attempt left running stays recorded with the lane.
|
|
1858
|
+
*/
|
|
1859
|
+
giveUp(ref, attempt, problems) {
|
|
1860
|
+
this.store.write(() => {
|
|
1861
|
+
const db = this.store.db;
|
|
1862
|
+
const park = readPark(db, ref.id);
|
|
1863
|
+
if (park?.attempt !== attempt)
|
|
1864
|
+
return;
|
|
1865
|
+
const now = this.clock.now();
|
|
1866
|
+
writePark(db, ref.id, {
|
|
1867
|
+
...park,
|
|
1868
|
+
state: 'parked',
|
|
1869
|
+
since: now,
|
|
1870
|
+
parkedAt: park.parkedAt ?? now,
|
|
1871
|
+
holder: null,
|
|
1872
|
+
attempt: null,
|
|
1873
|
+
problems,
|
|
1874
|
+
});
|
|
1875
|
+
});
|
|
1876
|
+
}
|
|
907
1877
|
touchOwn(owner, worktree, action) {
|
|
908
1878
|
this.caller(owner, worktree);
|
|
909
|
-
const lane = this.
|
|
1879
|
+
const lane = this.takeBack(owner, worktree);
|
|
910
1880
|
if (!lane)
|
|
911
1881
|
throw new NotClaimed(`${describeOwner(owner)} holds no lane in ${worktree}`, `${CLI} claim (run in ${worktree})`);
|
|
912
1882
|
return this.touch({ id: lane.id, token: lane.token }, action);
|
|
@@ -914,48 +1884,86 @@ export class Registry {
|
|
|
914
1884
|
recordProcess(ref, proc) {
|
|
915
1885
|
this.store.write(() => {
|
|
916
1886
|
this.requireLane(ref);
|
|
917
|
-
this.store.db
|
|
918
|
-
|
|
919
|
-
|
|
1887
|
+
const db = this.store.db;
|
|
1888
|
+
db.prepare('INSERT OR REPLACE INTO processes (claim_id, service, pid, start, cwd) VALUES (?, ?, ?, ?, ?)').run(ref.id, proc.service, proc.pid, proc.start, proc.cwd);
|
|
1889
|
+
db.prepare('DELETE FROM process_alive WHERE claim_id = ? AND service = ? AND NOT (pid = ? AND start = ?)').run(ref.id, proc.service, proc.pid, proc.start);
|
|
1890
|
+
});
|
|
1891
|
+
}
|
|
1892
|
+
/**
|
|
1893
|
+
* Records that a check at `at` found the lane's recorded process for `proc.service` running with `proc`'s pid and
|
|
1894
|
+
* start time. Only a later moment replaces an earlier one, and nothing is written unless the lane still records that
|
|
1895
|
+
* very process; returns whether the stored moment changed.
|
|
1896
|
+
*/
|
|
1897
|
+
recordLeaderAlive(laneId, proc, at) {
|
|
1898
|
+
return this.recordLeadersAlive([{ lane: laneId, process: proc }], at).length > 0;
|
|
1899
|
+
}
|
|
1900
|
+
/** `recordLeaderAlive` for each of `seen`, in one write; returns those whose stored moment changed. */
|
|
1901
|
+
recordLeadersAlive(seen, at) {
|
|
1902
|
+
if (seen.length === 0)
|
|
1903
|
+
return [];
|
|
1904
|
+
return this.store.write(() => {
|
|
1905
|
+
const insert = this.store.db.prepare(`INSERT INTO process_alive (claim_id, service, pid, start, alive_at)
|
|
1906
|
+
SELECT claim_id, service, pid, start, ? FROM processes WHERE claim_id = ? AND service = ? AND pid = ? AND start = ?
|
|
1907
|
+
ON CONFLICT (claim_id, service) DO UPDATE SET pid = excluded.pid, start = excluded.start, alive_at = excluded.alive_at
|
|
1908
|
+
WHERE NOT (pid = excluded.pid AND start = excluded.start AND alive_at >= excluded.alive_at)`);
|
|
1909
|
+
return seen.filter(s => Number(insert.run(at, s.lane, s.process.service, s.process.pid, s.process.start).changes) > 0);
|
|
1910
|
+
});
|
|
1911
|
+
}
|
|
1912
|
+
/** Every service process a lane records, with the moment a check last found it running. */
|
|
1913
|
+
serviceProcesses() {
|
|
1914
|
+
const db = this.store.db;
|
|
1915
|
+
return db
|
|
1916
|
+
.prepare('SELECT DISTINCT claim_id FROM processes ORDER BY claim_id')
|
|
1917
|
+
.all()
|
|
1918
|
+
.flatMap(row => {
|
|
1919
|
+
const lane = text(row, 'claim_id');
|
|
1920
|
+
return processesOf(db, lane).map(process => ({ lane, process }));
|
|
920
1921
|
});
|
|
921
1922
|
}
|
|
922
1923
|
forgetProcess(ref, service) {
|
|
923
1924
|
this.store.write(() => {
|
|
924
1925
|
this.requireLane(ref);
|
|
925
1926
|
this.store.db.prepare('DELETE FROM processes WHERE claim_id = ? AND service = ?').run(ref.id, service);
|
|
1927
|
+
this.store.db.prepare('DELETE FROM process_alive WHERE claim_id = ? AND service = ?').run(ref.id, service);
|
|
926
1928
|
});
|
|
927
1929
|
}
|
|
928
1930
|
/** Records what the resource's teardown needs before its create runs; `kept` when this lane already has it. */
|
|
929
1931
|
startResource(ref, spec) {
|
|
1932
|
+
const me = this.self();
|
|
930
1933
|
const holders = holdersBeforeLock(this.store.db, this.processes);
|
|
931
1934
|
return this.store.write(() => {
|
|
932
1935
|
const row = this.requireLane(ref);
|
|
933
|
-
return startResource(this.store.db, { id: ref.id, worktree: text(row, 'worktree') }, spec,
|
|
1936
|
+
return startResource(this.store.db, { id: ref.id, worktree: text(row, 'worktree') }, spec, me, holders, this.clock.now());
|
|
934
1937
|
});
|
|
935
1938
|
}
|
|
936
1939
|
finishResource(lane, name, exports) {
|
|
1940
|
+
const me = this.self();
|
|
937
1941
|
this.store.write(() => {
|
|
938
|
-
finishResource(this.store.db, lane, name, exports,
|
|
1942
|
+
finishResource(this.store.db, lane, name, exports, me);
|
|
939
1943
|
});
|
|
940
1944
|
}
|
|
941
1945
|
failResource(lane, name, problem) {
|
|
1946
|
+
const me = this.self();
|
|
942
1947
|
this.store.write(() => {
|
|
943
|
-
failResource(this.store.db, lane, name, problem,
|
|
1948
|
+
failResource(this.store.db, lane, name, problem, me);
|
|
944
1949
|
});
|
|
945
1950
|
}
|
|
946
1951
|
/** Leases the resources of ended lanes to this process for teardown, last created first. */
|
|
947
1952
|
takeTeardowns(choice = {}) {
|
|
1953
|
+
const me = this.self();
|
|
948
1954
|
const holders = holdersBeforeLock(this.store.db, this.processes);
|
|
949
|
-
return this.store.write(() => takeTeardowns(this.store.db,
|
|
1955
|
+
return this.store.write(() => takeTeardowns(this.store.db, me, holders, this.clock.now(), choice));
|
|
950
1956
|
}
|
|
951
1957
|
finishTeardown(lane, name) {
|
|
1958
|
+
const me = this.self();
|
|
952
1959
|
this.store.write(() => {
|
|
953
|
-
endTeardown(this.store.db, lane, name,
|
|
1960
|
+
endTeardown(this.store.db, lane, name, me);
|
|
954
1961
|
});
|
|
955
1962
|
}
|
|
956
1963
|
failTeardown(lane, name, problem) {
|
|
1964
|
+
const me = this.self();
|
|
957
1965
|
this.store.write(() => {
|
|
958
|
-
failTeardown(this.store.db, lane, name, problem,
|
|
1966
|
+
failTeardown(this.store.db, lane, name, problem, me, this.clock.now());
|
|
959
1967
|
});
|
|
960
1968
|
}
|
|
961
1969
|
/** The browsers of lanes that have ended, however they ended: their processes may still run. */
|
|
@@ -982,6 +1990,23 @@ export class Registry {
|
|
|
982
1990
|
pendingTeardowns() {
|
|
983
1991
|
return pendingTeardowns(this.store.db);
|
|
984
1992
|
}
|
|
1993
|
+
/**
|
|
1994
|
+
* The lanes whose files are kept: every lane the registry records, each ended one whose browser or resources still
|
|
1995
|
+
* wait for their stop, and each that ended at or after `endedSince`.
|
|
1996
|
+
*/
|
|
1997
|
+
keptLaneIds(endedSince) {
|
|
1998
|
+
const db = this.store.db;
|
|
1999
|
+
const ids = (sql, ...args) => db
|
|
2000
|
+
.prepare(sql)
|
|
2001
|
+
.all(...args)
|
|
2002
|
+
.map(row => text(row, 'id'));
|
|
2003
|
+
return new Set([
|
|
2004
|
+
...ids('SELECT id FROM claims'),
|
|
2005
|
+
...ids('SELECT claim_id AS id FROM browsers'),
|
|
2006
|
+
...ids('SELECT DISTINCT claim_id AS id FROM resources'),
|
|
2007
|
+
...ids('SELECT DISTINCT claim_id AS id FROM history WHERE ended_at >= ?', endedSince),
|
|
2008
|
+
]);
|
|
2009
|
+
}
|
|
985
2010
|
prunable() {
|
|
986
2011
|
return listPrunable(this.store, this.dir, this.backends);
|
|
987
2012
|
}
|