@botbuddy/cli 1.60.5 → 1.60.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/daemon/bin/darwin-arm64/botbuddy-daemon +0 -0
- package/daemon/bin/linux-x64/botbuddy-daemon +0 -0
- package/daemon-release/darwin-arm64.json +4 -4
- package/daemon-release/linux-x64.json +4 -4
- package/daemon-release/manifest.json +5 -5
- package/package.json +4 -2
- package/src/commands.mjs +28 -0
- package/src/pw/args.mjs +17 -0
- package/src/pw/coordinator.mjs +5 -1
- package/src/pw/run.mjs +73 -9
- package/src/pw/staging-auth.mjs +393 -0
- package/src/staging-pool/binding.mjs +122 -0
- package/src/staging-pool/checkouts.mjs +120 -0
- package/src/staging-pool/cli.mjs +548 -0
- package/src/staging-pool/config.mjs +110 -0
- package/src/staging-pool/errors.mjs +434 -0
- package/src/staging-pool/health.mjs +84 -0
- package/src/staging-pool/help.mjs +125 -0
- package/src/staging-pool/job.mjs +7 -0
- package/src/staging-pool/launchd.mjs +92 -0
- package/src/staging-pool/password-source.mjs +65 -0
- package/src/staging-pool/policy.mjs +186 -0
- package/src/staging-pool/pool.mjs +666 -0
- package/src/staging-pool/signin-gotrue-totp.mjs +198 -0
- package/src/staging-pool/state-file.mjs +166 -0
- package/src/staging-pool/totp-botbuddy.mjs +106 -0
|
@@ -0,0 +1,666 @@
|
|
|
1
|
+
// BOT-2084 (lifted from supplyspark-ent ENT-5120 pool.mjs): a small pool of
|
|
2
|
+
// single-use, signed-in session files.
|
|
3
|
+
//
|
|
4
|
+
// Node-stdlib-only and self-contained; every side effect (sign-in, clock,
|
|
5
|
+
// logging, liveness of a lock holder) is injected. The persona allowlist is the
|
|
6
|
+
// target's (policy.mjs, from the repo binding).
|
|
7
|
+
//
|
|
8
|
+
// Layout under poolDir (dir 0700, files 0600):
|
|
9
|
+
// ready/<slot>/{state.json,meta.json} warm sessions; `take` only ever looks here
|
|
10
|
+
// work/<slot>/ held by the job while it mints or refreshes
|
|
11
|
+
// taken/<slot>/ handed to one agent; never returned
|
|
12
|
+
// tick.lock, last-tick.json, health.json (backoff + below-target streaks)
|
|
13
|
+
//
|
|
14
|
+
// Every ownership change is one rename(2) of a slot directory, which is atomic on a
|
|
15
|
+
// single filesystem: exactly one of two racing takers — or a taker and the job's
|
|
16
|
+
// refresh — wins, so no two browsers ever hold the same refresh token.
|
|
17
|
+
|
|
18
|
+
import { randomBytes } from "node:crypto";
|
|
19
|
+
import * as nodeFs from "node:fs/promises";
|
|
20
|
+
import { join } from "node:path";
|
|
21
|
+
|
|
22
|
+
import { nextMintBackoffMs } from "./health.mjs";
|
|
23
|
+
import { assertPersonaAllowed } from "./policy.mjs";
|
|
24
|
+
import {
|
|
25
|
+
assertStorageStateClean,
|
|
26
|
+
buildStorageState,
|
|
27
|
+
describeStoredSession,
|
|
28
|
+
} from "./state-file.mjs";
|
|
29
|
+
|
|
30
|
+
export class PoolEmptyError extends Error {
|
|
31
|
+
constructor(persona, intervalHint) {
|
|
32
|
+
super(
|
|
33
|
+
`No ready ${persona} session in the pool. The human-owned pool job refills on its next tick${intervalHint}; ` +
|
|
34
|
+
"report this blocker and its remedy once, then stop; ask the human to check the job. " +
|
|
35
|
+
"Agents never sign in to staging themselves.",
|
|
36
|
+
);
|
|
37
|
+
this.name = "PoolEmptyError";
|
|
38
|
+
this.code = "pool_empty";
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const DIR_MODE = 0o700;
|
|
43
|
+
const FILE_MODE = 0o600;
|
|
44
|
+
// Lock liveness is a heartbeat, not a PID (Codex R13): a running tick touches tick.lock every
|
|
45
|
+
// lockHeartbeatMs; a lock whose mtime is older than this has no live holder, whatever its
|
|
46
|
+
// recorded PID now belongs to (PID reuse after a crash or reboot) and whether or not its JSON
|
|
47
|
+
// was ever written (a writer that died between create and write).
|
|
48
|
+
const STALE_LOCK_HEARTBEAT_MS = 5 * 60_000;
|
|
49
|
+
|
|
50
|
+
const personaKey = (persona) => persona.replace(/[^A-Za-z0-9]+/g, "_");
|
|
51
|
+
|
|
52
|
+
function defaultIsPidAlive(pid) {
|
|
53
|
+
try {
|
|
54
|
+
process.kill(pid, 0);
|
|
55
|
+
return true;
|
|
56
|
+
} catch (error) {
|
|
57
|
+
return error?.code === "EPERM";
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function createSessionPool({
|
|
62
|
+
poolDir,
|
|
63
|
+
target,
|
|
64
|
+
signIn,
|
|
65
|
+
fs = nodeFs,
|
|
66
|
+
now = () => Date.now(),
|
|
67
|
+
logger = () => {},
|
|
68
|
+
randomId = () => randomBytes(6).toString("hex"),
|
|
69
|
+
isPidAlive = defaultIsPidAlive,
|
|
70
|
+
pid = process.pid,
|
|
71
|
+
refreshAfterMs = 30 * 60_000,
|
|
72
|
+
minAccessRemainingMs = 10 * 60_000,
|
|
73
|
+
takenTtlMs = 60 * 60_000,
|
|
74
|
+
intervalMs = 300_000,
|
|
75
|
+
lockHeartbeatMs = 30_000,
|
|
76
|
+
tickIntervalHint = "",
|
|
77
|
+
} = {}) {
|
|
78
|
+
if (!poolDir) throw new Error("createSessionPool needs poolDir.");
|
|
79
|
+
if (!target) throw new Error("createSessionPool needs a resolved mint target.");
|
|
80
|
+
if (!signIn) throw new Error("createSessionPool needs a signIn recipe.");
|
|
81
|
+
|
|
82
|
+
const dirs = {
|
|
83
|
+
ready: join(poolDir, "ready"),
|
|
84
|
+
work: join(poolDir, "work"),
|
|
85
|
+
taken: join(poolDir, "taken"),
|
|
86
|
+
};
|
|
87
|
+
const lockPath = join(poolDir, "tick.lock");
|
|
88
|
+
const lastTickPath = join(poolDir, "last-tick.json");
|
|
89
|
+
const healthPath = join(poolDir, "health.json");
|
|
90
|
+
|
|
91
|
+
async function readHealth() {
|
|
92
|
+
try {
|
|
93
|
+
const health = await readJson(healthPath);
|
|
94
|
+
return { lastSuccessfulTickAt: health.lastSuccessfulTickAt ?? null, personas: health.personas ?? {} };
|
|
95
|
+
} catch {
|
|
96
|
+
return { lastSuccessfulTickAt: null, personas: {} };
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function ensureDir(path) {
|
|
101
|
+
await fs.mkdir(path, { recursive: true, mode: DIR_MODE });
|
|
102
|
+
await fs.chmod(path, DIR_MODE);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// A pool dir belongs to exactly one target (Codex R5): every operation first checks the
|
|
106
|
+
// binding, so a mistyped --pool-dir can never take, refresh or prune another target's slots.
|
|
107
|
+
const bindingPath = join(poolDir, "pool-target.json");
|
|
108
|
+
async function ensureLayout() {
|
|
109
|
+
await ensureDir(poolDir);
|
|
110
|
+
const binding = {
|
|
111
|
+
target: target.name,
|
|
112
|
+
appOrigin: target.appOrigin,
|
|
113
|
+
supabaseUrl: target.supabaseUrl,
|
|
114
|
+
storageKey: target.storageKey,
|
|
115
|
+
};
|
|
116
|
+
try {
|
|
117
|
+
await writeFileExclusive(bindingPath, `${JSON.stringify(binding)}\n`);
|
|
118
|
+
} catch (error) {
|
|
119
|
+
if (error?.code !== "EEXIST") throw error;
|
|
120
|
+
let existing = null;
|
|
121
|
+
try {
|
|
122
|
+
existing = await readJson(bindingPath);
|
|
123
|
+
} catch {
|
|
124
|
+
existing = null;
|
|
125
|
+
}
|
|
126
|
+
if (Object.keys(binding).some((key) => existing?.[key] !== binding[key])) {
|
|
127
|
+
throw Object.assign(
|
|
128
|
+
new Error(
|
|
129
|
+
`Pool ${poolDir} belongs to ${existing?.target ?? "an unknown target"} (${existing?.appOrigin ?? "?"}, ${existing?.supabaseUrl ?? "?"}), not ${binding.target} (${binding.appOrigin}, ${binding.supabaseUrl}). Use that pool's own target or another --pool-dir.`,
|
|
130
|
+
),
|
|
131
|
+
{ code: "pool_target_mismatch" },
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
for (const dir of Object.values(dirs)) await ensureDir(dir);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function list(dir) {
|
|
139
|
+
try {
|
|
140
|
+
return (await fs.readdir(dir)).filter((name) => !name.startsWith("."));
|
|
141
|
+
} catch (error) {
|
|
142
|
+
if (error?.code === "ENOENT") return [];
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// true = this caller now owns `to`; false = someone else moved it first.
|
|
148
|
+
async function claim(from, to) {
|
|
149
|
+
try {
|
|
150
|
+
await fs.rename(from, to);
|
|
151
|
+
return true;
|
|
152
|
+
} catch (error) {
|
|
153
|
+
if (error?.code === "ENOENT") return false;
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async function writeFileExclusive(path, data) {
|
|
159
|
+
await fs.writeFile(path, data, { mode: FILE_MODE, flag: "wx" });
|
|
160
|
+
await fs.chmod(path, FILE_MODE);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async function replaceFile(path, data) {
|
|
164
|
+
const next = `${path}.next`;
|
|
165
|
+
await fs.rm(next, { force: true });
|
|
166
|
+
await writeFileExclusive(next, data);
|
|
167
|
+
await fs.rename(next, path);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async function readJson(path) {
|
|
171
|
+
return JSON.parse(await fs.readFile(path, "utf8"));
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async function readSlot(dir) {
|
|
175
|
+
const state = await readJson(join(dir, "state.json"));
|
|
176
|
+
const storedSession = assertStorageStateClean(state, target);
|
|
177
|
+
const meta = await readJson(join(dir, "meta.json"));
|
|
178
|
+
return { storedSession, meta };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function buildMeta({ persona, slotId, storedSession, mintedAt, refreshCount }) {
|
|
182
|
+
const session = describeStoredSession(storedSession, { nowMs: now() });
|
|
183
|
+
return {
|
|
184
|
+
slotId,
|
|
185
|
+
persona,
|
|
186
|
+
target: target.name,
|
|
187
|
+
appOrigin: target.appOrigin,
|
|
188
|
+
mintedAt,
|
|
189
|
+
verifiedAt: new Date(now()).toISOString(),
|
|
190
|
+
refreshCount,
|
|
191
|
+
aal: session.aal,
|
|
192
|
+
amr: session.amr,
|
|
193
|
+
accessExpiresAt: session.accessExpiresAt,
|
|
194
|
+
sessionRef: session.sessionRef,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
async function writeSlotFiles(dir, { storedSession, meta, replace }) {
|
|
199
|
+
const state = buildStorageState({ target, storedSession });
|
|
200
|
+
assertStorageStateClean(state, target);
|
|
201
|
+
const write = replace ? replaceFile : writeFileExclusive;
|
|
202
|
+
await write(join(dir, "state.json"), `${JSON.stringify(state)}\n`);
|
|
203
|
+
await write(join(dir, "meta.json"), `${JSON.stringify(meta, null, 2)}\n`);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function removeSlot(dir) {
|
|
207
|
+
await fs.rm(dir, { recursive: true, force: true });
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// Tick lock (Codex R6): each holder writes a unique token. A stale lock is reclaimed by
|
|
211
|
+
// renaming it to a private tombstone and checking the tombstone is the holder judged stale;
|
|
212
|
+
// if a fresh holder replaced it in between, the tombstone is linked back and this tick skips.
|
|
213
|
+
// Release unlinks only a lock that still carries this tick's own token.
|
|
214
|
+
let heldToken = null;
|
|
215
|
+
|
|
216
|
+
async function readLock(path) {
|
|
217
|
+
try {
|
|
218
|
+
return await readJson(path);
|
|
219
|
+
} catch {
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// File age uses the real wall clock: mtime is set by the filesystem, not the injected clock.
|
|
225
|
+
async function isAbandonedLockFile(path) {
|
|
226
|
+
try {
|
|
227
|
+
const { mtimeMs } = await fs.stat(path);
|
|
228
|
+
return Date.now() - mtimeMs >= STALE_LOCK_HEARTBEAT_MS;
|
|
229
|
+
} catch {
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function acquireTickLock() {
|
|
235
|
+
await ensureLayout();
|
|
236
|
+
const token = `${pid}-${randomId()}`;
|
|
237
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
238
|
+
try {
|
|
239
|
+
await writeFileExclusive(lockPath, JSON.stringify({ pid, startedAt: now(), token }));
|
|
240
|
+
heldToken = token;
|
|
241
|
+
return true;
|
|
242
|
+
} catch (error) {
|
|
243
|
+
if (error?.code !== "EEXIST") throw error;
|
|
244
|
+
}
|
|
245
|
+
// A live holder keeps the lock however long it runs (Codex R10) because it heartbeats.
|
|
246
|
+
// Reclaim when the holder's process is gone, or when the heartbeat is stale even though
|
|
247
|
+
// the PID looks alive (reused, R13), or when the file was never written (R12).
|
|
248
|
+
const holder = await readLock(lockPath);
|
|
249
|
+
const heartbeatStale = await isAbandonedLockFile(lockPath);
|
|
250
|
+
const holderGone = holder ? !isPidAlive(Number(holder.pid)) || heartbeatStale : heartbeatStale;
|
|
251
|
+
if (!holderGone) return false;
|
|
252
|
+
const tomb = `${lockPath}.reclaim-${token}`;
|
|
253
|
+
try {
|
|
254
|
+
await fs.rename(lockPath, tomb);
|
|
255
|
+
} catch (error) {
|
|
256
|
+
if (error?.code === "ENOENT") continue; // released or reclaimed meanwhile: retry
|
|
257
|
+
throw error;
|
|
258
|
+
}
|
|
259
|
+
const moved = await readLock(tomb);
|
|
260
|
+
// The moved file must be the holder judged gone: same token, or (unreadable) still stale.
|
|
261
|
+
const sameHolder = holder ? moved && moved.token === holder.token && moved.pid === holder.pid : !moved && (await isAbandonedLockFile(tomb));
|
|
262
|
+
if (!sameHolder) {
|
|
263
|
+
// We moved a lock someone took after our read: give it back and stand down.
|
|
264
|
+
try {
|
|
265
|
+
await fs.link(tomb, lockPath);
|
|
266
|
+
} catch {
|
|
267
|
+
// A third tick holds the path now; the moved holder's release will find no lock.
|
|
268
|
+
}
|
|
269
|
+
await fs.rm(tomb, { force: true });
|
|
270
|
+
return false;
|
|
271
|
+
}
|
|
272
|
+
await fs.rm(tomb, { force: true });
|
|
273
|
+
logger({ event: "tick_lock_reclaimed", holderPid: holder?.pid ?? null });
|
|
274
|
+
}
|
|
275
|
+
return false;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
async function releaseTickLock() {
|
|
279
|
+
const holder = await readLock(lockPath);
|
|
280
|
+
if (holder?.token && holder.token === heldToken) await fs.rm(lockPath, { force: true });
|
|
281
|
+
heldToken = null;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
async function withTickLock(fn) {
|
|
285
|
+
if (!(await acquireTickLock())) return { skipped: "tick_in_progress" };
|
|
286
|
+
const token = heldToken;
|
|
287
|
+
const heartbeat = setInterval(() => {
|
|
288
|
+
readLock(lockPath)
|
|
289
|
+
.then((holder) => (holder?.token === token ? fs.utimes(lockPath, new Date(), new Date()) : undefined))
|
|
290
|
+
.catch(() => {});
|
|
291
|
+
}, lockHeartbeatMs);
|
|
292
|
+
try {
|
|
293
|
+
return await fn();
|
|
294
|
+
} finally {
|
|
295
|
+
clearInterval(heartbeat);
|
|
296
|
+
await releaseTickLock();
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
async function slotPersona(dir) {
|
|
301
|
+
try {
|
|
302
|
+
return (await readJson(join(dir, "meta.json"))).persona ?? null;
|
|
303
|
+
} catch {
|
|
304
|
+
return null;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
async function countReady(personas) {
|
|
309
|
+
const counts = Object.fromEntries(personas.map((persona) => [persona, 0]));
|
|
310
|
+
for (const slotId of await list(dirs.ready)) {
|
|
311
|
+
const persona = await slotPersona(join(dirs.ready, slotId));
|
|
312
|
+
if (persona && Object.hasOwn(counts, persona)) counts[persona] += 1;
|
|
313
|
+
}
|
|
314
|
+
return counts;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Caller holds the tick lock.
|
|
318
|
+
async function mintIntoReady(persona) {
|
|
319
|
+
const mintedAt = new Date(now()).toISOString();
|
|
320
|
+
const slotId = `${personaKey(persona)}~${now()}~${randomId()}`;
|
|
321
|
+
const workDir = join(dirs.work, slotId);
|
|
322
|
+
const storedSession = await signIn.mint({ target, persona });
|
|
323
|
+
await ensureDir(workDir);
|
|
324
|
+
try {
|
|
325
|
+
const meta = buildMeta({ persona, slotId, storedSession, mintedAt, refreshCount: 0 });
|
|
326
|
+
await writeSlotFiles(workDir, { storedSession, meta, replace: false });
|
|
327
|
+
await fs.rename(workDir, join(dirs.ready, slotId));
|
|
328
|
+
return { slotId, meta };
|
|
329
|
+
} catch (error) {
|
|
330
|
+
await removeSlot(workDir);
|
|
331
|
+
throw error;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function refreshDue(meta) {
|
|
336
|
+
const verifiedAt = Date.parse(meta.verifiedAt);
|
|
337
|
+
const accessExpiresAt = Date.parse(meta.accessExpiresAt);
|
|
338
|
+
return (
|
|
339
|
+
!Number.isFinite(verifiedAt) ||
|
|
340
|
+
now() - verifiedAt >= refreshAfterMs ||
|
|
341
|
+
!Number.isFinite(accessExpiresAt) ||
|
|
342
|
+
accessExpiresAt - now() <= minAccessRemainingMs
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Caller holds the tick lock. Returns "kept" | "refreshed" | "discarded" | "gone".
|
|
347
|
+
async function refreshUnclaimed(slotId) {
|
|
348
|
+
const readyDir = join(dirs.ready, slotId);
|
|
349
|
+
let meta;
|
|
350
|
+
try {
|
|
351
|
+
meta = await readJson(join(readyDir, "meta.json"));
|
|
352
|
+
} catch (error) {
|
|
353
|
+
if (error?.code === "ENOENT") return "gone";
|
|
354
|
+
meta = null;
|
|
355
|
+
}
|
|
356
|
+
if (meta && !refreshDue(meta)) return "kept";
|
|
357
|
+
const workDir = join(dirs.work, slotId);
|
|
358
|
+
if (!(await claim(readyDir, workDir))) return "gone"; // taken meanwhile
|
|
359
|
+
let slot;
|
|
360
|
+
try {
|
|
361
|
+
slot = await readSlot(workDir);
|
|
362
|
+
} catch (error) {
|
|
363
|
+
logger({ event: "refresh_discard", slotId, reason: error?.code ?? "unreadable" });
|
|
364
|
+
await removeSlot(workDir);
|
|
365
|
+
return "discarded";
|
|
366
|
+
}
|
|
367
|
+
const result = await signIn.refresh({ target, storedSession: slot.storedSession, persona: slot.meta.persona });
|
|
368
|
+
if (result.ok) {
|
|
369
|
+
const nextMeta = buildMeta({
|
|
370
|
+
persona: slot.meta.persona,
|
|
371
|
+
slotId,
|
|
372
|
+
storedSession: result.storedSession,
|
|
373
|
+
mintedAt: slot.meta.mintedAt,
|
|
374
|
+
refreshCount: (slot.meta.refreshCount ?? 0) + 1,
|
|
375
|
+
});
|
|
376
|
+
await writeSlotFiles(workDir, { storedSession: result.storedSession, meta: nextMeta, replace: true });
|
|
377
|
+
await fs.rename(workDir, readyDir);
|
|
378
|
+
logger({ event: "refresh_ok", slotId, persona: slot.meta.persona });
|
|
379
|
+
return "refreshed";
|
|
380
|
+
}
|
|
381
|
+
if (result.transient) {
|
|
382
|
+
await fs.rename(workDir, readyDir);
|
|
383
|
+
logger({ event: "refresh_inconclusive", slotId, reason: result.reason });
|
|
384
|
+
return "kept";
|
|
385
|
+
}
|
|
386
|
+
await removeSlot(workDir);
|
|
387
|
+
logger({ event: "refresh_discard", slotId, persona: slot.meta.persona, reason: result.reason });
|
|
388
|
+
return "discarded";
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
async function tick({ personas }) {
|
|
392
|
+
const plan = Object.entries(personas ?? {});
|
|
393
|
+
for (const [persona, count] of plan) {
|
|
394
|
+
assertPersonaAllowed(target, persona);
|
|
395
|
+
if (!Number.isInteger(count) || count < 0 || count > 4) {
|
|
396
|
+
throw new Error(`Pool size for ${persona} must be an integer 0-4.`);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
return withTickLock(async () => {
|
|
400
|
+
const summary = {
|
|
401
|
+
at: new Date(now()).toISOString(),
|
|
402
|
+
target: target.name,
|
|
403
|
+
signIns: 0,
|
|
404
|
+
refreshed: 0,
|
|
405
|
+
discarded: 0,
|
|
406
|
+
purgedTaken: 0,
|
|
407
|
+
recoveredWork: 0,
|
|
408
|
+
mintFailures: 0,
|
|
409
|
+
prunedSurplus: 0,
|
|
410
|
+
ready: {},
|
|
411
|
+
};
|
|
412
|
+
|
|
413
|
+
// Anything in work/ belongs to a crashed tick or mint: its token state is
|
|
414
|
+
// unknown, so it is discarded, never promoted.
|
|
415
|
+
for (const slotId of await list(dirs.work)) {
|
|
416
|
+
await removeSlot(join(dirs.work, slotId));
|
|
417
|
+
summary.recoveredWork += 1;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// taken/ is the agents' directory: a take may be mid-validation (claimed, takenAt not
|
|
421
|
+
// yet written). Age a slot by takenAt, else by its claim marker, else by its last
|
|
422
|
+
// verification, and purge only past the TTL — never because takenAt is still missing.
|
|
423
|
+
for (const slotId of await list(dirs.taken)) {
|
|
424
|
+
const dir = join(dirs.taken, slotId);
|
|
425
|
+
let meta = null;
|
|
426
|
+
try {
|
|
427
|
+
meta = await readJson(join(dir, "meta.json"));
|
|
428
|
+
} catch {
|
|
429
|
+
meta = null;
|
|
430
|
+
}
|
|
431
|
+
let claimedAt = Number.NaN;
|
|
432
|
+
try {
|
|
433
|
+
claimedAt = Date.parse((await readJson(join(dir, "claim.json"))).claimedAt);
|
|
434
|
+
} catch {
|
|
435
|
+
claimedAt = Number.NaN;
|
|
436
|
+
}
|
|
437
|
+
const anchor = [Date.parse(meta?.takenAt ?? ""), claimedAt, Date.parse(meta?.verifiedAt ?? "")].find((value) =>
|
|
438
|
+
Number.isFinite(value),
|
|
439
|
+
);
|
|
440
|
+
if (anchor === undefined || now() - anchor >= takenTtlMs) {
|
|
441
|
+
await removeSlot(dir);
|
|
442
|
+
summary.purgedTaken += 1;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// A persona removed from the config, or lowered (0 disables it), keeps no takeable
|
|
447
|
+
// surplus: discard the least recently verified extras before spending refreshes.
|
|
448
|
+
const wanted = new Map(plan);
|
|
449
|
+
const byPersona = new Map();
|
|
450
|
+
for (const slotId of await list(dirs.ready)) {
|
|
451
|
+
let meta = null;
|
|
452
|
+
try {
|
|
453
|
+
meta = await readJson(join(dirs.ready, slotId, "meta.json"));
|
|
454
|
+
} catch {
|
|
455
|
+
meta = null;
|
|
456
|
+
}
|
|
457
|
+
const persona = meta?.persona ?? null;
|
|
458
|
+
if (!byPersona.has(persona)) byPersona.set(persona, []);
|
|
459
|
+
byPersona.get(persona).push({ slotId, verifiedAt: Date.parse(meta?.verifiedAt ?? "") || 0 });
|
|
460
|
+
}
|
|
461
|
+
for (const [persona, slots] of byPersona) {
|
|
462
|
+
const keep = persona && wanted.has(persona) ? wanted.get(persona) : 0;
|
|
463
|
+
slots.sort((a, b) => b.verifiedAt - a.verifiedAt);
|
|
464
|
+
for (const { slotId } of slots.slice(keep)) {
|
|
465
|
+
const workDir = join(dirs.work, slotId);
|
|
466
|
+
if (!(await claim(join(dirs.ready, slotId), workDir))) continue; // taken meanwhile
|
|
467
|
+
await removeSlot(workDir);
|
|
468
|
+
summary.prunedSurplus += 1;
|
|
469
|
+
logger({ event: "surplus_discard", slotId, persona, configured: keep });
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
for (const slotId of await list(dirs.ready)) {
|
|
474
|
+
const outcome = await refreshUnclaimed(slotId);
|
|
475
|
+
if (outcome === "refreshed") summary.refreshed += 1;
|
|
476
|
+
if (outcome === "discarded") summary.discarded += 1;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const counts = await countReady(plan.map(([persona]) => persona));
|
|
480
|
+
const health = await readHealth();
|
|
481
|
+
summary.backoffSkips = 0;
|
|
482
|
+
for (const [persona, want] of plan) {
|
|
483
|
+
const entry = (health.personas[persona] ??= {
|
|
484
|
+
belowTargetTicks: 0,
|
|
485
|
+
consecutiveMintFailures: 0,
|
|
486
|
+
nextMintAttemptAt: null,
|
|
487
|
+
lastMintError: null,
|
|
488
|
+
});
|
|
489
|
+
// Every refill sign-in's MFA verify deletes the persona's AAL1 sessions on
|
|
490
|
+
// staging (ENT-4994), so repeated failures back off instead of retrying
|
|
491
|
+
// every tick.
|
|
492
|
+
if (counts[persona] < want && entry.nextMintAttemptAt && Date.parse(entry.nextMintAttemptAt) > now()) {
|
|
493
|
+
summary.backoffSkips += 1;
|
|
494
|
+
logger({
|
|
495
|
+
event: "mint_backoff_skip",
|
|
496
|
+
persona,
|
|
497
|
+
consecutiveMintFailures: entry.consecutiveMintFailures,
|
|
498
|
+
lastMintError: entry.lastMintError,
|
|
499
|
+
nextMintAttemptAt: entry.nextMintAttemptAt,
|
|
500
|
+
});
|
|
501
|
+
} else {
|
|
502
|
+
while (counts[persona] < want) {
|
|
503
|
+
summary.signIns += 1;
|
|
504
|
+
try {
|
|
505
|
+
await mintIntoReady(persona);
|
|
506
|
+
counts[persona] += 1;
|
|
507
|
+
entry.consecutiveMintFailures = 0;
|
|
508
|
+
entry.nextMintAttemptAt = null;
|
|
509
|
+
entry.lastMintError = null;
|
|
510
|
+
} catch (error) {
|
|
511
|
+
summary.mintFailures += 1;
|
|
512
|
+
entry.consecutiveMintFailures += 1;
|
|
513
|
+
entry.lastMintError = error?.code ?? "error";
|
|
514
|
+
entry.nextMintAttemptAt = new Date(
|
|
515
|
+
now() + nextMintBackoffMs({ failures: entry.consecutiveMintFailures, intervalMs }),
|
|
516
|
+
).toISOString();
|
|
517
|
+
logger({
|
|
518
|
+
event: "mint_failed",
|
|
519
|
+
persona,
|
|
520
|
+
code: entry.lastMintError,
|
|
521
|
+
message: error?.message ?? String(error),
|
|
522
|
+
consecutiveMintFailures: entry.consecutiveMintFailures,
|
|
523
|
+
nextMintAttemptAt: entry.nextMintAttemptAt,
|
|
524
|
+
});
|
|
525
|
+
break;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
entry.belowTargetTicks = counts[persona] < want ? entry.belowTargetTicks + 1 : 0;
|
|
530
|
+
}
|
|
531
|
+
// A persona the config no longer serves (removed or 0) drops its backoff and streaks, so
|
|
532
|
+
// re-enabling it after a repair mints on the next tick (Codex R13).
|
|
533
|
+
for (const persona of Object.keys(health.personas)) {
|
|
534
|
+
if (!(wanted.get(persona) > 0)) delete health.personas[persona];
|
|
535
|
+
}
|
|
536
|
+
if (summary.mintFailures === 0 && summary.backoffSkips === 0) health.lastSuccessfulTickAt = summary.at;
|
|
537
|
+
await replaceFile(healthPath, `${JSON.stringify(health, null, 2)}\n`);
|
|
538
|
+
summary.ready = counts;
|
|
539
|
+
logger({ event: "tick", ...summary });
|
|
540
|
+
await replaceFile(lastTickPath, `${JSON.stringify(summary, null, 2)}\n`);
|
|
541
|
+
return summary;
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
async function mintOne({ persona }) {
|
|
546
|
+
assertPersonaAllowed(target, persona);
|
|
547
|
+
const result = await withTickLock(async () => {
|
|
548
|
+
const minted = await mintIntoReady(persona);
|
|
549
|
+
logger({ event: "mint_one", persona, slotId: minted.slotId, signIns: 1 });
|
|
550
|
+
return minted;
|
|
551
|
+
});
|
|
552
|
+
if (result.skipped) throw Object.assign(new Error("A pool tick is running; retry in a minute."), { code: result.skipped });
|
|
553
|
+
return result;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
async function take({ persona, owner }) {
|
|
557
|
+
assertPersonaAllowed(target, persona);
|
|
558
|
+
await ensureLayout();
|
|
559
|
+
const candidates = [];
|
|
560
|
+
for (const slotId of await list(dirs.ready)) {
|
|
561
|
+
if (!slotId.startsWith(`${personaKey(persona)}~`)) continue;
|
|
562
|
+
try {
|
|
563
|
+
const meta = await readJson(join(dirs.ready, slotId, "meta.json"));
|
|
564
|
+
if (meta.persona === persona) candidates.push({ slotId, verifiedAt: Date.parse(meta.verifiedAt) || 0 });
|
|
565
|
+
} catch {
|
|
566
|
+
// Moved by a racer or half-written: not a candidate.
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
candidates.sort((a, b) => b.verifiedAt - a.verifiedAt);
|
|
570
|
+
for (const { slotId } of candidates) {
|
|
571
|
+
const takenDir = join(dirs.taken, slotId);
|
|
572
|
+
if (!(await claim(join(dirs.ready, slotId), takenDir))) continue;
|
|
573
|
+
// From here this caller is the slot's only holder; the slot never goes back. The claim
|
|
574
|
+
// marker tells a concurrent tick this slot is in use even before takenAt exists.
|
|
575
|
+
await writeFileExclusive(join(takenDir, "claim.json"), JSON.stringify({ claimedAt: new Date(now()).toISOString() }));
|
|
576
|
+
let slot;
|
|
577
|
+
try {
|
|
578
|
+
slot = await readSlot(takenDir);
|
|
579
|
+
} catch (error) {
|
|
580
|
+
logger({ event: "take_discard", slotId, reason: error?.code ?? "unreadable" });
|
|
581
|
+
await removeSlot(takenDir);
|
|
582
|
+
continue;
|
|
583
|
+
}
|
|
584
|
+
const result = await signIn.refresh({ target, storedSession: slot.storedSession, persona });
|
|
585
|
+
if (!result.ok) {
|
|
586
|
+
logger({ event: "take_discard", slotId, persona, reason: result.reason, transient: Boolean(result.transient) });
|
|
587
|
+
await removeSlot(takenDir);
|
|
588
|
+
continue;
|
|
589
|
+
}
|
|
590
|
+
const meta = {
|
|
591
|
+
...buildMeta({
|
|
592
|
+
persona,
|
|
593
|
+
slotId,
|
|
594
|
+
storedSession: result.storedSession,
|
|
595
|
+
mintedAt: slot.meta.mintedAt,
|
|
596
|
+
refreshCount: (slot.meta.refreshCount ?? 0) + 1,
|
|
597
|
+
}),
|
|
598
|
+
takenAt: new Date(now()).toISOString(),
|
|
599
|
+
owner: String(owner ?? "unknown"),
|
|
600
|
+
};
|
|
601
|
+
await writeSlotFiles(takenDir, { storedSession: result.storedSession, meta, replace: true });
|
|
602
|
+
logger({ event: "take_ok", slotId, persona, owner: meta.owner });
|
|
603
|
+
return { slotId, statePath: join(takenDir, "state.json"), dir: takenDir, meta };
|
|
604
|
+
}
|
|
605
|
+
throw new PoolEmptyError(persona, tickIntervalHint);
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
async function discardTaken(slotId) {
|
|
609
|
+
await removeSlot(join(dirs.taken, slotId));
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
async function status() {
|
|
613
|
+
const describe = async (dir, slotId) => {
|
|
614
|
+
const meta = await readJson(join(dir, slotId, "meta.json"));
|
|
615
|
+
return {
|
|
616
|
+
slotId,
|
|
617
|
+
persona: meta.persona,
|
|
618
|
+
aal: meta.aal,
|
|
619
|
+
sessionRef: meta.sessionRef,
|
|
620
|
+
ageSeconds: Math.round((now() - Date.parse(meta.mintedAt)) / 1000),
|
|
621
|
+
verifiedAgeSeconds: Math.round((now() - Date.parse(meta.verifiedAt)) / 1000),
|
|
622
|
+
accessExpiresInSeconds: Math.round((Date.parse(meta.accessExpiresAt) - now()) / 1000),
|
|
623
|
+
};
|
|
624
|
+
};
|
|
625
|
+
const collect = async (dir) => {
|
|
626
|
+
const rows = [];
|
|
627
|
+
for (const slotId of await list(dir)) {
|
|
628
|
+
try {
|
|
629
|
+
rows.push(await describe(dir, slotId));
|
|
630
|
+
} catch {
|
|
631
|
+
rows.push({ slotId, unreadable: true });
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
return rows;
|
|
635
|
+
};
|
|
636
|
+
let lastTick = null;
|
|
637
|
+
try {
|
|
638
|
+
lastTick = await readJson(lastTickPath);
|
|
639
|
+
} catch {
|
|
640
|
+
lastTick = null;
|
|
641
|
+
}
|
|
642
|
+
return {
|
|
643
|
+
poolDir,
|
|
644
|
+
target: target.name,
|
|
645
|
+
health: await readHealth(),
|
|
646
|
+
ready: await collect(dirs.ready),
|
|
647
|
+
taken: (await list(dirs.taken)).length,
|
|
648
|
+
work: (await list(dirs.work)).length,
|
|
649
|
+
tickRunning: (await list(poolDir)).includes("tick.lock"),
|
|
650
|
+
lastTick,
|
|
651
|
+
};
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
async function purge() {
|
|
655
|
+
let removed = 0;
|
|
656
|
+
for (const dir of Object.values(dirs)) {
|
|
657
|
+
for (const slotId of await list(dir)) {
|
|
658
|
+
await removeSlot(join(dir, slotId));
|
|
659
|
+
removed += 1;
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
return removed;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
return { tick, mintOne, take, discardTaken, status, purge, paths: { ...dirs, lockPath, lastTickPath } };
|
|
666
|
+
}
|