@parcha/pi-durable-disk 0.0.0-stage → 0.1.0-beta.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +329 -0
  2. package/LICENSE +202 -0
  3. package/NOTICE +2 -0
  4. package/README.md +695 -2
  5. package/bin/archil-scoped +165 -0
  6. package/dist/app.d.ts +40 -0
  7. package/dist/app.js +47 -0
  8. package/dist/claim.d.ts +222 -0
  9. package/dist/claim.js +722 -0
  10. package/dist/cli.d.ts +64 -0
  11. package/dist/cli.js +664 -0
  12. package/dist/commands.d.ts +67 -0
  13. package/dist/commands.js +299 -0
  14. package/dist/confine.d.ts +91 -0
  15. package/dist/confine.js +341 -0
  16. package/dist/env.d.ts +129 -0
  17. package/dist/env.js +608 -0
  18. package/dist/errors.d.ts +47 -0
  19. package/dist/errors.js +55 -0
  20. package/dist/fork.d.ts +41 -0
  21. package/dist/fork.js +181 -0
  22. package/dist/hosts/daytona-launch.d.ts +52 -0
  23. package/dist/hosts/daytona-launch.js +270 -0
  24. package/dist/hosts/daytona.d.ts +147 -0
  25. package/dist/hosts/daytona.js +368 -0
  26. package/dist/hosts/docker.d.ts +103 -0
  27. package/dist/hosts/docker.js +362 -0
  28. package/dist/hosts/local-host.d.ts +71 -0
  29. package/dist/hosts/local-host.js +356 -0
  30. package/dist/index.d.ts +16 -0
  31. package/dist/index.js +17 -0
  32. package/dist/lease.d.ts +7 -0
  33. package/dist/lease.js +6 -0
  34. package/dist/local-run.d.ts +10 -0
  35. package/dist/local-run.js +63 -0
  36. package/dist/park.d.ts +92 -0
  37. package/dist/park.js +229 -0
  38. package/dist/run.d.ts +290 -0
  39. package/dist/run.js +785 -0
  40. package/dist/serve.d.ts +101 -0
  41. package/dist/serve.js +393 -0
  42. package/dist/status.d.ts +123 -0
  43. package/dist/status.js +264 -0
  44. package/dist/store.d.ts +103 -0
  45. package/dist/store.js +217 -0
  46. package/dist/supervise.d.ts +340 -0
  47. package/dist/supervise.js +528 -0
  48. package/dist/watchdog.d.ts +39 -0
  49. package/dist/watchdog.js +137 -0
  50. package/package.json +87 -4
@@ -0,0 +1,101 @@
1
+ import type { Context, JsonValue } from "@earendil-works/chord";
2
+ import type { Harness } from "@earendil-works/pi-durable";
3
+ import type { RunRef } from "./claim.js";
4
+ import { PdaError } from "./errors.js";
5
+ import type { DurableRun } from "./run.js";
6
+ import { type EnsureOptions, type EnsureResult, type HostDriver } from "./supervise.js";
7
+ /** 503 codes the client answers by asking the supervisor again and retrying. */
8
+ export declare const RETRY_CODES: readonly ["OPENING", "PARKING", "RELEASED"];
9
+ export type ServeCode = (typeof RETRY_CODES)[number] | "BAD_REQUEST" | "UNAUTHORIZED" | "NOT_FOUND" | "BUSY" | "FAILED";
10
+ export interface ServeOptions {
11
+ /** Default 0: any free port (the address is in `url` and in run.json's holder). */
12
+ readonly port?: number;
13
+ /** Default 127.0.0.1. Any address that is not loopback needs `token`; a wildcard (0.0.0.0, ::) needs `url` too. */
14
+ readonly host?: string;
15
+ /**
16
+ * The address clients reach this server at (`http://` or `https://`), which goes into run.json's holder. Default: the
17
+ * bound address, which only works for a specific host; a wildcard bind has no address a client can use.
18
+ */
19
+ readonly url?: string;
20
+ /** Every request must carry `authorization: Bearer <token>` (see `readServeToken`). Optional on loopback only. */
21
+ readonly token?: string;
22
+ /** How the root conversation is created when absent (pi's `root()` options); a submit without `conversationId` goes there. */
23
+ readonly root?: Parameters<Harness["root"]>[1];
24
+ /** Called each time the last open request ends (parking looks again). */
25
+ readonly onIdle?: () => void;
26
+ readonly context?: Context;
27
+ }
28
+ export interface RunServer {
29
+ /** `http://<host>:<port>`. */
30
+ readonly url: string;
31
+ /** Open requests: submits waiting for their answer, event streams. */
32
+ readonly active: number;
33
+ /** Serve this run (until then every request is 503 OPENING). */
34
+ attach(run: DurableRun): void;
35
+ /** Refuse new submissions with 503 PARKING; the returned function admits them again. */
36
+ pause(): () => void;
37
+ close(): Promise<void>;
38
+ }
39
+ export type ServeConfigErrorCode = "SERVE_TOKEN_REQUIRED" | "SERVE_TOKEN_INVALID" | "SERVE_URL_REQUIRED" | "SERVE_URL_INVALID";
40
+ /**
41
+ * A non-loopback address without a token, a token file that is missing, empty or not private, or a wildcard bind
42
+ * without the URL clients reach it at. Exit 2 (usage).
43
+ */
44
+ export declare class ServeConfigError extends PdaError {
45
+ constructor(code: ServeConfigErrorCode, message: string, options?: {
46
+ cause?: unknown;
47
+ });
48
+ }
49
+ /** 127.0.0.0/8, ::1 (also as an IPv4-mapped address) and `localhost`; everything else reaches the network. */
50
+ export declare function isLoopback(host: string): boolean;
51
+ /**
52
+ * The serve token from `path`: a regular file no one but its owner may read or write (0600 or stricter), holding one
53
+ * token without whitespace. Read from a file so it is never in argv or an environment.
54
+ */
55
+ export declare function readServeToken(path: string): string;
56
+ /** Start listening; requests are answered once `attach` hands over the open run. */
57
+ export declare function serveRun(options?: ServeOptions): Promise<RunServer>;
58
+ export type ServeErrorCode = "RUN_TERMINAL" | "NOT_SERVED";
59
+ /** The run is done or failed (RUN_TERMINAL), or no instance answered before the deadline (NOT_SERVED). */
60
+ export declare class ServeError extends PdaError {
61
+ readonly ensured: readonly EnsureResult[];
62
+ constructor(code: ServeErrorCode, message: string, ensured: readonly EnsureResult[]);
63
+ }
64
+ export interface RequestOptions {
65
+ readonly host: HostDriver;
66
+ /** ensureRunning's options; every call here is a demand. */
67
+ readonly ensure: Omit<EnsureOptions, "demand">;
68
+ /** Give up after this long; default 120 s. A submit that waits for its answer needs room for the model. */
69
+ readonly timeoutMs?: number;
70
+ /**
71
+ * How long an instance this call started may take to open before the supervisor is asked again; default 60 s. Until
72
+ * then a second start would only be refused (76): the control API can list the new mount late.
73
+ */
74
+ readonly startTimeoutMs?: number;
75
+ /** The instance's serve token, sent as `authorization: Bearer <token>`. */
76
+ readonly token?: string;
77
+ readonly fetch?: typeof fetch;
78
+ }
79
+ export type RunResponse = {
80
+ readonly status: number;
81
+ readonly body: JsonValue;
82
+ /** The instance that answered. */
83
+ readonly url: string;
84
+ /** Every ensureRunning result on the way, in order (empty when the instance answered at once). */
85
+ readonly ensured: readonly EnsureResult[];
86
+ };
87
+ /**
88
+ * Send one request to run `ref`'s instance, waking it if needed: read run.json (S3) and send the request to the running
89
+ * holder's `serve` address while its lease is fresh. When there is none, it does not answer, it answers 503 with a
90
+ * retry code, or its lease lapses while the request is open (a frozen instance accepts connections and never answers;
91
+ * run.json is read again every third of a lease), ask `ensureRunning` with demand (start a released or lost run) and
92
+ * try again until `timeoutMs`. After a start, its own or one the supervisor reports `starting`, it waits for that
93
+ * instance to write its generation (or `startTimeoutMs`) before asking again, and an instance still opening (503
94
+ * OPENING) is waited for. A submit carries a `requestId`, so
95
+ * resending it to the next incarnation is safe: pi admits it once.
96
+ */
97
+ export declare function requestRun(ref: RunRef, request: {
98
+ readonly method: "GET" | "POST";
99
+ readonly path: string;
100
+ readonly body?: JsonValue;
101
+ }, options: RequestOptions): Promise<RunResponse>;
package/dist/serve.js ADDED
@@ -0,0 +1,393 @@
1
+ // The serve mode, a Durable Object stub's fetch over pi's own API. The instance answers `POST /submit` (pi's
2
+ // submit with the caller's `requestId`, which pi deduplicates in the run's store, so a retry that lands on the next
3
+ // incarnation finds the same submission), `POST /abort`, `GET /events` (pi's agent events as server-sent events) and
4
+ // `GET /status`. It listens before the run opens, so its address can go into run.json's holder, and answers 503 with a
5
+ // code while the run opens, parks or is released. `requestRun` is the client: it finds the instance through run.json,
6
+ // starts the run through `ensureRunning` (with demand) when it is released or gone, and retries while it starts.
7
+ // Open requests keep the instance up (parking waits for none). On loopback a bearer token is optional; on any other
8
+ // address it is required, read from a private file, and checked in constant time on every route.
9
+ import { createHash, timingSafeEqual } from "node:crypto";
10
+ import { readFileSync, statSync } from "node:fs";
11
+ import { createServer } from "node:http";
12
+ import { isIP } from "node:net";
13
+ import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context";
14
+ import { ConversationBusy, watchEvents } from "@earendil-works/pi-durable";
15
+ import { PdaError } from "./errors.js";
16
+ import { busyState } from "./park.js";
17
+ import { isLeaseFresh } from "./status.js";
18
+ import { CONTROL_TIMEOUT_MS, ensureRunning, LEASE_EXPIRY_MS, readRunStatus, withTimeouts } from "./supervise.js";
19
+ /** 503 codes the client answers by asking the supervisor again and retrying. */
20
+ export const RETRY_CODES = ["OPENING", "PARKING", "RELEASED"];
21
+ const MAX_BODY = 1 << 20;
22
+ const REQUEST_ID = /^[\x21-\x7e]{1,200}$/;
23
+ /**
24
+ * A non-loopback address without a token, a token file that is missing, empty or not private, or a wildcard bind
25
+ * without the URL clients reach it at. Exit 2 (usage).
26
+ */
27
+ export class ServeConfigError extends PdaError {
28
+ constructor(code, message, options = {}) {
29
+ super(code, message, { cause: options.cause, exitCode: 2 });
30
+ }
31
+ }
32
+ /** 127.0.0.0/8, ::1 (also as an IPv4-mapped address) and `localhost`; everything else reaches the network. */
33
+ export function isLoopback(host) {
34
+ const h = host.toLowerCase();
35
+ if (h === "localhost" || h === "::1")
36
+ return true;
37
+ if (isIP(h) === 4)
38
+ return h.startsWith("127.");
39
+ return h.startsWith("::ffff:127.") && isIP(h) === 6;
40
+ }
41
+ /**
42
+ * The serve token from `path`: a regular file no one but its owner may read or write (0600 or stricter), holding one
43
+ * token without whitespace. Read from a file so it is never in argv or an environment.
44
+ */
45
+ export function readServeToken(path) {
46
+ let mode;
47
+ try {
48
+ const st = statSync(path);
49
+ if (!st.isFile())
50
+ throw new Error("not a regular file");
51
+ mode = st.mode & 0o777;
52
+ }
53
+ catch (cause) {
54
+ throw new ServeConfigError("SERVE_TOKEN_INVALID", `cannot use ${path} as the serve token file: ${cause.message}`, { cause });
55
+ }
56
+ if (mode & 0o077)
57
+ throw new ServeConfigError("SERVE_TOKEN_INVALID", `${path} has mode ${mode.toString(8)}; the serve token file must be 0600`);
58
+ const token = readFileSync(path, "utf8").trim();
59
+ if (!token || /\s/.test(token))
60
+ throw new ServeConfigError("SERVE_TOKEN_INVALID", `${path} holds no token, or one with whitespace`);
61
+ return token;
62
+ }
63
+ const digest = (text) => createHash("sha256").update(text).digest();
64
+ /** 0.0.0.0, :: and their spellings: every address of the host, none of which a client can be told. */
65
+ const isWildcard = (host) => {
66
+ const h = host.toLowerCase();
67
+ return h === "0.0.0.0" || h === "::" || h === "[::]" || h === "0:0:0:0:0:0:0:0" || h === "::ffff:0.0.0.0";
68
+ };
69
+ class HttpError extends Error {
70
+ status;
71
+ code;
72
+ constructor(status, code, message) {
73
+ super(message);
74
+ this.status = status;
75
+ this.code = code;
76
+ }
77
+ }
78
+ const unavailable = (code, message) => new HttpError(503, code, message);
79
+ function send(res, status, body) {
80
+ const text = JSON.stringify(body);
81
+ res.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(text), ...(status === 503 ? { "retry-after": "1" } : {}) });
82
+ res.end(text);
83
+ }
84
+ async function readJson(req) {
85
+ let size = 0;
86
+ const chunks = [];
87
+ for await (const chunk of req) {
88
+ size += chunk.length;
89
+ if (size > MAX_BODY)
90
+ throw new HttpError(413, "BAD_REQUEST", `body over ${MAX_BODY} bytes`);
91
+ chunks.push(chunk);
92
+ }
93
+ if (size === 0)
94
+ return {};
95
+ let value;
96
+ try {
97
+ value = JSON.parse(Buffer.concat(chunks).toString("utf8"));
98
+ }
99
+ catch {
100
+ throw new HttpError(400, "BAD_REQUEST", "body is not JSON");
101
+ }
102
+ if (typeof value !== "object" || value === null || Array.isArray(value))
103
+ throw new HttpError(400, "BAD_REQUEST", "body is not a JSON object");
104
+ return value;
105
+ }
106
+ const optionalId = (value, name) => {
107
+ if (value === undefined)
108
+ return undefined;
109
+ if (!Number.isSafeInteger(value) || value < 0)
110
+ throw new HttpError(400, "BAD_REQUEST", `${name} must be a non-negative integer`);
111
+ return value;
112
+ };
113
+ /** The text of an answer entry (an assistant message's text blocks). */
114
+ async function answerOf(conversation, entry, context) {
115
+ const page = await conversation.entries({ minEntryId: entry, maxEntryId: entry }, 1, undefined, context);
116
+ const text = (page.items[0]?.model ?? [])
117
+ .flatMap((m) => (m.role === "assistant" ? m.content.flatMap((b) => (b.type === "text" ? [b.text] : [])) : []))
118
+ .join("");
119
+ return { entry, text };
120
+ }
121
+ /** Start listening; requests are answered once `attach` hands over the open run. */
122
+ export async function serveRun(options = {}) {
123
+ const base = options.context ?? BACKGROUND_CONTEXT;
124
+ const bind = options.host ?? "127.0.0.1";
125
+ if (options.token !== undefined && (!options.token || /\s/.test(options.token)))
126
+ throw new ServeConfigError("SERVE_TOKEN_INVALID", "the serve token is empty or has whitespace");
127
+ if (options.token === undefined && !isLoopback(bind)) {
128
+ throw new ServeConfigError("SERVE_TOKEN_REQUIRED", `serving on ${bind}, which is not loopback, needs a bearer token (--serve-token-file)`);
129
+ }
130
+ if (options.url !== undefined && !["http:", "https:"].includes(URL.parse(options.url)?.protocol ?? "")) {
131
+ throw new ServeConfigError("SERVE_URL_INVALID", `${options.url} is not an http:// or https:// address`);
132
+ }
133
+ if (options.url === undefined && isWildcard(bind)) {
134
+ throw new ServeConfigError("SERVE_URL_REQUIRED", `serving on ${bind} binds every address; name the one clients reach (--serve-url)`);
135
+ }
136
+ // Both sides are hashed to one length, so the comparison takes the same time whatever the caller sent.
137
+ const expected = options.token === undefined ? undefined : digest(`Bearer ${options.token}`);
138
+ const authorized = (req) => expected === undefined || timingSafeEqual(digest(req.headers.authorization ?? ""), expected);
139
+ let run;
140
+ let closed = false;
141
+ let pauses = 0;
142
+ let active = 0;
143
+ const hold = () => void active++;
144
+ const letGo = () => {
145
+ if (--active === 0)
146
+ options.onIdle?.();
147
+ };
148
+ const harness = () => {
149
+ if (!run)
150
+ throw unavailable("OPENING", "the run is still opening");
151
+ if (closed || run.fenced)
152
+ throw unavailable("RELEASED", "the run is released or fenced on this instance");
153
+ return run.harness;
154
+ };
155
+ const conversationOf = async (id, context) => {
156
+ const h = harness();
157
+ if (id === undefined)
158
+ return h.root(context, options.root);
159
+ const conversation = await h.conversation(id, context);
160
+ if (!conversation)
161
+ throw new HttpError(404, "NOT_FOUND", `no conversation ${id}`);
162
+ return conversation;
163
+ };
164
+ async function submit(body, context) {
165
+ const { requestId, content, whenBusy, wait } = body;
166
+ if (typeof requestId !== "string" || !REQUEST_ID.test(requestId))
167
+ throw new HttpError(400, "BAD_REQUEST", "requestId must be 1 to 200 printable ASCII characters");
168
+ if (typeof content !== "string" && !Array.isArray(content))
169
+ throw new HttpError(400, "BAD_REQUEST", "content must be a string or an array of content blocks");
170
+ if (whenBusy !== undefined && whenBusy !== "steer" && whenBusy !== "followUp" && whenBusy !== "reject")
171
+ throw new HttpError(400, "BAD_REQUEST", "whenBusy is steer, followUp or reject");
172
+ if (pauses > 0)
173
+ throw unavailable("PARKING", "the run is parking or draining");
174
+ const conversation = await conversationOf(optionalId(body.conversationId, "conversationId"), context);
175
+ const submission = await conversation.submit({ type: "input", requestId, content: content, ...(whenBusy === undefined ? {} : { whenBusy }) }, context);
176
+ let record = await submission.status(context);
177
+ if (wait === true)
178
+ record = await submission.wait(context);
179
+ const answer = record.type === "input" && record.status === "done" ? await answerOf(conversation, record.answer, context) : undefined;
180
+ return { run: run.ref.id, generation: run.generation, submission: record, ...(answer ? { answer } : {}) };
181
+ }
182
+ async function abort(body, context) {
183
+ const submissionId = optionalId(body.submissionId, "submissionId");
184
+ const conversationId = optionalId(body.conversationId, "conversationId");
185
+ if (submissionId !== undefined) {
186
+ const result = await harness().abortSubmission(submissionId, context, conversationId);
187
+ if (result === "not_found")
188
+ throw new HttpError(404, "NOT_FOUND", `no submission ${submissionId}`);
189
+ return { result };
190
+ }
191
+ await (await conversationOf(conversationId, context)).abort(context);
192
+ return { result: "aborted" };
193
+ }
194
+ async function status(context) {
195
+ const inspection = await harness().inspect(context);
196
+ return {
197
+ run: run.ref.id,
198
+ generation: run.generation,
199
+ status: run.record.status,
200
+ busy: busyState(inspection, Date.now(), 0),
201
+ scheduling: inspection.scheduling,
202
+ tasks: inspection.tasks.map((t) => ({ id: t.record.id, kind: t.record.kind, state: t.state.kind })),
203
+ submissions: inspection.submissions.length,
204
+ };
205
+ }
206
+ async function events(req, res, url, context) {
207
+ const raw = url.searchParams.get("conversationId");
208
+ const conversation = await conversationOf(raw === null ? undefined : optionalId(Number(raw), "conversationId"), context);
209
+ const stream = await watchEvents(harness(), conversation.id, base);
210
+ const write = (event, data) => void res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
211
+ res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" });
212
+ hold();
213
+ write("snapshot", stream.snapshot);
214
+ stream.start(async (batch) => write("events", batch));
215
+ req.once("close", () => void stream.stop());
216
+ void stream.closed.then((end) => {
217
+ write("end", { reason: end.reason });
218
+ res.end();
219
+ letGo();
220
+ });
221
+ }
222
+ const server = createServer((req, res) => {
223
+ if (!authorized(req)) {
224
+ res.setHeader("www-authenticate", "Bearer");
225
+ return send(res, 401, { error: "UNAUTHORIZED", message: "a bearer token is required" });
226
+ }
227
+ const abortOnClose = new AbortController();
228
+ res.once("close", () => abortOnClose.abort());
229
+ const context = withAbortSignal(abortOnClose.signal, base);
230
+ const url = new URL(req.url ?? "/", "http://serve");
231
+ const route = `${req.method} ${url.pathname}`;
232
+ const answer = async () => {
233
+ switch (route) {
234
+ case "POST /submit": {
235
+ const body = await readJson(req);
236
+ hold();
237
+ try {
238
+ return send(res, 200, await submit(body, context));
239
+ }
240
+ finally {
241
+ letGo();
242
+ }
243
+ }
244
+ case "POST /abort":
245
+ return send(res, 200, await abort(await readJson(req), context));
246
+ case "GET /status":
247
+ return send(res, 200, await status(context));
248
+ case "GET /events":
249
+ return events(req, res, url, context);
250
+ default:
251
+ throw new HttpError(404, "NOT_FOUND", `no route ${route}`);
252
+ }
253
+ };
254
+ answer().catch((error) => {
255
+ if (res.headersSent)
256
+ return void res.end();
257
+ if (error instanceof HttpError)
258
+ return send(res, error.status, { error: error.code, message: error.message });
259
+ if (error instanceof ConversationBusy)
260
+ return send(res, 409, { error: "BUSY", message: error.message });
261
+ if (run && (closed || run.fenced))
262
+ return send(res, 503, { error: "RELEASED", message: error.message });
263
+ send(res, 500, { error: "FAILED", message: error?.message ?? String(error) });
264
+ });
265
+ });
266
+ await new Promise((resolve, reject) => {
267
+ server.once("error", reject);
268
+ server.listen(options.port ?? 0, bind, () => resolve());
269
+ });
270
+ const address = server.address();
271
+ const host = address.family === "IPv6" ? `[${address.address}]` : address.address;
272
+ return {
273
+ url: options.url ?? `http://${host}:${address.port}`,
274
+ get active() {
275
+ return active;
276
+ },
277
+ attach(open) {
278
+ run = open;
279
+ open.harness.subscribeClose(() => void (closed = true));
280
+ },
281
+ pause() {
282
+ pauses++;
283
+ let undone = false;
284
+ return () => {
285
+ if (!undone)
286
+ (undone = true), pauses--;
287
+ };
288
+ },
289
+ close() {
290
+ closed = true;
291
+ return new Promise((resolve) => {
292
+ server.close(() => resolve());
293
+ server.closeAllConnections();
294
+ });
295
+ },
296
+ };
297
+ }
298
+ /** The run is done or failed (RUN_TERMINAL), or no instance answered before the deadline (NOT_SERVED). */
299
+ export class ServeError extends PdaError {
300
+ ensured;
301
+ constructor(code, message, ensured) {
302
+ super(code, message);
303
+ this.ensured = ensured;
304
+ }
305
+ }
306
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
307
+ /** The running holder's serve address while its lease is fresh; otherwise there is no instance to talk to. */
308
+ function servedBy(record, now, leaseMs) {
309
+ if (record?.status !== "running" || !isLeaseFresh(record, now, leaseMs))
310
+ return undefined;
311
+ return typeof record.holder?.serve === "string" ? record.holder.serve : undefined;
312
+ }
313
+ /**
314
+ * Send one request to run `ref`'s instance, waking it if needed: read run.json (S3) and send the request to the running
315
+ * holder's `serve` address while its lease is fresh. When there is none, it does not answer, it answers 503 with a
316
+ * retry code, or its lease lapses while the request is open (a frozen instance accepts connections and never answers;
317
+ * run.json is read again every third of a lease), ask `ensureRunning` with demand (start a released or lost run) and
318
+ * try again until `timeoutMs`. After a start, its own or one the supervisor reports `starting`, it waits for that
319
+ * instance to write its generation (or `startTimeoutMs`) before asking again, and an instance still opening (503
320
+ * OPENING) is waited for. A submit carries a `requestId`, so
321
+ * resending it to the next incarnation is safe: pi admits it once.
322
+ */
323
+ export async function requestRun(ref, request, options) {
324
+ const fetchFn = options.fetch ?? fetch;
325
+ const now = options.ensure.now ?? Date.now;
326
+ const leaseMs = options.ensure.leaseExpiryMs ?? LEASE_EXPIRY_MS;
327
+ const timeoutMs = options.timeoutMs ?? 120_000;
328
+ const deadline = Date.now() + timeoutMs;
329
+ const ensured = [];
330
+ // Every control call is bounded by the control timeout, and every step of this loop by what is left of `timeoutMs`.
331
+ const control = withTimeouts(options.ensure.control, options.ensure.controlTimeoutMs ?? CONTROL_TIMEOUT_MS);
332
+ const notServed = () => new ServeError("NOT_SERVED", `run ${ref.id} did not answer in ${timeoutMs} ms`, ensured);
333
+ const inTime = (step) => new Promise((resolve, reject) => {
334
+ const timer = setTimeout(() => reject(notServed()), Math.max(0, deadline - Date.now()));
335
+ step.then((value) => (clearTimeout(timer), resolve(value)), (error) => (clearTimeout(timer), reject(error)));
336
+ });
337
+ const attempt = async (url) => {
338
+ const stop = new AbortController();
339
+ let looking = false;
340
+ const watch = setInterval(() => {
341
+ if (looking)
342
+ return;
343
+ looking = true;
344
+ readRunStatus(control, ref.id)
345
+ .then((r) => servedBy(r, now(), leaseMs) !== url && stop.abort(), () => { })
346
+ .finally(() => (looking = false));
347
+ }, Math.min(Math.max(1_000, leaseMs / 3), 10_000));
348
+ try {
349
+ return await fetchFn(new URL(request.path, url), {
350
+ method: request.method,
351
+ headers: {
352
+ ...(request.body === undefined ? {} : { "content-type": "application/json" }),
353
+ ...(options.token === undefined ? {} : { authorization: `Bearer ${options.token}` }),
354
+ },
355
+ ...(request.body === undefined ? {} : { body: JSON.stringify(request.body) }),
356
+ signal: AbortSignal.any([stop.signal, AbortSignal.timeout(Math.max(1, deadline - Date.now()))]),
357
+ }).then(async (res) => ({ status: res.status, body: (await res.json().catch(() => null)) }), () => undefined);
358
+ }
359
+ finally {
360
+ clearInterval(watch);
361
+ }
362
+ };
363
+ let pendingStart;
364
+ for (let pause = 100;; pause = Math.min(pause * 2, 1_000)) {
365
+ const record = await inTime(readRunStatus(control, ref.id));
366
+ const url = servedBy(record, now(), leaseMs);
367
+ let opening = false;
368
+ if (url) {
369
+ const r = await attempt(url);
370
+ const code = r?.body?.error;
371
+ if (r && !(r.status === 503 && RETRY_CODES.includes(code)))
372
+ return { ...r, url, ensured };
373
+ opening = code === "OPENING";
374
+ }
375
+ // An instance that is opening, or a start in flight whose instance has not written its generation yet, is left to open.
376
+ const generation = record?.generation ?? 0;
377
+ const awaitingStart = pendingStart !== undefined && Date.now() < pendingStart.until && generation < pendingStart.generation;
378
+ if (!opening && !awaitingStart) {
379
+ const result = await inTime(ensureRunning(ref, options.host, { ...options.ensure, demand: true }));
380
+ ensured.push(result);
381
+ if (result.action === "terminal")
382
+ throw new ServeError("RUN_TERMINAL", `run ${ref.id} is ${result.status}`, ensured);
383
+ // A start by this call, or one the supervisor's start grace reports in flight: wait for that generation.
384
+ if (result.action === "started")
385
+ pendingStart = { until: Date.now() + (options.startTimeoutMs ?? 60_000), generation: generation + 1 };
386
+ if (result.action === "starting")
387
+ pendingStart = { until: Date.now() + (options.startTimeoutMs ?? 60_000), generation: result.generation };
388
+ }
389
+ if (Date.now() + pause > deadline)
390
+ throw notServed();
391
+ await sleep(pause);
392
+ }
393
+ }
@@ -0,0 +1,123 @@
1
+ import type { JsonValue } from "@earendil-works/chord";
2
+ import { PdaError } from "./errors.js";
3
+ export declare const RUN_JSON = "run.json";
4
+ /** The temporary file every write goes through; renamed over `run.json` once it is fsynced. */
5
+ export declare const RUN_JSON_TEMP = "run.json.tmp";
6
+ /** Heartbeat period: the instance rewrites `heartbeatAt` this often. */
7
+ export declare const DEFAULT_HEARTBEAT_MS = 20000;
8
+ /** A lease whose last heartbeat is older than this has expired; the supervisor may revoke. */
9
+ export declare const DEFAULT_LEASE_EXPIRY_MS = 90000;
10
+ /** The instance fences itself this long before its lease expires, on its own monotonic clock. */
11
+ export declare const DEFAULT_LEASE_MARGIN_MS = 15000;
12
+ export declare const RUN_STATUSES: readonly ["running", "sleeping", "paused", "done", "failed"];
13
+ export type RunStatus = (typeof RUN_STATUSES)[number];
14
+ /**
15
+ * The incarnation that wrote the record: the host driver's name and enough to find and stop the process. Fields beyond
16
+ * the five named here are the host driver's own handle (a unit name, a mountpoint), kept as written so the supervisor
17
+ * can hand them back to the driver.
18
+ */
19
+ export interface RunHolder {
20
+ readonly driver: string;
21
+ readonly host: string;
22
+ /** `/proc/sys/kernel/random/boot_id`; null where the platform has none. */
23
+ readonly bootId: string | null;
24
+ readonly pid: number;
25
+ /** When this incarnation claimed the run (ISO 8601). */
26
+ readonly since: string;
27
+ readonly [field: string]: JsonValue;
28
+ }
29
+ export interface RunRecord {
30
+ readonly run: string;
31
+ readonly status: RunStatus;
32
+ /** Incremented on every claim, so a log line or receipt can name the incarnation that wrote it. */
33
+ readonly generation: number;
34
+ /** The store's last committed sequence at a clean release; null while running and after a crash. */
35
+ readonly sealedSeq: number | null;
36
+ /** When a `sleeping` run should be started again (ISO 8601); null otherwise. */
37
+ readonly wakeAt: string | null;
38
+ readonly holder: RunHolder | null;
39
+ /** Wall-clock time at which the last successful heartbeat write started (ISO 8601). */
40
+ readonly heartbeatAt: string | null;
41
+ /** Wall-clock time of the last status transition (ISO 8601). */
42
+ readonly updatedAt: string;
43
+ readonly detail: JsonValue | null;
44
+ }
45
+ /** `run.json` is missing a field, has one of the wrong type, or is not JSON. Exit 65 (EX_DATAERR): a restart cannot fix it. */
46
+ export declare class RunRecordError extends PdaError {
47
+ constructor(message: string, options?: {
48
+ cause?: unknown;
49
+ });
50
+ }
51
+ /**
52
+ * Parse and validate the text of a `run.json`. Unknown fields are ignored, so a newer writer stays readable. Throws
53
+ * RunRecordError on anything else. This is the reader the supervisor uses on the bytes it gets over the S3 API.
54
+ */
55
+ export declare function parseRunRecord(text: string): RunRecord;
56
+ /** The `run.json` under a run's root on a local mount (or in a claim directory), or undefined when the run has none yet. */
57
+ export declare function readRunRecord(root: string | ClaimDir): Promise<RunRecord | undefined>;
58
+ /**
59
+ * True while the record's last heartbeat is younger than `expiryMs` on the reader's clock. The supervisor allows for
60
+ * clock skew between hosts inside the expiry.
61
+ */
62
+ export declare function isLeaseFresh(record: RunRecord, nowMs?: number, expiryMs?: number): boolean;
63
+ /**
64
+ * Writes `text` as `<root>/run.json` durably and atomically; replaceable for fault injection. Once `signal` is aborted
65
+ * the write must not publish: an instance that is fenced never renames a record over `run.json`.
66
+ */
67
+ export type PersistRecord = (root: string, text: string, signal?: AbortSignal) => Promise<void>;
68
+ /** Write to `run.json.tmp`, fsync it, rename it over `run.json`, fsync the directory. */
69
+ export declare const persistRecord: PersistRecord;
70
+ /**
71
+ * Write `record`; any error at all is a fence (`ownWriteFenced`), whatever its errno. Into a claim directory, the write
72
+ * goes through the open directory, after `assertMounted` (its CLAIM_UNMOUNTED fence is rethrown as is). `signal` as in
73
+ * `PersistRecord`.
74
+ */
75
+ export declare function writeRunRecord(root: string | ClaimDir, record: RunRecord, persist?: PersistRecord, signal?: AbortSignal): Promise<void>;
76
+ /** The FencedError code of a claim root that is no longer the mount the instance opened (unmounted, lazily or not). */
77
+ export declare const CLAIM_UNMOUNTED = "CLAIM_UNMOUNTED";
78
+ /** The file system type of an Archil mount in the mount table. */
79
+ export declare const ARCHIL_FSTYPE = "fuse.archil";
80
+ /**
81
+ * A run root held open as a directory for the life of the instance. `at` is `/proc/self/fd/<n>`, a path the kernel
82
+ * resolves through the open directory and never through the root's path (Node has no openat; the magic link stands in
83
+ * for it): a file made, written or renamed under `at` is in the directory opened at acquire, or the call fails (ENOTCONN
84
+ * once the FUSE daemon is gone). It never lands on the local disk under a mountpoint whose mount went away.
85
+ */
86
+ export interface ClaimDir {
87
+ /** The root's path, as the claim names it. */
88
+ readonly root: string;
89
+ /** `/proc/self/fd/<n>`. Throws once closed: the number may then name another file. */
90
+ readonly at: string;
91
+ /**
92
+ * Resolves while the root's path still resolves to the directory held open (same device, same inode). A mount that was
93
+ * unmounted (lazily, by an operator, or by its daemon's `auto_unmount`) leaves the path on another device, and a dead
94
+ * daemon fails the stat: both reject with a FencedError CLAIM_UNMOUNTED. The stat runs off the main thread.
95
+ */
96
+ assertMounted(): Promise<void>;
97
+ /** Resolves while `<root>/<name>` by path is the entry `name` of the directory held open, on its device; else as `assertMounted`. */
98
+ assertSame(name: string): Promise<void>;
99
+ /** Close the directory (idempotent). A caller closes it only once nothing can still address `at`. */
100
+ close(): Promise<void>;
101
+ }
102
+ export interface OpenClaimDirOptions {
103
+ /**
104
+ * The root must be the mount point of a mount of this type whose device is the opened directory's (default
105
+ * `fuse.archil`). Null skips the mount table: a run root on a local directory, for tests.
106
+ */
107
+ readonly fstype?: string | null;
108
+ /** Default `/proc/self/mountinfo`. */
109
+ readonly mountinfo?: string;
110
+ }
111
+ /** `major:minor` of a device number, as `/proc/self/mountinfo` writes it (glibc's encoding). */
112
+ export declare function deviceName(dev: bigint): string;
113
+ /**
114
+ * Open `root` as the claim's directory. With an `fstype` (default `fuse.archil`), the mount table must list a mount of
115
+ * that type at `root` on the device of the directory just opened; otherwise, and when the root cannot be opened, a
116
+ * FencedError CLAIM_UNMOUNTED (exit 75).
117
+ */
118
+ export declare function openClaimDir(root: string, options?: OpenClaimDirOptions): Promise<ClaimDir>;
119
+ /**
120
+ * This process as a holder: the host driver's `handle` (default driver `local`, this machine's host name and boot id)
121
+ * with this process's pid and `since`. A handle whose named fields have the wrong type is refused (RunRecordError).
122
+ */
123
+ export declare function currentHolder(handle?: Readonly<Record<string, JsonValue>>, since?: Date): RunHolder;