@parcha/pi-durable-disk 0.1.0-beta.14 → 0.1.0-beta.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +211 -0
- package/README.md +225 -15
- package/dist/app.d.ts +5 -1
- package/dist/claim.d.ts +45 -2
- package/dist/claim.js +145 -15
- package/dist/cli.js +16 -15
- package/dist/errors.d.ts +2 -1
- package/dist/hosts/attempt-env.d.ts +3 -0
- package/dist/hosts/attempt-env.js +15 -0
- package/dist/hosts/daytona-launch.js +8 -7
- package/dist/hosts/daytona.d.ts +27 -2
- package/dist/hosts/daytona.js +92 -3
- package/dist/hosts/docker.d.ts +2 -2
- package/dist/hosts/docker.js +22 -5
- package/dist/hosts/local-host.d.ts +2 -2
- package/dist/hosts/local-host.js +34 -6
- package/dist/index.d.ts +8 -4
- package/dist/index.js +6 -2
- package/dist/lease.d.ts +1 -1
- package/dist/place.d.ts +236 -0
- package/dist/place.js +859 -0
- package/dist/run.d.ts +51 -10
- package/dist/run.js +75 -33
- package/dist/status.d.ts +29 -0
- package/dist/status.js +89 -0
- package/dist/supervise.d.ts +44 -14
- package/dist/supervise.js +5 -18
- package/dist/with-claim.d.ts +63 -0
- package/dist/with-claim.js +325 -0
- package/package.json +3 -2
package/dist/run.d.ts
CHANGED
|
@@ -57,7 +57,21 @@ export interface LeaseOptions {
|
|
|
57
57
|
/** How often the self-fence timer looks; default 1 s. */
|
|
58
58
|
readonly checkMs?: number;
|
|
59
59
|
}
|
|
60
|
-
export type OpenStep = "acquire" | "owner-lock" | "
|
|
60
|
+
export type OpenStep = "acquire" | "owner-lock" | "layout" | "commands" | "after-claim" | "seed" | "run.json" | "heartbeat" | "store" | "seal" | "harness" | "live" | "before-resume" | "resume";
|
|
61
|
+
/** Where a new run's seed writes: the claim's root, the store file the Harness will open, and the workspace. */
|
|
62
|
+
export interface RunSeedDir {
|
|
63
|
+
readonly root: string;
|
|
64
|
+
readonly storePath: string;
|
|
65
|
+
readonly workDir: string;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* What a new run starts with, written before anything can see the run: its store file (`storePath`, a SQLite database
|
|
69
|
+
* the Harness then opens) and files in its workspace. It runs inside the run's first claim, holding the owner lock, after
|
|
70
|
+
* `store/`, `work/` and `tmp/` exist and `afterClaim` has run, and before `run.json` publishes the run. It never runs on a
|
|
71
|
+
* later open. If it throws (or its process dies) the open fails and nothing is published, so the next open seeds again: it
|
|
72
|
+
* must overwrite what an earlier attempt wrote. A connection it opens on `storePath` must be closed before it returns.
|
|
73
|
+
*/
|
|
74
|
+
export type RunSeed = (dir: RunSeedDir) => void | Promise<void>;
|
|
61
75
|
export type ReleaseStep = "close" | "cleanup" | "barrier" | "seal" | "unlock" | "unmount";
|
|
62
76
|
export interface OpenRunLeaseOptions {
|
|
63
77
|
/**
|
|
@@ -104,12 +118,30 @@ export interface OpenRunLeaseOptions {
|
|
|
104
118
|
readonly claimDir?: (root: string) => Promise<ClaimDir>;
|
|
105
119
|
/** Replaceable for tests: the monotonic clock the self-fence measures with, in ms (default `performance.now`). */
|
|
106
120
|
readonly clock?: () => number;
|
|
121
|
+
/**
|
|
122
|
+
* A new run's starting state (`RunSeed`): run once, on the open that finds no `run.json`, before `run.json` is written
|
|
123
|
+
* and the store opened; what it wrote is made durable (the claim's barrier) before the run is published.
|
|
124
|
+
*/
|
|
125
|
+
readonly seed?: RunSeed;
|
|
107
126
|
}
|
|
108
127
|
export interface OpenDurableRunOptions<Tool extends ToolRegistration = ToolRegistration> extends Omit<OpenRunLeaseOptions, "cleanup"> {
|
|
109
128
|
/** Default `exclusive`. */
|
|
110
129
|
readonly profile?: Profile;
|
|
111
|
-
/**
|
|
112
|
-
|
|
130
|
+
/**
|
|
131
|
+
* pi's Harness options; the registry holds every extension before the Harness opens (recovered tasks need them).
|
|
132
|
+
* Required, here or from `afterClaim`.
|
|
133
|
+
*/
|
|
134
|
+
readonly harness?: Omit<HarnessOptions<Tool>, "env">;
|
|
135
|
+
/**
|
|
136
|
+
* Awaited once the claim is held (the run's directory mounted, its owner lock taken, `store/`, `work/` and `tmp/` made,
|
|
137
|
+
* what a killed incarnation's commands left running reaped) and before this incarnation's `run.json`, with the claim's
|
|
138
|
+
* paths: what it returns (`harness`, `beforeResume`, a new run's `seed`) goes into the open, over what these options
|
|
139
|
+
* give; the seed runs right after it, on a new run only. For code that
|
|
140
|
+
* must run on the live directory, as an app module's factory does under the CLI. It runs before the heartbeat, inside a
|
|
141
|
+
* supervisor's start grace. A rejection fails the open with `run.json` as the open found it (none on a new run), so a
|
|
142
|
+
* supervisor counts a failed start and backs off; the claim is released.
|
|
143
|
+
*/
|
|
144
|
+
readonly afterClaim?: (claim: Pick<Claim, "ref" | "root" | "work" | "store">) => Promise<AfterClaim<Tool>>;
|
|
113
145
|
/** The run's environment, built on the claim; default `archilEnv(claim)`. Its `cleanup` is the lease's. */
|
|
114
146
|
readonly env?: (claim: Claim) => RunEnv;
|
|
115
147
|
/** Wraps the raw database inside the store's fence (instrumentation, fault injection). */
|
|
@@ -122,6 +154,13 @@ export interface OpenDurableRunOptions<Tool extends ToolRegistration = ToolRegis
|
|
|
122
154
|
*/
|
|
123
155
|
readonly beforeResume?: (run: BeforeResume) => void | Promise<void>;
|
|
124
156
|
}
|
|
157
|
+
/** What `afterClaim` gives the open: the Harness options, the `beforeResume` hook and a new run's seed, when it holds them. */
|
|
158
|
+
export interface AfterClaim<Tool extends ToolRegistration = ToolRegistration> {
|
|
159
|
+
readonly harness?: Omit<HarnessOptions<Tool>, "env">;
|
|
160
|
+
readonly beforeResume?: (run: BeforeResume) => void | Promise<void>;
|
|
161
|
+
/** A new run's seed, over the options' `seed`; ignored on an open that finds `run.json` (a resume). */
|
|
162
|
+
readonly seed?: RunSeed;
|
|
163
|
+
}
|
|
125
164
|
/**
|
|
126
165
|
* What `beforeResume` sees: the open Harness, not yet resumed, the open's context, the incarnation it belongs to, and the
|
|
127
166
|
* store the Harness was opened over with its head (a recorder snapshots every document at `head` here).
|
|
@@ -164,9 +203,10 @@ export interface RunClaim {
|
|
|
164
203
|
}
|
|
165
204
|
/**
|
|
166
205
|
* The claim's lifecycle for an app that owns its store and Harness (`openDurableRun` without the store and the Harness). Open
|
|
167
|
-
* order, all done by `openRunLease`: acquire and verify the claim, take the owner lock,
|
|
168
|
-
*
|
|
169
|
-
* a
|
|
206
|
+
* order, all done by `openRunLease`: acquire and verify the claim, take the owner lock, create `store/`, `work/` and
|
|
207
|
+
* `tmp/`, reap what a killed incarnation's commands left running, seed a new run (`seed`), write `run.json` (`running`, generation + 1, this
|
|
208
|
+
* holder, a heartbeat, the previous seal kept), start the heartbeat and the self-fence (a timer and a watchdog thread).
|
|
209
|
+
* The app then opens its store, calls `checkSeal` before
|
|
170
210
|
* anything commits, opens its Harness over `observe(storage)`, and calls `live()`. Once fenced, every member that
|
|
171
211
|
* writes rejects with the first fence; nothing is retried.
|
|
172
212
|
*/
|
|
@@ -282,15 +322,16 @@ export interface StoreHeadSource {
|
|
|
282
322
|
*/
|
|
283
323
|
export declare function storeHead(store: StoreHeadSource): Promise<number>;
|
|
284
324
|
/**
|
|
285
|
-
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock,
|
|
286
|
-
* `
|
|
325
|
+
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock, create
|
|
326
|
+
* `store/`, `work/` and `tmp/`, reap a killed incarnation's commands, seed a new run, write `run.json` and start the
|
|
327
|
+
* heartbeat and the self-fence. Errors are typed as for
|
|
287
328
|
* `openDurableRun`: HeldError (76), FencedError (75, also passed to `onFenced`), RunRecordError (65), RunError
|
|
288
329
|
* INVALID_ARGUMENT before any claim.
|
|
289
330
|
*/
|
|
290
331
|
export declare function openRunLease(ref: RunRef, options: OpenRunLeaseOptions): Promise<RunLease>;
|
|
291
332
|
/**
|
|
292
|
-
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock,
|
|
293
|
-
* self-fence
|
|
333
|
+
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock, layout, the commands reap, then
|
|
334
|
+
* `afterClaim`, a new run's seed, then `run.json`, heartbeat and self-fence), then the store, the seal check before anything commits, pi's Harness with the app's registry,
|
|
294
335
|
* the seal cleared in `run.json`, resume.
|
|
295
336
|
*
|
|
296
337
|
* Errors are typed: HeldError (76) when another client, another process on this host or another connection holds the
|
package/dist/run.js
CHANGED
|
@@ -15,7 +15,7 @@ import { archilEnv } from "./env.js";
|
|
|
15
15
|
import { EXIT_FENCED, FencedError, HeldError, ownWriteFenced, PdaError } from "./errors.js";
|
|
16
16
|
import { openArchilStore } from "./store.js";
|
|
17
17
|
import { LeaseWatchdog } from "./watchdog.js";
|
|
18
|
-
import { currentHolder, DEFAULT_HEARTBEAT_MS, DEFAULT_LEASE_EXPIRY_MS, DEFAULT_LEASE_MARGIN_MS, openClaimDir, persistRecord, readRunRecord, RUN_JSON, RunRecordError, writeRunRecord, } from "./status.js";
|
|
18
|
+
import { currentHolder, DEFAULT_HEARTBEAT_MS, DEFAULT_LEASE_EXPIRY_MS, DEFAULT_LEASE_MARGIN_MS, keepStartMark, openClaimDir, persistRecord, readRunRecord, RUN_JSON, RunRecordError, writeRunRecord, } from "./status.js";
|
|
19
19
|
/** The same-host owner lock under the run's root. */
|
|
20
20
|
export const OWNER_LOCK = "owner.lock";
|
|
21
21
|
/** pi's store under the claim's `store/` directory. */
|
|
@@ -311,12 +311,15 @@ class Lease {
|
|
|
311
311
|
return this.#fence?.done ?? Promise.resolve();
|
|
312
312
|
}
|
|
313
313
|
/**
|
|
314
|
-
* The order after the claim
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
314
|
+
* The order after the claim: the owner lock, the layout, the reap of a killed incarnation's commands, `afterClaim`
|
|
315
|
+
* (openDurableRun's), a new run's seed (afterClaim's over the options'), then `run.json` (`running`, generation + 1,
|
|
316
|
+
* this holder, a heartbeat) and the heartbeat, before
|
|
317
|
+
* the store and the Harness open. Nothing is published before that write: an open that fails earlier leaves `run.json`
|
|
318
|
+
* as it found it, so a supervisor counts a failed start and backs off; its start grace covers the steps before it.
|
|
319
|
+
* Once written, a supervisor never sees the previous incarnation's stale heartbeat on the live claim. That first write
|
|
320
|
+
* keeps the previous `sealedSeq`, which the seal check still needs; `live()` clears it once the store is open.
|
|
318
321
|
*/
|
|
319
|
-
async open(step, holder) {
|
|
322
|
+
async open(step, holder, afterClaim) {
|
|
320
323
|
const claim = this.rawClaim;
|
|
321
324
|
// SQLite resolves symbolic links in a file name, `/proc/self/fd/<n>` included, so `owner.lock` is the one file opened
|
|
322
325
|
// by path: right after the mount check, and the file it names must then be the one in the claim directory.
|
|
@@ -341,6 +344,34 @@ class Lease {
|
|
|
341
344
|
const previous = await this.#readPrevious();
|
|
342
345
|
this.#previous = previous;
|
|
343
346
|
this.#lastSeq = previous?.sealedSeq ?? 0;
|
|
347
|
+
await step("layout", () => this.#layout());
|
|
348
|
+
// A claim that records its commands (a local run's): end what a killed incarnation left running, before any code of
|
|
349
|
+
// this incarnation runs, so no command of it still writes in work/ once the run resumes. A process of it that outlives
|
|
350
|
+
// its SIGKILL refuses the open (HeldError, 76): the run never resumes beside it, and its record stays for the next open.
|
|
351
|
+
const commands = claim.commands;
|
|
352
|
+
if (commands !== undefined)
|
|
353
|
+
await step("commands", () => reapOrRefuse(commands));
|
|
354
|
+
const given = afterClaim ? await step("after-claim", () => afterClaim(claim)) : undefined;
|
|
355
|
+
// A new run (no run.json): its seed writes what it starts with, and only then is it published. afterClaim's seed
|
|
356
|
+
// goes over the options'.
|
|
357
|
+
const seed = given?.seed ?? this.#options.seed;
|
|
358
|
+
if (seed !== undefined && previous === undefined) {
|
|
359
|
+
await step("seed", async () => {
|
|
360
|
+
await this.#dir.assertMounted();
|
|
361
|
+
// While a new run seeds it has no `run.json` heartbeat, and a supervisor keeps a start in its grace only while its
|
|
362
|
+
// start mark is younger than the grace: the mark is kept fresh until the seed ends, through the held claim directory.
|
|
363
|
+
// A seeder that dies stops, and the grace runs out as for any start.
|
|
364
|
+
const stop = keepStartMark(this.#dir.at, this.#lease.heartbeatMs);
|
|
365
|
+
try {
|
|
366
|
+
await seed({ root: claim.root, storePath: join(claim.store, STORE_FILE), workDir: claim.work });
|
|
367
|
+
}
|
|
368
|
+
finally {
|
|
369
|
+
await stop();
|
|
370
|
+
}
|
|
371
|
+
await this.#dir.assertMounted();
|
|
372
|
+
await claim.barrier();
|
|
373
|
+
});
|
|
374
|
+
}
|
|
344
375
|
await step("run.json", async () => {
|
|
345
376
|
const now = new Date().toISOString();
|
|
346
377
|
this.#noteBeat(await this.#write(() => ({
|
|
@@ -366,19 +397,21 @@ class Lease {
|
|
|
366
397
|
ownCgroup: this.#options.ownCgroup === true,
|
|
367
398
|
});
|
|
368
399
|
});
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
}
|
|
400
|
+
}
|
|
401
|
+
/** `store/`, `work/` and `tmp/`, made through the held claim directory. */
|
|
402
|
+
async #layout() {
|
|
403
|
+
const claim = this.rawClaim;
|
|
404
|
+
for (const dir of [claim.store, claim.work, join(claim.root, "tmp")]) {
|
|
405
|
+
const under = relative(claim.root, dir);
|
|
406
|
+
if (under === "" || under.startsWith("..") || isAbsolute(under))
|
|
407
|
+
throw new RunError("INVALID_ARGUMENT", `${dir} is not under the claim root ${claim.root}`);
|
|
408
|
+
try {
|
|
409
|
+
await mkdir(join(this.#dir.at, under), { recursive: true });
|
|
380
410
|
}
|
|
381
|
-
|
|
411
|
+
catch (error) {
|
|
412
|
+
throw ownWriteFenced(dir, error);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
382
415
|
}
|
|
383
416
|
async checkSeal(store) {
|
|
384
417
|
const sealed = this.sealedSeq;
|
|
@@ -611,7 +644,7 @@ class Lease {
|
|
|
611
644
|
throw new RunError("RUN_RELEASED", `run ${this.ref.id} is released`);
|
|
612
645
|
}
|
|
613
646
|
}
|
|
614
|
-
async function openLease(ref, options, step) {
|
|
647
|
+
async function openLease(ref, options, step, afterClaim) {
|
|
615
648
|
const context = options.context ?? BACKGROUND_CONTEXT;
|
|
616
649
|
leaseOf(options.lease);
|
|
617
650
|
let holder;
|
|
@@ -633,7 +666,7 @@ async function openLease(ref, options, step) {
|
|
|
633
666
|
}));
|
|
634
667
|
const lease = new Lease(ref, claim, options, context);
|
|
635
668
|
try {
|
|
636
|
-
await lease.open(step, holder);
|
|
669
|
+
await lease.open(step, holder, afterClaim);
|
|
637
670
|
}
|
|
638
671
|
catch (error) {
|
|
639
672
|
await lease.abandon(error);
|
|
@@ -642,8 +675,9 @@ async function openLease(ref, options, step) {
|
|
|
642
675
|
return lease;
|
|
643
676
|
}
|
|
644
677
|
/**
|
|
645
|
-
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock,
|
|
646
|
-
* `
|
|
678
|
+
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock, create
|
|
679
|
+
* `store/`, `work/` and `tmp/`, reap a killed incarnation's commands, seed a new run, write `run.json` and start the
|
|
680
|
+
* heartbeat and the self-fence. Errors are typed as for
|
|
647
681
|
* `openDurableRun`: HeldError (76), FencedError (75, also passed to `onFenced`), RunRecordError (65), RunError
|
|
648
682
|
* INVALID_ARGUMENT before any claim.
|
|
649
683
|
*/
|
|
@@ -697,6 +731,7 @@ class Run {
|
|
|
697
731
|
// Both refusals here are terminal: the run is marked `failed`, keeping its seal, so the supervisor stops on it.
|
|
698
732
|
await step("seal", () => lease.checkSeal(this.#store));
|
|
699
733
|
const storage = lease.observe(this.#store.storage);
|
|
734
|
+
// openDurableRun refused an open without Harness options (here or from afterClaim) before this runs.
|
|
700
735
|
this.#harness = await step("harness", () => Harness.open(storage, { ...this.#options.harness, env: this.env }, this.#context));
|
|
701
736
|
await step("live", () => lease.live());
|
|
702
737
|
const before = this.#options.beforeResume;
|
|
@@ -749,8 +784,8 @@ class Run {
|
|
|
749
784
|
}
|
|
750
785
|
}
|
|
751
786
|
/**
|
|
752
|
-
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock,
|
|
753
|
-
* self-fence
|
|
787
|
+
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock, layout, the commands reap, then
|
|
788
|
+
* `afterClaim`, a new run's seed, then `run.json`, heartbeat and self-fence), then the store, the seal check before anything commits, pi's Harness with the app's registry,
|
|
754
789
|
* the seal cleared in `run.json`, resume.
|
|
755
790
|
*
|
|
756
791
|
* Errors are typed: HeldError (76) when another client, another process on this host or another connection holds the
|
|
@@ -758,25 +793,32 @@ class Run {
|
|
|
758
793
|
* RunError STORE_HEAD_UNREADABLE (70), each after marking the run `failed` and releasing the claim.
|
|
759
794
|
*/
|
|
760
795
|
export async function openDurableRun(ref, options) {
|
|
796
|
+
if (!options.harness && !options.afterClaim)
|
|
797
|
+
throw new RunError("INVALID_ARGUMENT", "openDurableRun needs pi's Harness options, from its options or from afterClaim");
|
|
761
798
|
const context = options.context ?? BACKGROUND_CONTEXT;
|
|
762
799
|
const step = stepper(options.onStep);
|
|
763
800
|
let env;
|
|
764
|
-
|
|
801
|
+
let opening = options;
|
|
802
|
+
const given = options.afterClaim;
|
|
803
|
+
// Inside the lease's open, before run.json: a factory that fails publishes nothing of this incarnation.
|
|
804
|
+
const afterClaim = given &&
|
|
805
|
+
(async ({ ref: claimRef, root, work, store }) => {
|
|
806
|
+
const got = await given({ ref: claimRef, root, work, store });
|
|
807
|
+
opening = { ...options, ...(got.harness ? { harness: got.harness } : {}), ...(got.beforeResume ? { beforeResume: got.beforeResume } : {}) };
|
|
808
|
+
if (!opening.harness)
|
|
809
|
+
throw new RunError("INVALID_ARGUMENT", "openDurableRun needs pi's Harness options: afterClaim returned none, and its options give none");
|
|
810
|
+
return got.seed ? { seed: got.seed } : {};
|
|
811
|
+
});
|
|
812
|
+
// The lease's cleanup is the environment's: abandoning the lease cleans both.
|
|
813
|
+
const lease = await openLease(ref, { ...options, context, cleanup: (c) => env?.cleanup(c) ?? Promise.resolve() }, step, afterClaim);
|
|
765
814
|
try {
|
|
766
|
-
// A claim that records its commands (a local run's): end what a killed incarnation left running, now that the owner
|
|
767
|
-
// lock is held and before the Harness opens, so no command of it still writes in work/ once the run resumes.
|
|
768
|
-
const commands = lease.rawClaim.commands;
|
|
769
|
-
// A process of a killed incarnation that outlives its SIGKILL refuses the open (HeldError, 76): the run never resumes
|
|
770
|
-
// beside it, and its record stays for the next open.
|
|
771
|
-
if (commands !== undefined)
|
|
772
|
-
await step("commands", () => reapOrRefuse(commands));
|
|
773
815
|
env = (options.env ?? ((c) => archilEnv(c)))(lease.rawClaim);
|
|
774
816
|
}
|
|
775
817
|
catch (error) {
|
|
776
818
|
await lease.abandon(error);
|
|
777
819
|
throw error;
|
|
778
820
|
}
|
|
779
|
-
const run = new Run(lease, env,
|
|
821
|
+
const run = new Run(lease, env, opening, context);
|
|
780
822
|
try {
|
|
781
823
|
await run.open(step);
|
|
782
824
|
}
|
package/dist/status.d.ts
CHANGED
|
@@ -55,6 +55,14 @@ export declare class RunRecordError extends PdaError {
|
|
|
55
55
|
export declare function parseRunRecord(text: string): RunRecord;
|
|
56
56
|
/** The `run.json` under a run's root on a local mount (or in a claim directory), or undefined when the run has none yet. */
|
|
57
57
|
export declare function readRunRecord(root: string | ClaimDir): Promise<RunRecord | undefined>;
|
|
58
|
+
/**
|
|
59
|
+
* The status of a run on a local directory (`localRun`): `<root>/run.json`, where `root` is the directory `localRun(root)`
|
|
60
|
+
* takes (not its `work/`). Read as `readRunStatus` reads a disk's through the control API: the same strict parser and the
|
|
61
|
+
* same `RunRecord`. It only reads: no lock, no mount and no store are touched, so it is safe beside a live instance, whose
|
|
62
|
+
* next write replaces the file whole. A run with no `run.json` yet, or no directory, is `null`; a record that is malformed
|
|
63
|
+
* is `RunRecordError` (`RUN_JSON_INVALID`); any other read error is thrown.
|
|
64
|
+
*/
|
|
65
|
+
export declare function readLocalRunStatus(root: string): Promise<RunRecord | null>;
|
|
58
66
|
/**
|
|
59
67
|
* True while the record's last heartbeat is younger than `expiryMs` on the reader's clock. The supervisor allows for
|
|
60
68
|
* clock skew between hosts inside the expiry.
|
|
@@ -73,6 +81,27 @@ export declare const persistRecord: PersistRecord;
|
|
|
73
81
|
* `PersistRecord`.
|
|
74
82
|
*/
|
|
75
83
|
export declare function writeRunRecord(root: string | ClaimDir, record: RunRecord, persist?: PersistRecord, signal?: AbortSignal): Promise<void>;
|
|
84
|
+
/**
|
|
85
|
+
* The supervisor's record of its last start of a run, in the run's directory: the generation the instance will write,
|
|
86
|
+
* when, by whom, and the failed starts right before it (`failures`; `lastExit`, what the decision saw of the last one).
|
|
87
|
+
*/
|
|
88
|
+
export declare const START_MARK = "start.json";
|
|
89
|
+
export type StartMark = {
|
|
90
|
+
generation: number;
|
|
91
|
+
at: string;
|
|
92
|
+
by: string | null;
|
|
93
|
+
failures: number;
|
|
94
|
+
lastExit: string | null;
|
|
95
|
+
};
|
|
96
|
+
/** A start mark's text as a mark, or null when it is not one (an unreadable mark gives no grace). */
|
|
97
|
+
export declare function parseStartMark(text: string): StartMark | null;
|
|
98
|
+
/**
|
|
99
|
+
* Keeps a supervisor's start mark for generation 1 in `dir` fresh: its `at` rewritten every `everyMs` (temp file, fsync,
|
|
100
|
+
* rename, fsync the directory), the rest of it as the supervisor wrote it; nothing is made when there is none. On a thread of
|
|
101
|
+
* its own, so a caller whose thread is blocked (a synchronous import) is kept fresh too. Best effort: a refused write leaves
|
|
102
|
+
* the mark as it was. Returns the stop, which resolves once the thread has exited, so no write outlives it.
|
|
103
|
+
*/
|
|
104
|
+
export declare function keepStartMark(dir: string, everyMs: number): () => Promise<void>;
|
|
76
105
|
/** The FencedError code of a claim root that is no longer the mount the instance opened (unmounted, lazily or not). */
|
|
77
106
|
export declare const CLAIM_UNMOUNTED = "CLAIM_UNMOUNTED";
|
|
78
107
|
/** The file system type of an Archil mount in the mount table. */
|
package/dist/status.js
CHANGED
|
@@ -6,6 +6,7 @@ import { constants } from "node:fs";
|
|
|
6
6
|
import { open, readFile, rename, rm, stat } from "node:fs/promises";
|
|
7
7
|
import { hostname } from "node:os";
|
|
8
8
|
import { join } from "node:path";
|
|
9
|
+
import { Worker } from "node:worker_threads";
|
|
9
10
|
import { FencedError, ownWriteFenced, PdaError } from "./errors.js";
|
|
10
11
|
import { thisBoot } from "./process-table.js";
|
|
11
12
|
export const RUN_JSON = "run.json";
|
|
@@ -96,6 +97,16 @@ export async function readRunRecord(root) {
|
|
|
96
97
|
}
|
|
97
98
|
return parseRunRecord(text);
|
|
98
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* The status of a run on a local directory (`localRun`): `<root>/run.json`, where `root` is the directory `localRun(root)`
|
|
102
|
+
* takes (not its `work/`). Read as `readRunStatus` reads a disk's through the control API: the same strict parser and the
|
|
103
|
+
* same `RunRecord`. It only reads: no lock, no mount and no store are touched, so it is safe beside a live instance, whose
|
|
104
|
+
* next write replaces the file whole. A run with no `run.json` yet, or no directory, is `null`; a record that is malformed
|
|
105
|
+
* is `RunRecordError` (`RUN_JSON_INVALID`); any other read error is thrown.
|
|
106
|
+
*/
|
|
107
|
+
export async function readLocalRunStatus(root) {
|
|
108
|
+
return (await readRunRecord(root)) ?? null;
|
|
109
|
+
}
|
|
99
110
|
/**
|
|
100
111
|
* True while the record's last heartbeat is younger than `expiryMs` on the reader's clock. The supervisor allows for
|
|
101
112
|
* clock skew between hosts inside the expiry.
|
|
@@ -148,6 +159,84 @@ export async function writeRunRecord(root, record, persist = persistRecord, sign
|
|
|
148
159
|
throw ownWriteFenced(join(typeof root === "string" ? root : root.root, RUN_JSON), error);
|
|
149
160
|
}
|
|
150
161
|
}
|
|
162
|
+
// ---- the supervisor's start mark --------------------------------------------------------------------------------------
|
|
163
|
+
/**
|
|
164
|
+
* The supervisor's record of its last start of a run, in the run's directory: the generation the instance will write,
|
|
165
|
+
* when, by whom, and the failed starts right before it (`failures`; `lastExit`, what the decision saw of the last one).
|
|
166
|
+
*/
|
|
167
|
+
export const START_MARK = "start.json";
|
|
168
|
+
/** A start mark's text as a mark, or null when it is not one (an unreadable mark gives no grace). */
|
|
169
|
+
export function parseStartMark(text) {
|
|
170
|
+
try {
|
|
171
|
+
const v = JSON.parse(text);
|
|
172
|
+
if (!Number.isSafeInteger(v.generation) || v.generation < 1 || typeof v.at !== "string" || !Number.isFinite(Date.parse(v.at)))
|
|
173
|
+
return null;
|
|
174
|
+
const failures = Number.isSafeInteger(v.failures) && v.failures >= 0 ? v.failures : 0;
|
|
175
|
+
const lastExit = typeof v.lastExit === "string" ? v.lastExit : null;
|
|
176
|
+
return { generation: v.generation, at: v.at, by: typeof v.by === "string" ? v.by : null, failures, lastExit };
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
// The keeper's code is plain CommonJS in a string, started with `eval: true` and no inherited Node flags, as the lease
|
|
183
|
+
// watchdog's is (src/watchdog.ts says why). One refresh at a time, all synchronous: a stop (the message) is handled only
|
|
184
|
+
// between two refreshes, so none is still writing once the thread has exited.
|
|
185
|
+
const KEEPER_SOURCE = `"use strict";
|
|
186
|
+
const fs = require("node:fs");
|
|
187
|
+
const path = require("node:path");
|
|
188
|
+
const { parentPort, workerData } = require("node:worker_threads");
|
|
189
|
+
const { dir, name, everyMs } = workerData;
|
|
190
|
+
const file = path.join(dir, name);
|
|
191
|
+
function refresh() {
|
|
192
|
+
try {
|
|
193
|
+
const mark = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
194
|
+
if (mark === null || typeof mark !== "object" || mark.generation !== 1 || typeof mark.at !== "string" || !Number.isFinite(Date.parse(mark.at))) return;
|
|
195
|
+
const temp = file + ".tmp";
|
|
196
|
+
const fd = fs.openSync(temp, "w", 0o644);
|
|
197
|
+
try {
|
|
198
|
+
fs.writeSync(fd, JSON.stringify({ ...mark, at: new Date().toISOString() }));
|
|
199
|
+
fs.fsyncSync(fd);
|
|
200
|
+
} finally {
|
|
201
|
+
fs.closeSync(fd);
|
|
202
|
+
}
|
|
203
|
+
fs.renameSync(temp, file);
|
|
204
|
+
const handle = fs.openSync(dir, "r");
|
|
205
|
+
try {
|
|
206
|
+
fs.fsyncSync(handle);
|
|
207
|
+
} finally {
|
|
208
|
+
fs.closeSync(handle);
|
|
209
|
+
}
|
|
210
|
+
} catch {
|
|
211
|
+
// absent, unreadable or refused
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
refresh();
|
|
215
|
+
const timer = setInterval(refresh, everyMs);
|
|
216
|
+
parentPort.once("message", () => {
|
|
217
|
+
clearInterval(timer);
|
|
218
|
+
parentPort.close();
|
|
219
|
+
});
|
|
220
|
+
`;
|
|
221
|
+
/**
|
|
222
|
+
* Keeps a supervisor's start mark for generation 1 in `dir` fresh: its `at` rewritten every `everyMs` (temp file, fsync,
|
|
223
|
+
* rename, fsync the directory), the rest of it as the supervisor wrote it; nothing is made when there is none. On a thread of
|
|
224
|
+
* its own, so a caller whose thread is blocked (a synchronous import) is kept fresh too. Best effort: a refused write leaves
|
|
225
|
+
* the mark as it was. Returns the stop, which resolves once the thread has exited, so no write outlives it.
|
|
226
|
+
*/
|
|
227
|
+
export function keepStartMark(dir, everyMs) {
|
|
228
|
+
const worker = new Worker(KEEPER_SOURCE, { eval: true, workerData: { dir, name: START_MARK, everyMs }, execArgv: [] });
|
|
229
|
+
worker.unref();
|
|
230
|
+
worker.on("error", () => {
|
|
231
|
+
// the thread failed to start or threw: nothing more is refreshed, and the grace runs out as for any start
|
|
232
|
+
});
|
|
233
|
+
const exited = new Promise((resolve) => worker.once("exit", () => resolve()));
|
|
234
|
+
return () => {
|
|
235
|
+
worker.ref();
|
|
236
|
+
worker.postMessage("stop");
|
|
237
|
+
return exited;
|
|
238
|
+
};
|
|
239
|
+
}
|
|
151
240
|
// ---- the claim directory ----------------------------------------------------------------------------------------------
|
|
152
241
|
/** The FencedError code of a claim root that is no longer the mount the instance opened (unmounted, lazily or not). */
|
|
153
242
|
export const CLAIM_UNMOUNTED = "CLAIM_UNMOUNTED";
|
package/dist/supervise.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Delegation } from "disk";
|
|
2
2
|
import { type ArchilHost, type ControlApi, type PathlessResolver, type RunRef } from "./claim.js";
|
|
3
3
|
import { PdaError } from "./errors.js";
|
|
4
|
-
import { type RunRecord } from "./status.js";
|
|
4
|
+
import { type RunRecord, type StartMark } from "./status.js";
|
|
5
5
|
export type Json = null | boolean | number | string | Json[] | {
|
|
6
6
|
[key: string]: Json;
|
|
7
7
|
};
|
|
@@ -12,9 +12,30 @@ export type HostHandle = {
|
|
|
12
12
|
driver: string;
|
|
13
13
|
[key: string]: Json;
|
|
14
14
|
};
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* Which start this is: the generation the instance will open (run.json's generation + 1), also in the token's nickname.
|
|
17
|
+
* `env`: what this start adds to the instance's environment (a place's per-attempt order), merged over the driver's own
|
|
18
|
+
* `env`. PATH and PDA_HOLDER are the driver's: a start that sets either, or a name that is not a variable name, is refused.
|
|
19
|
+
*/
|
|
16
20
|
export type StartAttempt = {
|
|
17
21
|
attempt: number;
|
|
22
|
+
env?: Record<string, string>;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* A box a driver made for an instance, as a receipt: who provided it, what kind, when it was made and when it ended
|
|
26
|
+
* (null while it lives), and its published list price per second (`usdPerSecond`), with the page that price was read
|
|
27
|
+
* from (`source`) and the ISO date it was read (`priced`). A box with no price (a kind the provider's page does not
|
|
28
|
+
* price, or of unknown size) has none of the three: never a price of 0. No field holds the provider's id of the box:
|
|
29
|
+
* the supervisor labels boxes itself and computes their seconds and cost.
|
|
30
|
+
*/
|
|
31
|
+
export type HostBox = {
|
|
32
|
+
provider: string;
|
|
33
|
+
kind: string;
|
|
34
|
+
from: string;
|
|
35
|
+
to: string | null;
|
|
36
|
+
usdPerSecond?: number;
|
|
37
|
+
source?: string;
|
|
38
|
+
priced?: string;
|
|
18
39
|
};
|
|
19
40
|
/** Compute is disposable and pluggable: a driver is these three calls. */
|
|
20
41
|
export interface HostDriver {
|
|
@@ -28,6 +49,16 @@ export interface HostDriver {
|
|
|
28
49
|
status(handle: HostHandle): Promise<HostStatus>;
|
|
29
50
|
/** Stop the instance and whatever it left on its host. A handle the driver cannot reach is a no-op. */
|
|
30
51
|
stop(handle: HostHandle): Promise<void>;
|
|
52
|
+
/**
|
|
53
|
+
* Optional: take the instance away abruptly, as a crash would. No drain, no release, nothing cleaned: the run's claim is
|
|
54
|
+
* left as a crash leaves it (orphaned with its box, or held by a live mount until its lease lapses), and the
|
|
55
|
+
* supervisor's next pass decides what follows. A handle the driver cannot reach is a no-op.
|
|
56
|
+
*/
|
|
57
|
+
kill?(handle: HostHandle): Promise<void>;
|
|
58
|
+
/** What `kill` takes: `process`, the instance's processes while its machine stays (a restart policy may bring them back), or `box`, its machine. */
|
|
59
|
+
readonly kills?: "process" | "box";
|
|
60
|
+
/** Optional, for a driver whose instances get machines of their own: the box `handle`'s instance runs in, as a receipt. */
|
|
61
|
+
box?(handle: HostHandle): HostBox | null;
|
|
31
62
|
}
|
|
32
63
|
/** The control-plane calls the supervisor uses; an archil `Disk` satisfies it. */
|
|
33
64
|
export interface SupervisorControl extends ControlApi {
|
|
@@ -73,6 +104,16 @@ export interface EnsureOptions {
|
|
|
73
104
|
tokenTtl?: string;
|
|
74
105
|
/** Prefix of each mount token's nickname. */
|
|
75
106
|
tokenPrefix?: string;
|
|
107
|
+
/**
|
|
108
|
+
* A mount token the caller minted for this start (`mintMountToken`), for a caller that mints its own (a place's
|
|
109
|
+
* `token(attempt)`): a start uses it instead of minting one, and removes it as it would its own when the driver starts
|
|
110
|
+
* nothing or adopts. A decision that starts nothing leaves it to the caller.
|
|
111
|
+
*/
|
|
112
|
+
mountToken?: {
|
|
113
|
+
token: string;
|
|
114
|
+
identifier: string;
|
|
115
|
+
nickname: string;
|
|
116
|
+
};
|
|
76
117
|
/** The caller has work for the run now: start a paused run, or a sleeping one before its `wakeAt`. */
|
|
77
118
|
demand?: boolean;
|
|
78
119
|
/** Create `runs/<id>/` with this owner when it does not exist and nothing holds it. */
|
|
@@ -204,18 +245,7 @@ export type EnsureResult =
|
|
|
204
245
|
* supervisors are safe: the mount admits one claimant and the other instance exits 76.
|
|
205
246
|
*/
|
|
206
247
|
export declare function ensureRunning(ref: RunRef, host: HostDriver, options: EnsureOptions): Promise<EnsureResult>;
|
|
207
|
-
|
|
208
|
-
* The supervisor's record of its last start of a run: the generation the instance will write, when, by whom, and the
|
|
209
|
-
* failed starts right before it (`failures`; `lastExit`, what the decision saw of the last one).
|
|
210
|
-
*/
|
|
211
|
-
export declare const START_MARK = "start.json";
|
|
212
|
-
export type StartMark = {
|
|
213
|
-
generation: number;
|
|
214
|
-
at: string;
|
|
215
|
-
by: string | null;
|
|
216
|
-
failures: number;
|
|
217
|
-
lastExit: string | null;
|
|
218
|
-
};
|
|
248
|
+
export { START_MARK, type StartMark } from "./status.js";
|
|
219
249
|
/** The grace of a start that follows `failures` failed ones: the base doubled per failure, capped, never below the base. */
|
|
220
250
|
export declare function startGrace(baseMs: number, failures: number, maxMs?: number): number;
|
|
221
251
|
/**
|
package/dist/supervise.js
CHANGED
|
@@ -8,7 +8,7 @@ import { hostname } from "node:os";
|
|
|
8
8
|
import { join } from "node:path";
|
|
9
9
|
import { acquire, createRunDir, findDelegations, matchDelegations, pathlessResolver, revokeBestEffort, revokeCompanions, mintMountToken, MOUNT_TOKEN_TTL, parseTokenNickname, removeMountToken, runPath, unmountClaim, } from "./claim.js";
|
|
10
10
|
import { FencedError, HeldError, PdaError } from "./errors.js";
|
|
11
|
-
import { parseRunRecord, RUN_JSON } from "./status.js";
|
|
11
|
+
import { parseRunRecord, parseStartMark, RUN_JSON, START_MARK } from "./status.js";
|
|
12
12
|
export const CONTROL_TIMEOUT_MS = 10_000;
|
|
13
13
|
/** The same control API with every call bounded: one that does not settle in `ms` fails as CONTROL_API_FAILED. */
|
|
14
14
|
export function withTimeouts(control, ms) {
|
|
@@ -160,11 +160,8 @@ export async function ensureRunning(ref, host, options) {
|
|
|
160
160
|
await revokeListed(opts.control, ref.id, held, all);
|
|
161
161
|
return start("lease-expired", held, stonith);
|
|
162
162
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
* failed starts right before it (`failures`; `lastExit`, what the decision saw of the last one).
|
|
166
|
-
*/
|
|
167
|
-
export const START_MARK = "start.json";
|
|
163
|
+
// The start mark (`start.json`): its name, shape and parser live in status.ts, which the instance also reads it with.
|
|
164
|
+
export { START_MARK } from "./status.js";
|
|
168
165
|
/** The grace of a start that follows `failures` failed ones: the base doubled per failure, capped, never below the base. */
|
|
169
166
|
export function startGrace(baseMs, failures, maxMs = START_BACKOFF_MAX_MS) {
|
|
170
167
|
if (baseMs <= 0)
|
|
@@ -193,17 +190,7 @@ export async function readStartMark(control, id) {
|
|
|
193
190
|
return null;
|
|
194
191
|
throw new SuperviseError("CONTROL_API_FAILED", `GetObject ${runPath(id)}/${START_MARK} failed`, { cause: err });
|
|
195
192
|
}
|
|
196
|
-
|
|
197
|
-
const v = JSON.parse(text);
|
|
198
|
-
if (!Number.isSafeInteger(v.generation) || v.generation < 1 || typeof v.at !== "string" || !Number.isFinite(Date.parse(v.at)))
|
|
199
|
-
return null;
|
|
200
|
-
const failures = Number.isSafeInteger(v.failures) && v.failures >= 0 ? v.failures : 0;
|
|
201
|
-
const lastExit = typeof v.lastExit === "string" ? v.lastExit : null;
|
|
202
|
-
return { generation: v.generation, at: v.at, by: typeof v.by === "string" ? v.by : null, failures, lastExit };
|
|
203
|
-
}
|
|
204
|
-
catch {
|
|
205
|
-
return null;
|
|
206
|
-
}
|
|
193
|
+
return parseStartMark(text);
|
|
207
194
|
}
|
|
208
195
|
/** Only a run with no run.json and no holder gets here, so the directory is either absent or idle. */
|
|
209
196
|
async function ensureRunDir(control, id, owner) {
|
|
@@ -282,7 +269,7 @@ async function startInstance(ref, host, opts, why) {
|
|
|
282
269
|
let token;
|
|
283
270
|
try {
|
|
284
271
|
// The attempt is the generation the instance will write, so the token user names the incarnation it served.
|
|
285
|
-
token = await mintMountToken(opts.control, { run: { id: ref.id, attempt: why.generation + 1 }, prefix: opts.tokenPrefix, ttl: opts.tokenTtl ?? TOKEN_TTL, now: opts.now });
|
|
272
|
+
token = opts.mountToken ?? await mintMountToken(opts.control, { run: { id: ref.id, attempt: why.generation + 1 }, prefix: opts.tokenPrefix, ttl: opts.tokenTtl ?? TOKEN_TTL, now: opts.now });
|
|
286
273
|
}
|
|
287
274
|
catch (err) {
|
|
288
275
|
throw new SuperviseError("CONTROL_API_FAILED", `minting a mount token for ${runPath(ref.id)} failed`, { cause: err });
|