@parcha/pi-durable-disk 0.1.0-beta.14 → 0.1.0-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/README.md +181 -14
- package/dist/app.d.ts +5 -1
- package/dist/claim.d.ts +45 -2
- package/dist/claim.js +145 -15
- package/dist/cli.js +16 -15
- package/dist/errors.d.ts +2 -1
- package/dist/hosts/attempt-env.d.ts +3 -0
- package/dist/hosts/attempt-env.js +15 -0
- package/dist/hosts/daytona-launch.js +8 -7
- package/dist/hosts/daytona.d.ts +2 -2
- package/dist/hosts/daytona.js +33 -2
- package/dist/hosts/docker.d.ts +2 -2
- package/dist/hosts/docker.js +22 -5
- package/dist/hosts/local-host.d.ts +2 -2
- package/dist/hosts/local-host.js +34 -6
- package/dist/index.d.ts +6 -2
- package/dist/index.js +5 -1
- package/dist/lease.d.ts +1 -1
- package/dist/place.d.ts +190 -0
- package/dist/place.js +526 -0
- package/dist/run.d.ts +51 -10
- package/dist/run.js +65 -32
- package/dist/status.d.ts +8 -0
- package/dist/status.js +10 -0
- package/dist/supervise.d.ts +24 -1
- package/dist/supervise.js +1 -1
- package/dist/with-claim.d.ts +63 -0
- package/dist/with-claim.js +325 -0
- package/package.json +2 -1
package/dist/run.js
CHANGED
|
@@ -311,12 +311,15 @@ class Lease {
|
|
|
311
311
|
return this.#fence?.done ?? Promise.resolve();
|
|
312
312
|
}
|
|
313
313
|
/**
|
|
314
|
-
* The order after the claim
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
314
|
+
* The order after the claim: the owner lock, the layout, the reap of a killed incarnation's commands, `afterClaim`
|
|
315
|
+
* (openDurableRun's), a new run's seed (afterClaim's over the options'), then `run.json` (`running`, generation + 1,
|
|
316
|
+
* this holder, a heartbeat) and the heartbeat, before
|
|
317
|
+
* the store and the Harness open. Nothing is published before that write: an open that fails earlier leaves `run.json`
|
|
318
|
+
* as it found it, so a supervisor counts a failed start and backs off; its start grace covers the steps before it.
|
|
319
|
+
* Once written, a supervisor never sees the previous incarnation's stale heartbeat on the live claim. That first write
|
|
320
|
+
* keeps the previous `sealedSeq`, which the seal check still needs; `live()` clears it once the store is open.
|
|
318
321
|
*/
|
|
319
|
-
async open(step, holder) {
|
|
322
|
+
async open(step, holder, afterClaim) {
|
|
320
323
|
const claim = this.rawClaim;
|
|
321
324
|
// SQLite resolves symbolic links in a file name, `/proc/self/fd/<n>` included, so `owner.lock` is the one file opened
|
|
322
325
|
// by path: right after the mount check, and the file it names must then be the one in the claim directory.
|
|
@@ -341,6 +344,25 @@ class Lease {
|
|
|
341
344
|
const previous = await this.#readPrevious();
|
|
342
345
|
this.#previous = previous;
|
|
343
346
|
this.#lastSeq = previous?.sealedSeq ?? 0;
|
|
347
|
+
await step("layout", () => this.#layout());
|
|
348
|
+
// A claim that records its commands (a local run's): end what a killed incarnation left running, before any code of
|
|
349
|
+
// this incarnation runs, so no command of it still writes in work/ once the run resumes. A process of it that outlives
|
|
350
|
+
// its SIGKILL refuses the open (HeldError, 76): the run never resumes beside it, and its record stays for the next open.
|
|
351
|
+
const commands = claim.commands;
|
|
352
|
+
if (commands !== undefined)
|
|
353
|
+
await step("commands", () => reapOrRefuse(commands));
|
|
354
|
+
const given = afterClaim ? await step("after-claim", () => afterClaim(claim)) : undefined;
|
|
355
|
+
// A new run (no run.json): its seed writes what it starts with, and only then is it published. afterClaim's seed
|
|
356
|
+
// goes over the options'.
|
|
357
|
+
const seed = given?.seed ?? this.#options.seed;
|
|
358
|
+
if (seed !== undefined && previous === undefined) {
|
|
359
|
+
await step("seed", async () => {
|
|
360
|
+
await this.#dir.assertMounted();
|
|
361
|
+
await seed({ root: claim.root, storePath: join(claim.store, STORE_FILE), workDir: claim.work });
|
|
362
|
+
await this.#dir.assertMounted();
|
|
363
|
+
await claim.barrier();
|
|
364
|
+
});
|
|
365
|
+
}
|
|
344
366
|
await step("run.json", async () => {
|
|
345
367
|
const now = new Date().toISOString();
|
|
346
368
|
this.#noteBeat(await this.#write(() => ({
|
|
@@ -366,19 +388,21 @@ class Lease {
|
|
|
366
388
|
ownCgroup: this.#options.ownCgroup === true,
|
|
367
389
|
});
|
|
368
390
|
});
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
}
|
|
391
|
+
}
|
|
392
|
+
/** `store/`, `work/` and `tmp/`, made through the held claim directory. */
|
|
393
|
+
async #layout() {
|
|
394
|
+
const claim = this.rawClaim;
|
|
395
|
+
for (const dir of [claim.store, claim.work, join(claim.root, "tmp")]) {
|
|
396
|
+
const under = relative(claim.root, dir);
|
|
397
|
+
if (under === "" || under.startsWith("..") || isAbsolute(under))
|
|
398
|
+
throw new RunError("INVALID_ARGUMENT", `${dir} is not under the claim root ${claim.root}`);
|
|
399
|
+
try {
|
|
400
|
+
await mkdir(join(this.#dir.at, under), { recursive: true });
|
|
380
401
|
}
|
|
381
|
-
|
|
402
|
+
catch (error) {
|
|
403
|
+
throw ownWriteFenced(dir, error);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
382
406
|
}
|
|
383
407
|
async checkSeal(store) {
|
|
384
408
|
const sealed = this.sealedSeq;
|
|
@@ -611,7 +635,7 @@ class Lease {
|
|
|
611
635
|
throw new RunError("RUN_RELEASED", `run ${this.ref.id} is released`);
|
|
612
636
|
}
|
|
613
637
|
}
|
|
614
|
-
async function openLease(ref, options, step) {
|
|
638
|
+
async function openLease(ref, options, step, afterClaim) {
|
|
615
639
|
const context = options.context ?? BACKGROUND_CONTEXT;
|
|
616
640
|
leaseOf(options.lease);
|
|
617
641
|
let holder;
|
|
@@ -633,7 +657,7 @@ async function openLease(ref, options, step) {
|
|
|
633
657
|
}));
|
|
634
658
|
const lease = new Lease(ref, claim, options, context);
|
|
635
659
|
try {
|
|
636
|
-
await lease.open(step, holder);
|
|
660
|
+
await lease.open(step, holder, afterClaim);
|
|
637
661
|
}
|
|
638
662
|
catch (error) {
|
|
639
663
|
await lease.abandon(error);
|
|
@@ -642,8 +666,9 @@ async function openLease(ref, options, step) {
|
|
|
642
666
|
return lease;
|
|
643
667
|
}
|
|
644
668
|
/**
|
|
645
|
-
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock,
|
|
646
|
-
* `
|
|
669
|
+
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock, create
|
|
670
|
+
* `store/`, `work/` and `tmp/`, reap a killed incarnation's commands, seed a new run, write `run.json` and start the
|
|
671
|
+
* heartbeat and the self-fence. Errors are typed as for
|
|
647
672
|
* `openDurableRun`: HeldError (76), FencedError (75, also passed to `onFenced`), RunRecordError (65), RunError
|
|
648
673
|
* INVALID_ARGUMENT before any claim.
|
|
649
674
|
*/
|
|
@@ -697,6 +722,7 @@ class Run {
|
|
|
697
722
|
// Both refusals here are terminal: the run is marked `failed`, keeping its seal, so the supervisor stops on it.
|
|
698
723
|
await step("seal", () => lease.checkSeal(this.#store));
|
|
699
724
|
const storage = lease.observe(this.#store.storage);
|
|
725
|
+
// openDurableRun refused an open without Harness options (here or from afterClaim) before this runs.
|
|
700
726
|
this.#harness = await step("harness", () => Harness.open(storage, { ...this.#options.harness, env: this.env }, this.#context));
|
|
701
727
|
await step("live", () => lease.live());
|
|
702
728
|
const before = this.#options.beforeResume;
|
|
@@ -749,8 +775,8 @@ class Run {
|
|
|
749
775
|
}
|
|
750
776
|
}
|
|
751
777
|
/**
|
|
752
|
-
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock,
|
|
753
|
-
* self-fence
|
|
778
|
+
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock, layout, the commands reap, then
|
|
779
|
+
* `afterClaim`, a new run's seed, then `run.json`, heartbeat and self-fence), then the store, the seal check before anything commits, pi's Harness with the app's registry,
|
|
754
780
|
* the seal cleared in `run.json`, resume.
|
|
755
781
|
*
|
|
756
782
|
* Errors are typed: HeldError (76) when another client, another process on this host or another connection holds the
|
|
@@ -758,25 +784,32 @@ class Run {
|
|
|
758
784
|
* RunError STORE_HEAD_UNREADABLE (70), each after marking the run `failed` and releasing the claim.
|
|
759
785
|
*/
|
|
760
786
|
export async function openDurableRun(ref, options) {
|
|
787
|
+
if (!options.harness && !options.afterClaim)
|
|
788
|
+
throw new RunError("INVALID_ARGUMENT", "openDurableRun needs pi's Harness options, from its options or from afterClaim");
|
|
761
789
|
const context = options.context ?? BACKGROUND_CONTEXT;
|
|
762
790
|
const step = stepper(options.onStep);
|
|
763
791
|
let env;
|
|
764
|
-
|
|
792
|
+
let opening = options;
|
|
793
|
+
const given = options.afterClaim;
|
|
794
|
+
// Inside the lease's open, before run.json: a factory that fails publishes nothing of this incarnation.
|
|
795
|
+
const afterClaim = given &&
|
|
796
|
+
(async ({ ref: claimRef, root, work, store }) => {
|
|
797
|
+
const got = await given({ ref: claimRef, root, work, store });
|
|
798
|
+
opening = { ...options, ...(got.harness ? { harness: got.harness } : {}), ...(got.beforeResume ? { beforeResume: got.beforeResume } : {}) };
|
|
799
|
+
if (!opening.harness)
|
|
800
|
+
throw new RunError("INVALID_ARGUMENT", "openDurableRun needs pi's Harness options: afterClaim returned none, and its options give none");
|
|
801
|
+
return got.seed ? { seed: got.seed } : {};
|
|
802
|
+
});
|
|
803
|
+
// The lease's cleanup is the environment's: abandoning the lease cleans both.
|
|
804
|
+
const lease = await openLease(ref, { ...options, context, cleanup: (c) => env?.cleanup(c) ?? Promise.resolve() }, step, afterClaim);
|
|
765
805
|
try {
|
|
766
|
-
// A claim that records its commands (a local run's): end what a killed incarnation left running, now that the owner
|
|
767
|
-
// lock is held and before the Harness opens, so no command of it still writes in work/ once the run resumes.
|
|
768
|
-
const commands = lease.rawClaim.commands;
|
|
769
|
-
// A process of a killed incarnation that outlives its SIGKILL refuses the open (HeldError, 76): the run never resumes
|
|
770
|
-
// beside it, and its record stays for the next open.
|
|
771
|
-
if (commands !== undefined)
|
|
772
|
-
await step("commands", () => reapOrRefuse(commands));
|
|
773
806
|
env = (options.env ?? ((c) => archilEnv(c)))(lease.rawClaim);
|
|
774
807
|
}
|
|
775
808
|
catch (error) {
|
|
776
809
|
await lease.abandon(error);
|
|
777
810
|
throw error;
|
|
778
811
|
}
|
|
779
|
-
const run = new Run(lease, env,
|
|
812
|
+
const run = new Run(lease, env, opening, context);
|
|
780
813
|
try {
|
|
781
814
|
await run.open(step);
|
|
782
815
|
}
|
package/dist/status.d.ts
CHANGED
|
@@ -55,6 +55,14 @@ export declare class RunRecordError extends PdaError {
|
|
|
55
55
|
export declare function parseRunRecord(text: string): RunRecord;
|
|
56
56
|
/** The `run.json` under a run's root on a local mount (or in a claim directory), or undefined when the run has none yet. */
|
|
57
57
|
export declare function readRunRecord(root: string | ClaimDir): Promise<RunRecord | undefined>;
|
|
58
|
+
/**
|
|
59
|
+
* The status of a run on a local directory (`localRun`): `<root>/run.json`, where `root` is the directory `localRun(root)`
|
|
60
|
+
* takes (not its `work/`). Read as `readRunStatus` reads a disk's through the control API: the same strict parser and the
|
|
61
|
+
* same `RunRecord`. It only reads: no lock, no mount and no store are touched, so it is safe beside a live instance, whose
|
|
62
|
+
* next write replaces the file whole. A run with no `run.json` yet, or no directory, is `null`; a record that is malformed
|
|
63
|
+
* is `RunRecordError` (`RUN_JSON_INVALID`); any other read error is thrown.
|
|
64
|
+
*/
|
|
65
|
+
export declare function readLocalRunStatus(root: string): Promise<RunRecord | null>;
|
|
58
66
|
/**
|
|
59
67
|
* True while the record's last heartbeat is younger than `expiryMs` on the reader's clock. The supervisor allows for
|
|
60
68
|
* clock skew between hosts inside the expiry.
|
package/dist/status.js
CHANGED
|
@@ -96,6 +96,16 @@ export async function readRunRecord(root) {
|
|
|
96
96
|
}
|
|
97
97
|
return parseRunRecord(text);
|
|
98
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* The status of a run on a local directory (`localRun`): `<root>/run.json`, where `root` is the directory `localRun(root)`
|
|
101
|
+
* takes (not its `work/`). Read as `readRunStatus` reads a disk's through the control API: the same strict parser and the
|
|
102
|
+
* same `RunRecord`. It only reads: no lock, no mount and no store are touched, so it is safe beside a live instance, whose
|
|
103
|
+
* next write replaces the file whole. A run with no `run.json` yet, or no directory, is `null`; a record that is malformed
|
|
104
|
+
* is `RunRecordError` (`RUN_JSON_INVALID`); any other read error is thrown.
|
|
105
|
+
*/
|
|
106
|
+
export async function readLocalRunStatus(root) {
|
|
107
|
+
return (await readRunRecord(root)) ?? null;
|
|
108
|
+
}
|
|
99
109
|
/**
|
|
100
110
|
* True while the record's last heartbeat is younger than `expiryMs` on the reader's clock. The supervisor allows for
|
|
101
111
|
* clock skew between hosts inside the expiry.
|
package/dist/supervise.d.ts
CHANGED
|
@@ -12,9 +12,14 @@ export type HostHandle = {
|
|
|
12
12
|
driver: string;
|
|
13
13
|
[key: string]: Json;
|
|
14
14
|
};
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* Which start this is: the generation the instance will open (run.json's generation + 1), also in the token's nickname.
|
|
17
|
+
* `env`: what this start adds to the instance's environment (a place's per-attempt order), merged over the driver's own
|
|
18
|
+
* `env`. PATH and PDA_HOLDER are the driver's: a start that sets either, or a name that is not a variable name, is refused.
|
|
19
|
+
*/
|
|
16
20
|
export type StartAttempt = {
|
|
17
21
|
attempt: number;
|
|
22
|
+
env?: Record<string, string>;
|
|
18
23
|
};
|
|
19
24
|
/** Compute is disposable and pluggable: a driver is these three calls. */
|
|
20
25
|
export interface HostDriver {
|
|
@@ -28,6 +33,14 @@ export interface HostDriver {
|
|
|
28
33
|
status(handle: HostHandle): Promise<HostStatus>;
|
|
29
34
|
/** Stop the instance and whatever it left on its host. A handle the driver cannot reach is a no-op. */
|
|
30
35
|
stop(handle: HostHandle): Promise<void>;
|
|
36
|
+
/**
|
|
37
|
+
* Optional: take the instance away abruptly, as a crash would. No drain, no release, nothing cleaned: the run's claim is
|
|
38
|
+
* left as a crash leaves it (orphaned with its box, or held by a live mount until its lease lapses), and the
|
|
39
|
+
* supervisor's next pass decides what follows. A handle the driver cannot reach is a no-op.
|
|
40
|
+
*/
|
|
41
|
+
kill?(handle: HostHandle): Promise<void>;
|
|
42
|
+
/** What `kill` takes: `process`, the instance's processes while its machine stays (a restart policy may bring them back), or `box`, its machine. */
|
|
43
|
+
readonly kills?: "process" | "box";
|
|
31
44
|
}
|
|
32
45
|
/** The control-plane calls the supervisor uses; an archil `Disk` satisfies it. */
|
|
33
46
|
export interface SupervisorControl extends ControlApi {
|
|
@@ -73,6 +86,16 @@ export interface EnsureOptions {
|
|
|
73
86
|
tokenTtl?: string;
|
|
74
87
|
/** Prefix of each mount token's nickname. */
|
|
75
88
|
tokenPrefix?: string;
|
|
89
|
+
/**
|
|
90
|
+
* A mount token the caller minted for this start (`mintMountToken`), for a caller that mints its own (a place's
|
|
91
|
+
* `token(attempt)`): a start uses it instead of minting one, and removes it as it would its own when the driver starts
|
|
92
|
+
* nothing or adopts. A decision that starts nothing leaves it to the caller.
|
|
93
|
+
*/
|
|
94
|
+
mountToken?: {
|
|
95
|
+
token: string;
|
|
96
|
+
identifier: string;
|
|
97
|
+
nickname: string;
|
|
98
|
+
};
|
|
76
99
|
/** The caller has work for the run now: start a paused run, or a sleeping one before its `wakeAt`. */
|
|
77
100
|
demand?: boolean;
|
|
78
101
|
/** Create `runs/<id>/` with this owner when it does not exist and nothing holds it. */
|
package/dist/supervise.js
CHANGED
|
@@ -282,7 +282,7 @@ async function startInstance(ref, host, opts, why) {
|
|
|
282
282
|
let token;
|
|
283
283
|
try {
|
|
284
284
|
// The attempt is the generation the instance will write, so the token user names the incarnation it served.
|
|
285
|
-
token = await mintMountToken(opts.control, { run: { id: ref.id, attempt: why.generation + 1 }, prefix: opts.tokenPrefix, ttl: opts.tokenTtl ?? TOKEN_TTL, now: opts.now });
|
|
285
|
+
token = opts.mountToken ?? await mintMountToken(opts.control, { run: { id: ref.id, attempt: why.generation + 1 }, prefix: opts.tokenPrefix, ttl: opts.tokenTtl ?? TOKEN_TTL, now: opts.now });
|
|
286
286
|
}
|
|
287
287
|
catch (err) {
|
|
288
288
|
throw new SuperviseError("CONTROL_API_FAILED", `minting a mount token for ${runPath(ref.id)} failed`, { cause: err });
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { type ArchilHost } from "./claim.js";
|
|
2
|
+
import type { SupervisorControl } from "./supervise.js";
|
|
3
|
+
/** The control API both calls use; `archilControl`'s satisfies it. */
|
|
4
|
+
export type ClaimControl = Pick<SupervisorControl, "putObject" | "addUser" | "removeUser" | "listDelegations" | "revokeDelegation" | "exec" | "getObject" | "headObject">;
|
|
5
|
+
/** The path `fn` holds: `root` is where it is mounted on this host. */
|
|
6
|
+
export interface HeldPath {
|
|
7
|
+
readonly root: string;
|
|
8
|
+
readonly path: string;
|
|
9
|
+
/** Aborted once the claim is known lost (a barrier refused) or the caller's `signal` aborts. */
|
|
10
|
+
readonly signal: AbortSignal;
|
|
11
|
+
/** `archil sync` now: every write so far is durable when it resolves; a refusal is FENCED. */
|
|
12
|
+
barrier(): Promise<{
|
|
13
|
+
ms: number;
|
|
14
|
+
}>;
|
|
15
|
+
}
|
|
16
|
+
export interface WithClaimOptions {
|
|
17
|
+
/** How long the holder may hold the path; written in its `.claim` as its deadline. Default 60 s. */
|
|
18
|
+
readonly holdMs?: number;
|
|
19
|
+
/** How long to queue behind a live holder before CLAIM_HELD. Default 30 s. */
|
|
20
|
+
readonly waitMs?: number;
|
|
21
|
+
/** Where the mount goes: `<mountRoot>/runs/<pid>-<uuid>`, of this call's own. Default `/mnt/archil/.claims`. */
|
|
22
|
+
readonly mountRoot?: string;
|
|
23
|
+
readonly host?: ArchilHost;
|
|
24
|
+
readonly signal?: AbortSignal;
|
|
25
|
+
/** The owner and mode of the path's directories it makes when the disk lacks them. Default this process's, 0700. */
|
|
26
|
+
readonly owner?: {
|
|
27
|
+
readonly uid: number;
|
|
28
|
+
readonly gid: number;
|
|
29
|
+
readonly mode?: number;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export declare const DEFAULT_HOLD_MS = 60000;
|
|
33
|
+
export declare const DEFAULT_WAIT_MS = 30000;
|
|
34
|
+
/**
|
|
35
|
+
* Hold `path` exclusively while `fn` runs: wait for a live holder (`waitMs`), revoke a lost one, mount `<disk>:/<path>` at
|
|
36
|
+
* a fresh `<mountRoot>/runs/<pid>-<uuid>`, write `.claim` with this holder's deadline (`holdMs`), run `fn`, then barrier,
|
|
37
|
+
* unmount and give the mount token back, whether `fn` resolved or threw. Returns `fn`'s value; throws `fn`'s own error,
|
|
38
|
+
* else INVALID_ARGUMENT, CLAIM_HELD (76, retryable), CONTROL_API_FAILED, the mount's errors, or FENCED (75: the claim was
|
|
39
|
+
* revoked from under `fn`, whose writes are then not durable).
|
|
40
|
+
*/
|
|
41
|
+
export declare function withClaim<T>(where: {
|
|
42
|
+
readonly control: ClaimControl;
|
|
43
|
+
readonly disk: string;
|
|
44
|
+
readonly region: string;
|
|
45
|
+
readonly path: string;
|
|
46
|
+
}, fn: (held: HeldPath) => Promise<T>, options?: WithClaimOptions): Promise<T>;
|
|
47
|
+
/**
|
|
48
|
+
* `<path>/<file>` over S3, once it is released: no delegation on the path (or an ancestor or a descendant) before or
|
|
49
|
+
* after the read, for SQLite (`sqlite`) no `-wal` with content (its writer checkpointed with TRUNCATE and closed), and the
|
|
50
|
+
* object unchanged across the read. A held path is CLAIM_HELD (76, retryable; never read an older release instead), a
|
|
51
|
+
* WAL with content NOT_RELEASED; a file that moved during every one of `attempts` reads (default 3) is CLAIM_HELD. Null
|
|
52
|
+
* when the file does not exist. It never mounts, so it never blocks a carry back.
|
|
53
|
+
*/
|
|
54
|
+
export declare function readReleased(where: {
|
|
55
|
+
readonly control: ClaimControl;
|
|
56
|
+
readonly path: string;
|
|
57
|
+
}, file: string, options?: {
|
|
58
|
+
readonly sqlite?: boolean;
|
|
59
|
+
readonly attempts?: number;
|
|
60
|
+
}): Promise<{
|
|
61
|
+
bytes: Uint8Array;
|
|
62
|
+
etag: string | null;
|
|
63
|
+
} | null>;
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
// A short exclusive claim on any path of a disk (withClaim), and a consistent S3 read of a released file (readReleased),
|
|
2
|
+
// for a host that keeps state beside its runs: pi-durable-agent's agent home under `agent/`, carried back at a run's end.
|
|
3
|
+
//
|
|
4
|
+
// A claim is one exclusive mount. Archil refuses a second client's mount of the path, of an ancestor or of a descendant,
|
|
5
|
+
// and that refusal is the exclusivity; the delegation listing only decides whom to wait for and whom to revoke. A holder
|
|
6
|
+
// is lost when its client is gone (orphaned), or when it holds exactly the path and the `.claim` there names a deadline
|
|
7
|
+
// that has passed and stays the same across a few seconds (a holder that just mounted rewrites it at once); only a lost
|
|
8
|
+
// holder is revoked. Safety rests on the fence, never on the deadline: a holder revoked while still alive gets EIO at
|
|
9
|
+
// its next fsync, so its commit rolls back and its own call throws FENCED. Both calls are supervisor side: they need the
|
|
10
|
+
// API key's control API.
|
|
11
|
+
import { randomUUID } from "node:crypto";
|
|
12
|
+
import { hostname } from "node:os";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
15
|
+
import { CLAIM_PROBE, DEFAULT_MOUNT_ROOT, exclusiveMount, mintMountToken, removeMountToken, revokeCompanions } from "./claim.js";
|
|
16
|
+
import { ClaimError, HeldError } from "./errors.js";
|
|
17
|
+
export const DEFAULT_HOLD_MS = 60_000;
|
|
18
|
+
export const DEFAULT_WAIT_MS = 30_000;
|
|
19
|
+
const POLL_MS = 250;
|
|
20
|
+
/**
|
|
21
|
+
* How long a past-deadline `.claim` must stay the same, with the same holder, before that holder counts as lost. A holder
|
|
22
|
+
* that mounted a moment ago writes its own `.claim` right after its mount; until then the one there is its predecessor's.
|
|
23
|
+
*/
|
|
24
|
+
const CONFIRM_MS = 3_000;
|
|
25
|
+
const TOKEN_TTL = "10m";
|
|
26
|
+
const SEGMENT = /^[A-Za-z0-9._-]+$/;
|
|
27
|
+
/** CLAIM_HELD reads as retryable: the path is not released yet, and nothing older stands in for it. */
|
|
28
|
+
const notReleased = (path, why, cause) => new HeldError("claim", `${path} is held by another client and is not released yet (${why}). CLAIM_HELD is retryable: retry within waitMs, ` +
|
|
29
|
+
"and never fall back to an older release.", { cause });
|
|
30
|
+
/** `path` if it is one relative path of safe segments that no run owns; else INVALID_ARGUMENT. */
|
|
31
|
+
function claimPath(path) {
|
|
32
|
+
const segments = path.split("/");
|
|
33
|
+
if (path === "" || segments.some((s) => !SEGMENT.test(s) || s === "." || s === "..")) {
|
|
34
|
+
throw new ClaimError("INVALID_ARGUMENT", `${JSON.stringify(path)} is not a relative path of safe segments (letters, digits, ".", "_", "-")`);
|
|
35
|
+
}
|
|
36
|
+
if (segments[0] === "runs")
|
|
37
|
+
throw new ClaimError("INVALID_ARGUMENT", `${path} is under runs/, which openDurableRun and the supervisor claim`);
|
|
38
|
+
if (segments[0] === ".archil")
|
|
39
|
+
throw new ClaimError("INVALID_ARGUMENT", `${path} is under .archil/, Archil's own`);
|
|
40
|
+
return path;
|
|
41
|
+
}
|
|
42
|
+
/** Whether a delegation's path is `path`, under it, or above it (the disk root included): each conflicts with a mount of it. */
|
|
43
|
+
const conflicts = (delegated, path) => {
|
|
44
|
+
const p = delegated.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
45
|
+
return p === "" || p === path || p.startsWith(`${path}/`) || path.startsWith(`${p}/`);
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* The delegations that conflict with a mount of `path`, from one listing. A pathless one is attributed by inode with one
|
|
49
|
+
* `exec` (the path's ancestors and everything under it); one that cannot be attributed fails CONTROL_API_FAILED, never
|
|
50
|
+
* reads as unheld.
|
|
51
|
+
*/
|
|
52
|
+
async function holdersOf(control, path) {
|
|
53
|
+
let all;
|
|
54
|
+
try {
|
|
55
|
+
all = await control.listDelegations();
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
throw new ClaimError("CONTROL_API_FAILED", "listing delegations failed", { cause: err });
|
|
59
|
+
}
|
|
60
|
+
const holders = all.filter((d) => d.path !== undefined && d.path !== null && conflicts(d.path, path));
|
|
61
|
+
const pathless = all.filter((d) => !d.path);
|
|
62
|
+
if (pathless.length > 0)
|
|
63
|
+
holders.push(...(await pathlessOn(control, path, pathless)));
|
|
64
|
+
return { holders, all };
|
|
65
|
+
}
|
|
66
|
+
async function pathlessOn(control, path, pathless) {
|
|
67
|
+
if (!control.exec)
|
|
68
|
+
throw new ClaimError("CONTROL_API_FAILED", `a delegation with no path may be on ${path}, and the control API has no exec to attribute it`);
|
|
69
|
+
const inodes = [...new Set(pathless.map((d) => d.inodeId))];
|
|
70
|
+
if (!inodes.every((i) => Number.isSafeInteger(i) && i >= 0))
|
|
71
|
+
throw new ClaimError("CONTROL_API_FAILED", "a pathless delegation has no valid inode id");
|
|
72
|
+
// Every path relative to the disk root as ./<path>, so a segment that begins with "-" is never read as an option.
|
|
73
|
+
const segments = path.split("/");
|
|
74
|
+
const above = [".", ...segments.slice(0, -1).map((_, i) => `./${segments.slice(0, i + 1).join("/")}`)];
|
|
75
|
+
const test = inodes.map((i) => `-inum ${i}`).join(" -o ");
|
|
76
|
+
let r;
|
|
77
|
+
try {
|
|
78
|
+
r = await control.exec(`for a in ${above.join(" ")}; do if [ -e "$a" ]; then stat -c '%i' -- "$a" || exit 3; fi; done; if [ -e ./${path} ]; then find ./${path} \\( ${test} \\) -printf '%i\\n' || exit 3; fi`);
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
throw new ClaimError("CONTROL_API_FAILED", `attributing pathless delegations to ${path} failed`, { cause: err });
|
|
82
|
+
}
|
|
83
|
+
if (r.exitCode !== 0)
|
|
84
|
+
throw new ClaimError("CONTROL_API_FAILED", `attributing pathless delegations to ${path}: exit ${r.exitCode}`);
|
|
85
|
+
const on = new Set(String(r.stdout ?? "").split("\n").filter(Boolean).map(Number));
|
|
86
|
+
return pathless.filter((d) => on.has(d.inodeId));
|
|
87
|
+
}
|
|
88
|
+
/** `<path>/.claim` over S3, and whether the deadline it names has passed; null when it cannot be read or parsed. */
|
|
89
|
+
async function readClaim(control, path) {
|
|
90
|
+
try {
|
|
91
|
+
const text = new TextDecoder().decode(await control.getObject(`${path}/${CLAIM_PROBE}`));
|
|
92
|
+
const claim = JSON.parse(text);
|
|
93
|
+
const deadline = typeof claim.deadline === "string" ? Date.parse(claim.deadline) : Number.NaN;
|
|
94
|
+
return Number.isFinite(deadline) ? { text, expired: deadline < Date.now() } : null;
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const exactly = (d, path) => typeof d.path === "string" && d.path.replace(/^\/+/, "").replace(/\/+$/, "") === path;
|
|
101
|
+
const clientsOf = (holders) => [...new Set(holders.map((d) => d.clientId))].sort().join(",");
|
|
102
|
+
/**
|
|
103
|
+
* Whether the live holders are lost past their deadline. Only a holder of exactly `path` has its deadline there, so any
|
|
104
|
+
* other (a descendant's, an ancestor's, one with no path) is never lost this way. And the `.claim` must be the same
|
|
105
|
+
* expired one, with the same holders, across CONFIRM_MS: a holder that just mounted rewrites it within that time.
|
|
106
|
+
*/
|
|
107
|
+
async function lostPastDeadline(control, path, holders, signal) {
|
|
108
|
+
if (!holders.every((d) => exactly(d, path)))
|
|
109
|
+
return false;
|
|
110
|
+
const first = await readClaim(control, path);
|
|
111
|
+
if (!first?.expired)
|
|
112
|
+
return false;
|
|
113
|
+
await sleep(CONFIRM_MS, undefined, { signal }).catch(() => signal?.throwIfAborted());
|
|
114
|
+
const again = await holdersOf(control, path);
|
|
115
|
+
if (clientsOf(again.holders) !== clientsOf(holders))
|
|
116
|
+
return false;
|
|
117
|
+
const second = await readClaim(control, path);
|
|
118
|
+
return second !== null && second.expired && second.text === first.text;
|
|
119
|
+
}
|
|
120
|
+
/** Until nothing conflicts with a mount of `path`: lost holders are revoked, live ones waited for until `deadline`. */
|
|
121
|
+
async function settle(control, path, deadline, signal) {
|
|
122
|
+
for (;;) {
|
|
123
|
+
signal?.throwIfAborted();
|
|
124
|
+
const { holders, all } = await holdersOf(control, path);
|
|
125
|
+
if (holders.length === 0)
|
|
126
|
+
return;
|
|
127
|
+
if (holders.every((d) => d.isOrphaned) || (await lostPastDeadline(control, path, holders, signal))) {
|
|
128
|
+
try {
|
|
129
|
+
for (const d of holders)
|
|
130
|
+
await control.revokeDelegation({ clientId: d.clientId, inodeId: d.inodeId });
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
throw new ClaimError("CONTROL_API_FAILED", `revoking the lost holder of ${path} failed`, { cause: err });
|
|
134
|
+
}
|
|
135
|
+
await revokeCompanions(control, holders, all);
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
signal?.throwIfAborted();
|
|
139
|
+
if (Date.now() >= deadline)
|
|
140
|
+
throw notReleased(path, `a live holder within its deadline: ${holders.map((d) => d.clientId).join(", ")}`);
|
|
141
|
+
await sleep(POLL_MS, undefined, { signal }).catch(() => signal?.throwIfAborted());
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* The directories of `path` the disk lacks, made over S3 with `owner`: archil mounts no path whose directory is missing
|
|
146
|
+
* ("the configured subdirectory does not exist"). A write refused with 409 met another caller that made the directory
|
|
147
|
+
* and holds it: HeldError, which the caller queues on as on a refused mount.
|
|
148
|
+
*/
|
|
149
|
+
async function ensureDirectories(control, path, owner) {
|
|
150
|
+
const segments = path.split("/");
|
|
151
|
+
for (let i = 1; i <= segments.length; i++) {
|
|
152
|
+
const key = `${segments.slice(0, i).join("/")}/`;
|
|
153
|
+
try {
|
|
154
|
+
if ((await control.headObject(key)) === null)
|
|
155
|
+
await control.putObject(key, "", owner);
|
|
156
|
+
}
|
|
157
|
+
catch (err) {
|
|
158
|
+
if (err.status === 409)
|
|
159
|
+
throw new HeldError("claim", `${key} is held by the caller that made it (409)`, { cause: err });
|
|
160
|
+
throw new ClaimError("CONTROL_API_FAILED", `making ${key} on the disk failed`, { cause: err });
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const positive = (name, value) => {
|
|
165
|
+
if (!Number.isSafeInteger(value) || value <= 0)
|
|
166
|
+
throw new ClaimError("INVALID_ARGUMENT", `${name} ${value} is not a positive whole number of milliseconds`);
|
|
167
|
+
return value;
|
|
168
|
+
};
|
|
169
|
+
/**
|
|
170
|
+
* Hold `path` exclusively while `fn` runs: wait for a live holder (`waitMs`), revoke a lost one, mount `<disk>:/<path>` at
|
|
171
|
+
* a fresh `<mountRoot>/runs/<pid>-<uuid>`, write `.claim` with this holder's deadline (`holdMs`), run `fn`, then barrier,
|
|
172
|
+
* unmount and give the mount token back, whether `fn` resolved or threw. Returns `fn`'s value; throws `fn`'s own error,
|
|
173
|
+
* else INVALID_ARGUMENT, CLAIM_HELD (76, retryable), CONTROL_API_FAILED, the mount's errors, or FENCED (75: the claim was
|
|
174
|
+
* revoked from under `fn`, whose writes are then not durable).
|
|
175
|
+
*/
|
|
176
|
+
export async function withClaim(where, fn, options = {}) {
|
|
177
|
+
const path = claimPath(where.path);
|
|
178
|
+
const holdMs = positive("holdMs", options.holdMs ?? DEFAULT_HOLD_MS);
|
|
179
|
+
const deadline = Date.now() + positive("waitMs", options.waitMs ?? DEFAULT_WAIT_MS);
|
|
180
|
+
const { control } = where;
|
|
181
|
+
const mountRoot = options.mountRoot ?? join(DEFAULT_MOUNT_ROOT, ".claims");
|
|
182
|
+
const owner = { uid: options.owner?.uid ?? process.getuid(), gid: options.owner?.gid ?? process.getgid(), mode: options.owner?.mode ?? 0o700 };
|
|
183
|
+
for (;;) {
|
|
184
|
+
await settle(control, path, deadline, options.signal);
|
|
185
|
+
options.signal?.throwIfAborted();
|
|
186
|
+
try {
|
|
187
|
+
await ensureDirectories(control, path, owner);
|
|
188
|
+
}
|
|
189
|
+
catch (err) {
|
|
190
|
+
// Another caller made the directory and holds it: queue again, within the same wait.
|
|
191
|
+
if (err instanceof HeldError && Date.now() < deadline && !options.signal?.aborted) {
|
|
192
|
+
await sleep(POLL_MS);
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
options.signal?.throwIfAborted();
|
|
196
|
+
throw err instanceof HeldError ? notReleased(path, "its directory is being made by another holder", err) : err;
|
|
197
|
+
}
|
|
198
|
+
const minted = await mintMountToken(control, { nickname: `pdc-${path.replaceAll("/", ".")}-${Date.now().toString(36)}`, ttl: TOKEN_TTL });
|
|
199
|
+
const at = new Date();
|
|
200
|
+
let mount;
|
|
201
|
+
try {
|
|
202
|
+
// <mountRoot>/runs/<name>: the shape archil-scoped's cleanup verbs take, for a host that cannot unmount (Sysbox).
|
|
203
|
+
mount = await exclusiveMount({ disk: where.disk, region: where.region, path }, join(mountRoot, "runs", `${process.pid}-${randomUUID()}`), minted.token, {
|
|
204
|
+
host: options.host,
|
|
205
|
+
probe: `${JSON.stringify({ pid: process.pid, host: hostname(), at: at.toISOString(), deadline: new Date(at.getTime() + holdMs).toISOString() })}\n`,
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
catch (err) {
|
|
209
|
+
await removeMountToken(control, minted.identifier).catch(() => { });
|
|
210
|
+
// Another caller mounted between the listing and the mount: queue again, within the same wait.
|
|
211
|
+
if (err instanceof HeldError && Date.now() < deadline && !options.signal?.aborted) {
|
|
212
|
+
await sleep(POLL_MS);
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
options.signal?.throwIfAborted();
|
|
216
|
+
throw err instanceof HeldError ? notReleased(path, "its mount was refused", err) : err;
|
|
217
|
+
}
|
|
218
|
+
// Aborted while it mounted: fn never runs, and nothing it holds is kept.
|
|
219
|
+
if (options.signal?.aborted) {
|
|
220
|
+
await mount.unmount().catch(() => undefined);
|
|
221
|
+
await removeMountToken(control, minted.identifier).catch(() => { });
|
|
222
|
+
options.signal.throwIfAborted();
|
|
223
|
+
}
|
|
224
|
+
const lost = new AbortController();
|
|
225
|
+
const onAbort = () => lost.abort(options.signal?.reason);
|
|
226
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
227
|
+
const barrier = async () => {
|
|
228
|
+
try {
|
|
229
|
+
return await mount.barrier();
|
|
230
|
+
}
|
|
231
|
+
catch (err) {
|
|
232
|
+
lost.abort(err);
|
|
233
|
+
throw err;
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
let outcome;
|
|
237
|
+
try {
|
|
238
|
+
outcome = { ok: true, value: await fn({ root: mount.root, path, signal: lost.signal, barrier }) };
|
|
239
|
+
}
|
|
240
|
+
catch (error) {
|
|
241
|
+
outcome = { ok: false, error };
|
|
242
|
+
}
|
|
243
|
+
let release;
|
|
244
|
+
try {
|
|
245
|
+
await barrier();
|
|
246
|
+
}
|
|
247
|
+
catch (err) {
|
|
248
|
+
release = err;
|
|
249
|
+
}
|
|
250
|
+
try {
|
|
251
|
+
await mount.unmount();
|
|
252
|
+
}
|
|
253
|
+
catch (err) {
|
|
254
|
+
release ??= err;
|
|
255
|
+
}
|
|
256
|
+
await removeMountToken(control, minted.identifier).catch(() => { });
|
|
257
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
258
|
+
if (!outcome.ok)
|
|
259
|
+
throw outcome.error;
|
|
260
|
+
if (release !== undefined)
|
|
261
|
+
throw release;
|
|
262
|
+
return outcome.value;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
/** An object's identity for "did it move": its etag, else its size and last-modified time. */
|
|
266
|
+
function identity(head) {
|
|
267
|
+
if (head === null || typeof head !== "object")
|
|
268
|
+
return null;
|
|
269
|
+
const { etag, size, lastModified } = head;
|
|
270
|
+
if (typeof etag === "string" && etag !== "")
|
|
271
|
+
return `etag:${etag}`;
|
|
272
|
+
return `size:${String(size)}@${lastModified instanceof Date ? lastModified.toISOString() : String(lastModified)}`;
|
|
273
|
+
}
|
|
274
|
+
const sizeOf = (head) => (head !== null && typeof head === "object" ? Number(head.size ?? 0) : 0);
|
|
275
|
+
/**
|
|
276
|
+
* `<path>/<file>` over S3, once it is released: no delegation on the path (or an ancestor or a descendant) before or
|
|
277
|
+
* after the read, for SQLite (`sqlite`) no `-wal` with content (its writer checkpointed with TRUNCATE and closed), and the
|
|
278
|
+
* object unchanged across the read. A held path is CLAIM_HELD (76, retryable; never read an older release instead), a
|
|
279
|
+
* WAL with content NOT_RELEASED; a file that moved during every one of `attempts` reads (default 3) is CLAIM_HELD. Null
|
|
280
|
+
* when the file does not exist. It never mounts, so it never blocks a carry back.
|
|
281
|
+
*/
|
|
282
|
+
export async function readReleased(where, file, options = {}) {
|
|
283
|
+
const path = claimPath(where.path);
|
|
284
|
+
const key = `${path}/${claimPath(file)}`;
|
|
285
|
+
const attempts = positive("attempts", options.attempts ?? 3);
|
|
286
|
+
const { control } = where;
|
|
287
|
+
const head = async (name) => {
|
|
288
|
+
try {
|
|
289
|
+
return await control.headObject(name);
|
|
290
|
+
}
|
|
291
|
+
catch (err) {
|
|
292
|
+
throw new ClaimError("CONTROL_API_FAILED", `head ${name} failed`, { cause: err });
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
for (let attempt = 1;; attempt++) {
|
|
296
|
+
const { holders } = await holdersOf(control, path);
|
|
297
|
+
if (holders.length > 0)
|
|
298
|
+
throw notReleased(path, `held by ${holders.map((d) => d.clientId).join(", ")}`);
|
|
299
|
+
if (options.sqlite && sizeOf(await head(`${key}-wal`)) > 0) {
|
|
300
|
+
throw new ClaimError("NOT_RELEASED", `${key}-wal has content: its writer did not checkpoint (TRUNCATE) and close, so ${key} alone is not the database`);
|
|
301
|
+
}
|
|
302
|
+
const before = await head(key);
|
|
303
|
+
if (before === null)
|
|
304
|
+
return null;
|
|
305
|
+
let bytes;
|
|
306
|
+
try {
|
|
307
|
+
bytes = await control.getObject(key);
|
|
308
|
+
}
|
|
309
|
+
catch (err) {
|
|
310
|
+
throw new ClaimError("CONTROL_API_FAILED", `get ${key} failed`, { cause: err });
|
|
311
|
+
}
|
|
312
|
+
const after = await head(key);
|
|
313
|
+
// A writer that mounted, committed to the WAL and let go during the read leaves the database file as it was.
|
|
314
|
+
if (options.sqlite && sizeOf(await head(`${key}-wal`)) > 0) {
|
|
315
|
+
throw new ClaimError("NOT_RELEASED", `${key}-wal gained content during the read: a writer committed without a checkpoint (TRUNCATE)`);
|
|
316
|
+
}
|
|
317
|
+
const still = (await holdersOf(control, path)).holders.length === 0;
|
|
318
|
+
if (still && after !== null && identity(after) === identity(before) && sizeOf(after) === bytes.length) {
|
|
319
|
+
const etag = after.etag;
|
|
320
|
+
return { bytes, etag: typeof etag === "string" ? etag : null };
|
|
321
|
+
}
|
|
322
|
+
if (attempt >= attempts)
|
|
323
|
+
throw notReleased(path, `${key} changed while it was read, ${attempts} times`);
|
|
324
|
+
}
|
|
325
|
+
}
|