@parcha/pi-durable-disk 0.0.0-stage → 0.1.0-beta.13

package/dist/supervise.d.ts

@@ -0,0 +1,340 @@
+import type { Delegation } from "disk";
+import { type ArchilHost, type ControlApi, type PathlessResolver, type RunRef } from "./claim.js";
+import { PdaError } from "./errors.js";
+import { type RunRecord } from "./status.js";
+export type Json = null | boolean | number | string | Json[] | {
+    [key: string]: Json;
+};
+/** What a host driver reports for an instance it started. `unknown`: the handle is not this driver's to judge. */
+export type HostStatus = "running" | "stopped" | "failed" | "gone" | "unknown";
+/** A driver's description of one instance; the instance writes it into `run.json` as `holder`. */
+export type HostHandle = {
+    driver: string;
+    [key: string]: Json;
+};
+/** Which start this is: the generation the instance will open (run.json's generation + 1), also in the token's nickname. */
+export type StartAttempt = {
+    attempt: number;
+};
+/** Compute is disposable and pluggable: a driver is these three calls. */
+export interface HostDriver {
+    /**
+     * Start an instance of the run that mounts with `mountToken` (a reusable token, minted for this attempt). A driver may
+     * key the start on `attempt`, so a retry of the same attempt finds the instance it already started; it then returns
+     * that instance's handle with `adopted: true`, and the supervisor removes the token it minted for this start, which no
+     * instance used.
+     */
+    start(ref: RunRef, mountToken: string, attempt?: StartAttempt): Promise<HostHandle>;
+    status(handle: HostHandle): Promise<HostStatus>;
+    /** Stop the instance and whatever it left on its host. A handle the driver cannot reach is a no-op. */
+    stop(handle: HostHandle): Promise<void>;
+}
+/** The control-plane calls the supervisor uses; an archil `Disk` satisfies it. */
+export interface SupervisorControl extends ControlApi {
+    getObject(key: string): Promise<Uint8Array>;
+    headObject(key: string): Promise<unknown | null>;
+}
+export declare const CONTROL_TIMEOUT_MS = 10000;
+/** The same control API with every call bounded: one that does not settle in `ms` fails as CONTROL_API_FAILED. */
+export declare function withTimeouts<T extends object>(control: T, ms: number): T;
+export type SuperviseErrorCode = "CONTROL_API_FAILED" | "HOST_START_FAILED" | "RUN_TREE_NOT_DELETED";
+/** A supervisor step failed; nothing was started. Exit 1; the next tick tries again. */
+export declare class SuperviseError extends PdaError {
+    constructor(code: SuperviseErrorCode, message: string, options?: {
+        cause?: unknown;
+    });
+}
+/**
+ * `run.json` over S3 GetObject, strongly consistent once the writer fsynced the file and its directory. Absent: null.
+ * status.ts's strict reader: an unknown status or a malformed field is RunRecordError (RUN_JSON_INVALID), which fails
+ * this run's decision and nothing else.
+ */
+export declare function readRunStatus(control: Pick<SupervisorControl, "getObject">, id: string): Promise<RunRecord | null>;
+export declare const LEASE_EXPIRY_MS = 90000;
+export declare const STONITH_TIMEOUT_MS = 30000;
+/** Each start attempt's mount token lives one day (reusable: a single-use token dies at the client's 5 min refresh). */
+export declare const TOKEN_TTL = "24h";
+/** A token user younger than this may belong to a start still in flight (minted, not yet mounted): never removed. */
+export declare const TOKEN_GRACE_MS: number;
+/** The longest start grace a run whose starts keep failing gets. */
+export declare const START_BACKOFF_MAX_MS: number;
+export interface EnsureOptions {
+    control: SupervisorControl;
+    /**
+     * Attributes delegations the control API lists without a path (`pathlessResolver`). `superviseRuns` shares one across
+     * its pass, so a pass costs at most one exec however many runs it decides; default: one per decision.
+     */
+    pathless?: PathlessResolver;
+    /** A held, not orphaned delegation whose `heartbeatAt` is older than this is a lost instance. Default 90 s. */
+    leaseExpiryMs?: number;
+    /** Bound on the best-effort stop of a lost holder (status plus stop). Default 30 s. */
+    stonithTimeoutMs?: number;
+    /** TTL of each mount token (default 24 h). An instance whose token expires is fenced at the next 5 min refresh. */
+    tokenTtl?: string;
+    /** Prefix of each mount token's nickname. */
+    tokenPrefix?: string;
+    /** The caller has work for the run now: start a paused run, or a sleeping one before its `wakeAt`. */
+    demand?: boolean;
+    /** Create `runs/<id>/` with this owner when it does not exist and nothing holds it. */
+    create?: {
+        uid: number;
+        gid: number;
+        mode?: number;
+    };
+    /** Bound on every control API call; one that does not answer fails this run's decision only. Default 10 s. */
+    controlTimeoutMs?: number;
+    /**
+     * After a start, how long a tick leaves the run to the instance it started (no revoke, no second start) until that
+     * instance writes run.json at its generation. Default: the lease expiry. 0 turns the grace, and the backoff, off.
+     * A start that follows failed ones gets this doubled per failure, up to `startBackoffMaxMs`.
+     */
+    startGraceMs?: number;
+    /** The cap on a backed-off start grace. Default 10 min. */
+    startBackoffMaxMs?: number;
+    /** Who starts, written into the start mark (`<host>:<pid>` by default). */
+    supervisorId?: string;
+    now?: () => number;
+}
+export type StonithOutcome = {
+    outcome: "no-holder";
+}
+/** The driver cannot reach the holder (another host, another driver) or its host is gone. */
+ | {
+    outcome: "unreachable";
+    status: "unknown" | "gone";
+}
+/** Stopped, and whatever it left on its host cleaned; `status` is what the driver reported first. */
+ | {
+    outcome: "stopped";
+    status: HostStatus;
+    ms: number;
+} | {
+    outcome: "timed-out";
+    ms: number;
+} | {
+    outcome: "failed";
+    ms: number;
+    error: string;
+};
+export type StartReason = "none" | "orphaned" | "lease-expired";
+export type EnsureResult = 
+/** `detail` is the instance's own word on why, e.g. `{ code: "STORE_BEHIND_SEAL", sealedSeq, head }`. */
+{
+    action: "terminal";
+    status: "done" | "failed";
+    generation: number;
+    detail: Json;
+} | {
+    action: "paused";
+    generation: number;
+} | {
+    action: "sleeping";
+    wakeAt: string | null;
+    generation: number;
+} | {
+    action: "pending";
+    delegations: number;
+}
+/**
+ * An instance this or another supervisor started is inside its start grace (it has not written run.json at its
+ * generation): nothing is revoked or started. `delegations` 0: not mounted yet. `failures` > 0: the start follows
+ * that many failed ones, and `graceMs` is its backed-off grace.
+ */
+ | {
+    action: "starting";
+    generation: number;
+    sinceMs: number;
+    graceMs: number;
+    failures: number;
+    lastExit: string | null;
+    by: string | null;
+    delegations: number;
+} | {
+    action: "healthy";
+    heartbeatAt: string;
+    ageMs: number;
+    generation: number;
+} | {
+    action: "started";
+    reason: StartReason;
+    /** The run was sleeping and its `wakeAt` was due (or the caller demanded it). */
+    woke: boolean;
+    revoked: Pick<Delegation, "clientId" | "inodeId" | "path" | "isOrphaned">[];
+    stonith?: StonithOutcome;
+    /** The run directory did not exist and was created with `create`'s owner. */
+    created: boolean;
+    handle: HostHandle;
+    /** The token user minted for this start; with `adopted` it was not used and is already removed. */
+    token: {
+        identifier: string;
+        nickname: string;
+    };
+    /** The driver found the instance an earlier start of this attempt made (another supervisor, a retried call). */
+    adopted?: true;
+    /** Whether the start mark (runs/<id>/start.json) was written; without it the next tick has no start grace. */
+    startMark: boolean;
+    /** Failed starts right before this one (0 after a start that wrote run.json), and what was seen of the last one. */
+    failures: number;
+    lastExit: string | null;
+    /** The grace this start gets: the base, doubled per failure, capped. */
+    graceMs: number;
+    /** Milliseconds from after the revoke (or the decision, when nothing was revoked) to the driver's return. */
+    startMs: number;
+};
+/**
+ * Make sure the run has a live instance, or has no business having one:
+ * - `run.json` done or failed (a store behind its seal is failed, never restarted into): nothing. Paused, or sleeping
+ *   with `wakeAt` later or null: nothing unless `demand`.
+ * - No delegation on `runs/<id>`: start. Every delegation orphaned (the holder's client is gone): revoke, start.
+ * - A delegation still checking out (`isPending`): someone is mounting; nothing.
+ * - Held and the lease fresh: healthy. Held and the lease expired (or no heartbeat at all): stop the holder if the
+ *   driver can reach it (bounded), revoke, start. Revoking a live holder is safe: its next fsync fails.
+ * - The start grace: every start first writes `runs/<id>/start.json` (the generation the new instance will write, and
+ *   when). While run.json is below that generation and the mark is younger than `startGraceMs`, a run with no
+ *   delegation or with a lease that looks expired is `starting`: nothing is revoked or started, so a tick between a
+ *   start and the instance's first run.json write never fences the instance it started. Past the grace, the start has
+ *   failed and the rules above apply. Every delegation orphaned inside the grace waits too: the start's client is gone,
+ *   which is a failed start like any other.
+ * - The backoff: a start whose generation run.json never reached failed (it ended before its first run.json write, 76
+ *   or on MOUNT_FAILED, or ran out its grace; 65 and 70 mark run.json failed, which is terminal). The next start counts
+ *   it in its mark (`failures`, `lastExit`) and gets the base grace doubled per consecutive failure, capped by
+ *   `startBackoffMaxMs`, so a run whose starts keep failing is started a handful of times an hour, not once a tick. A
+ *   start that writes run.json at its generation spends its mark, and the next start counts from 0.
+ * No wait follows a revoke: the fence is Archil's, enforced by the server at the old holder's next fsync. Two racing
+ * supervisors are safe: the mount admits one claimant and the other instance exits 76.
+ */
+export declare function ensureRunning(ref: RunRef, host: HostDriver, options: EnsureOptions): Promise<EnsureResult>;
+/**
+ * The supervisor's record of its last start of a run: the generation the instance will write, when, by whom, and the
+ * failed starts right before it (`failures`; `lastExit`, what the decision saw of the last one).
+ */
+export declare const START_MARK = "start.json";
+export type StartMark = {
+    generation: number;
+    at: string;
+    by: string | null;
+    failures: number;
+    lastExit: string | null;
+};
+/** The grace of a start that follows `failures` failed ones: the base doubled per failure, capped, never below the base. */
+export declare function startGrace(baseMs: number, failures: number, maxMs?: number): number;
+/**
+ * The start mark over S3, or null when absent or unreadable (an unreadable mark gives no grace). It lives in the run's
+ * directory, so it goes with the run (deleteRunTree removes it), and a later start overwrites it; a mark whose
+ * generation run.json has reached is spent.
+ */
+export declare function readStartMark(control: Pick<SupervisorControl, "getObject">, id: string): Promise<StartMark | null>;
+/** One pass over many runs: each decision on its own, so a run whose calls fail or time out costs only its own line. */
+export declare function superviseRuns(refs: RunRef[], host: HostDriver, opts: EnsureOptions): Promise<({
+    run: string;
+    ms: number;
+} & (EnsureResult | {
+    action: "error";
+    error: string;
+    message: string;
+    cause?: string;
+}))[]>;
+export type TokenUser = {
+    identifier?: string;
+    nickname?: string;
+    status?: string;
+    expiresAt?: string;
+    createdAt?: string;
+};
+export interface TokenSweepOptions {
+    /** The disk's token users (`getDisk(id).authorizedUsers`; the list lags by seconds). */
+    listUsers(): Promise<TokenUser[]>;
+    control: Pick<SupervisorControl, "getObject" | "listDelegations" | "removeUser" | "exec">;
+    /** Only token users whose nickname starts with this (the supervisor's `tokenPrefix`). */
+    prefix: string;
+    /** The runs this pass may clean; undefined: every run a token nickname names. */
+    runs?: readonly string[];
+    /** Also remove expired token users of any run under the prefix (`supervise --sweep-tokens`). */
+    expired?: boolean;
+    /** Users younger than this are never removed (a start may be in flight). Default 15 min. */
+    graceMs?: number;
+    now?: () => number;
+}
+export type TokenSweep = {
+    removed: {
+        identifier: string;
+        run: string | null;
+        why: "released" | "expired";
+    }[];
+    failed: {
+        identifier: string;
+        error: string;
+    }[];
+};
+/**
+ * Remove mount-token users that no live mount needs. Removing a token user under a live mount makes that mount lose its
+ * claim at the client's next re-authentication, so a run's users go only when the run holds no delegation, its
+ * run.json says it is released (paused, sleeping, done, failed; never running, absent or unreadable), and the user is
+ * older than `graceMs` (a start mints before it mounts). With `expired`, users whose TTL has passed go too: an expired
+ * token already fences its mount at the next refresh.
+ */
+export declare function sweepTokens(opts: TokenSweepOptions): Promise<TokenSweep>;
+export interface CheckControl extends ControlApi {
+    listObjects(prefix: string, options?: {
+        recursive?: boolean;
+    }): Promise<{
+        objects: {
+            key: string;
+        }[];
+        commonPrefixes: string[];
+    }>;
+    deleteObjects(keys: string[], options?: {
+        quiet?: boolean;
+    }): Promise<{
+        errors: unknown[];
+    }>;
+}
+export interface CheckOptions {
+    control: CheckControl;
+    disk: string;
+    region: string;
+    /** Where the probe's two mounts go: `<mountRoot>/runs/<probe>` and `<mountRoot>/.check-b/runs/<probe>`. */
+    mountRoot: string;
+    /** Owner of the probe directory (the instance's user). */
+    owner: {
+        uid: number;
+        gid: number;
+    };
+    host?: ArchilHost;
+    idPrefix?: string;
+    tokenPrefix?: string;
+    /** Bound on every control API call. Default 10 s. */
+    controlTimeoutMs?: number;
+    /** Called with each token user minted and each probe directory created, so a caller can keep a ledger. */
+    onResource?: (kind: "token" | "token-removed" | "subdir" | "subdir-deleted" | "mount" | "unmount", id: string, detail?: string) => void;
+}
+export type CheckStep = {
+    step: string;
+    ok: boolean;
+    ms: number;
+    detail?: string;
+};
+export type CheckReport = {
+    ok: boolean;
+    host: string;
+    run: string;
+    steps: CheckStep[];
+    cleanup: string[];
+};
+/**
+ * Prove on this host and client version that the claim's two server-side guarantees hold before serving runs: a second
+ * exclusive mount of a probe directory is refused while the first holds it, and a revoked mount's fsync fails (and its
+ * barrier fences). Then a third mount takes over. Every token, mount and the probe directory are removed.
+ */
+export declare function checkHost(options: CheckOptions): Promise<CheckReport>;
+/**
+ * Delete a run's directory tree over S3. A tree with a delegation on it, even an orphaned one, is not deleted: S3
+ * DeleteObjects leaves its objects in place and reports no error, and the run id would later reopen with its old
+ * store. So: revoke every delegation on the subtree and its holders' private directories, delete the files, then the
+ * directory markers deepest first (a directory with children refuses), then list the prefix. Anything left is
+ * RUN_TREE_NOT_DELETED; a non-empty prefix is never reported as deleted.
+ */
+export declare function deleteRunTree(control: CheckControl, id: string): Promise<{
+    objects: number;
+    revoked: number;
+}>;