@substrat-run/control-plane-api 0.148.0 → 0.150.0

package/dist/store-rebuild.d.ts

@@ -0,0 +1,142 @@
+/**
+ * The platform's rebuild of every store script (#2309, D-63).
+ *
+ * A store script is built from a driver's release and from what the platform adds to every script:
+ * the entry, its verifier, the compatibility flags and both audiences' public key lists. A push, a
+ * serve or a rollout rebuilds the scripts it reaches. This walk rebuilds the rest when the platform's
+ * half changes, so a key dropped from a list (a revocation) or a verifier fix reaches every script
+ * whether or not its vertical is pushed again.
+ *
+ * What the platform builds from is one digest, `storePlatformFingerprint`. Each driver has one
+ * rebuild row in the directory: the fingerprint it converges on, the last vertical it walked, and
+ * the verticals it could not rebuild. A pass:
+ *
+ * - does nothing for a driver whose row is `done` at the current fingerprint (one directory read);
+ * - takes the row's lease (so two passes never walk one driver), then a **shared** hold on the
+ *   driver, the hold a push takes. A rollout or rollback holds the driver exclusively, so a pass
+ *   that meets one stops and the next pass continues. Under the shared hold no rollout can move a
+ *   script between this walk's read of the release a script runs and its upload, which would
+ *   upload the old release back over the new one;
+ * - rebuilds each script at the release it already runs (no `to`), so a rollout that paused part
+ *   way, or a rollback that has not finished, keeps every script where it is. A script that does not
+ *   exist is left alone, and one already built from the current fingerprint is not uploaded again;
+ * - walks the driver's canary script first, then each vertical that declared the driver, in slug
+ *   order, recording the cursor after each, then retries the verticals that failed. At most
+ *   `budget` scripts per pass, across drivers.
+ *
+ * The cursor is a slug, and a vertical behind it is not walked again. That is sound because every
+ * upload builds from the platform's current inputs: a vertical pushed or served after the walk
+ * passed its slug was built from the fingerprint the walk converges on, and one ahead of the cursor
+ * is reached and found current (no upload).
+ */
+import type { PlatformActorId, ScopeStoreNeed, StoreDriverRegistration } from '@substrat-run/contracts';
+import { type HostAdmin, type StoreDriverRebuildRecord } from '@substrat-run/kernel';
+import { type StoreReleaseBytes } from './store-releases.js';
+import type { ScopeStoreEnsurerOptions } from './store-script.js';
+type LoadRelease = ScopeStoreEnsurerOptions['loadRelease'];
+/**
+ * The subrequests a pass of the walk may spend by default. A Worker invocation gets 1000 on the
+ * paid plan; the scheduled pass shares its invocation with every other phase, so it takes half.
+ * A staff request runs alone and may take {@link STORE_REBUILD_REQUEST_SUBREQUESTS}.
+ */
+export declare const STORE_REBUILD_PASS_SUBREQUESTS = 500;
+export declare const STORE_REBUILD_REQUEST_SUBREQUESTS = 900;
+/** The scripts a pass with `subrequests` to spend may walk: {@link STORE_REBUILD_SUBREQUESTS_PER_SCRIPT} each. */
+export declare const storeRebuildBudget: (subrequests: number) => number;
+/**
+ * How long a `done` rebuild stands before a scheduled pass walks it again. A walk that is done
+ * proves every script was current when it read it; a control plane still serving the older inputs
+ * (a deploy rolling out gradually) can upload a script after that. Walking it again reads each
+ * script's settings and uploads only a script that is not current.
+ */
+export declare const STORE_REBUILD_REVERIFY_MS: number;
+/**
+ * How long after a fingerprint first appears a walk is not yet proof (#2309 review). While a
+ * control-plane deploy rolls out, the version still building from the older inputs can push a
+ * script after the walk read it. Only a walk that STARTED this long after its fingerprint was first
+ * walked towards proves every script current: a walk that spans passes reads its first scripts when
+ * it starts, so when it finishes says nothing about them. A done walk that started inside the window
+ * reads as settling, and the scheduled pass walks it again once the window has passed. A deploy that
+ * takes longer to roll out is followed by `store-driver rebuild`, or by a person's attestation that
+ * it has finished. The default; the control plane reads `STORE_REBUILD_SETTLE_MINUTES`.
+ */
+export declare const STORE_REBUILD_SETTLE_MS: number;
+interface StoreFleetRebuildDeps {
+    admin: HostAdmin;
+    actor: PlatformActorId;
+    drivers: readonly StoreDriverRegistration[];
+    /** What the platform builds every script from now (`storePlatformFingerprint`). */
+    fingerprint: string;
+    bytes: StoreReleaseBytes;
+    /** Rebuild one vertical's scripts of a driver at the release each runs, existing scripts only. */
+    ensure: (input: {
+        vertical: string;
+        stores: readonly ScopeStoreNeed[];
+        existingOnly: true;
+        loadRelease: LoadRelease;
+    }) => Promise<void | {
+        uploaded: number;
+    }>;
+    /** Rebuild the driver's canary script the same way. */
+    ensureCanary: (input: {
+        driver: string;
+        existingOnly: true;
+        loadRelease: LoadRelease;
+    }) => Promise<{
+        uploaded: boolean;
+    }>;
+    /** The most scripts this pass walks, across drivers. */
+    budget: number;
+    /**
+     * When a `done` rebuild is walked again: `always` (a person asked for this pass), or once it has
+     * stood {@link STORE_REBUILD_REVERIFY_MS} (the scheduled pass).
+     */
+    reverify: 'always' | 'when-due';
+    /** For a test: the clock the reverification is judged on. */
+    now?: () => number;
+    /** The settle window; {@link STORE_REBUILD_SETTLE_MS} by default. */
+    settleMs?: number;
+    /**
+     * A person's attestation that no control-plane deploy is rolling out: the walk starts over from
+     * its first script, and once done proves every script current whatever the settle window says.
+     * The actor attesting. Only for one driver at a time, from a staff request.
+     */
+    attestedBy?: string;
+}
+export interface StoreDriverRebuildPass {
+    driver: string;
+    /**
+     * `current`: done at this fingerprint, nothing walked. `busy`: another pass holds the lease.
+     * `held`: a rollout or rollback holds the driver; the next pass continues. `paused`: the budget ran
+     * out with scripts left. `done`: every script walked, none failed. `failing`: every script walked,
+     * some failed; they are retried on the next pass. `error`: the pass could not walk the driver.
+     */
+    outcome: 'current' | 'busy' | 'held' | 'paused' | 'done' | 'failing' | 'error';
+    /** The scripts walked this pass: a vertical's slug, or the canary's marker. */
+    walked: string[];
+    /** The ones of those that were uploaded. */
+    uploaded: string[];
+    rebuild?: StoreDriverRebuildRecord;
+    error?: string;
+}
+/** One pass of the walk over every driver, within `budget` scripts. */
+export declare function runStoreFleetRebuild(deps: StoreFleetRebuildDeps): Promise<StoreDriverRebuildPass[]>;
+/**
+ * Whether every script of a driver is built from what the platform builds from now, and which may
+ * not be: while the rebuild is towards another fingerprint (or none ran yet), the canary and every
+ * vertical that declared the driver; while it runs, the ones ahead of its cursor and the ones that
+ * failed. A push or serve may have rebuilt some of them since, and the walk then finds them
+ * current, so `pending` may over-count, never under-count. A walk that is done but not settled (see
+ * {@link STORE_REBUILD_SETTLE_MS}) is not current: `settling` says when it is walked again. `nextWalk`
+ * is when the scheduled pass walks a done rebuild again. A walk under way at lists an earlier walk
+ * already proved is `current` and `reverifying` until a script it reaches fails.
+ */
+export declare function storeRebuildStatus(rebuild: StoreDriverRebuildRecord | undefined, fingerprint: string, declared: readonly string[], settleMs?: number): {
+    current: boolean;
+    pending: string[];
+    settling?: string;
+    nextWalk?: string;
+    reverifying?: boolean;
+};
+export {};
+//# sourceMappingURL=store-rebuild.d.ts.map

package/dist/store-rebuild.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store-rebuild.d.ts","sourceRoot":"","sources":["../src/store-rebuild.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAGxG,OAAO,EAAQ,KAAK,SAAS,EAAkC,KAAK,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAC3H,OAAO,EAAyC,KAAK,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpG,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAElE,KAAK,WAAW,GAAG,wBAAwB,CAAC,aAAa,CAAC,CAAC;AAQ3D;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,MAAM,CAAC;AAClD,eAAO,MAAM,iCAAiC,MAAM,CAAC;AACrD,kHAAkH;AAClH,eAAO,MAAM,kBAAkB,gBAAiB,MAAM,KAAG,MACoB,CAAC;AAC9E;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,QAAmB,CAAC;AAC1D;;;;;;;;;GASG;AACH,eAAO,MAAM,uBAAuB,QAAc,CAAC;AAoBnD,UAAU,qBAAqB;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,eAAe,CAAC;IACvB,OAAO,EAAE,SAAS,uBAAuB,EAAE,CAAC;IAC5C,mFAAmF;IACnF,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,iBAAiB,CAAC;IACzB,kGAAkG;IAClG,MAAM,EAAE,CAAC,KAAK,EAAE;QACd,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,SAAS,cAAc,EAAE,CAAC;QAClC,YAAY,EAAE,IAAI,CAAC;QACnB,WAAW,EAAE,WAAW,CAAC;KAC1B,KAAK,OAAO,CAAC,IAAI,GAAG;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3C,uDAAuD;IACvD,YAAY,EAAE,CAAC,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,IAAI,CAAC;QAAC,WAAW,EAAE,WAAW,CAAA;KAAE,KAAK,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC1H,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,QAAQ,EAAE,QAAQ,GAAG,UAAU,CAAC;IAChC,6DAA6D;IAC7D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,OAAO,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IAC/E,+EAA+E;IAC/E,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,uEAAuE;AACvE,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAqBzG;AA0HD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,wBAAwB,GAAG,SAAS,EAC7C,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,GAAE,MAAgC,GACzC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,CAWtG"}

package/dist/store-rebuild.js

@@ -0,0 +1,223 @@
+import { SCOPE_STORE_CANARY } from '@substrat-run/contracts';
+import { ControlPlaneError } from '@substrat-run/control-plane-client';
+import { ulid } from '@substrat-run/kernel';
+import { PUSH_HOLD_TTL_MS, storedReleaseLoader } from './store-releases.js';
+/**
+ * What one script of the walk costs in subrequests: the hold renewal, the script's settings read,
+ * the release's bytes (read once per release a pass meets, counted for every script to stay on the
+ * safe side), at most one upload, and the progress write.
+ */
+const STORE_REBUILD_SUBREQUESTS_PER_SCRIPT = 5;
+/**
+ * The subrequests a pass of the walk may spend by default. A Worker invocation gets 1000 on the
+ * paid plan; the scheduled pass shares its invocation with every other phase, so it takes half.
+ * A staff request runs alone and may take {@link STORE_REBUILD_REQUEST_SUBREQUESTS}.
+ */
+export const STORE_REBUILD_PASS_SUBREQUESTS = 500;
+export const STORE_REBUILD_REQUEST_SUBREQUESTS = 900;
+/** The scripts a pass with `subrequests` to spend may walk: {@link STORE_REBUILD_SUBREQUESTS_PER_SCRIPT} each. */
+export const storeRebuildBudget = (subrequests) => Math.max(1, Math.floor(subrequests / STORE_REBUILD_SUBREQUESTS_PER_SCRIPT));
+/**
+ * How long a `done` rebuild stands before a scheduled pass walks it again. A walk that is done
+ * proves every script was current when it read it; a control plane still serving the older inputs
+ * (a deploy rolling out gradually) can upload a script after that. Walking it again reads each
+ * script's settings and uploads only a script that is not current.
+ */
+export const STORE_REBUILD_REVERIFY_MS = 24 * 60 * 60_000;
+/**
+ * How long after a fingerprint first appears a walk is not yet proof (#2309 review). While a
+ * control-plane deploy rolls out, the version still building from the older inputs can push a
+ * script after the walk read it. Only a walk that STARTED this long after its fingerprint was first
+ * walked towards proves every script current: a walk that spans passes reads its first scripts when
+ * it starts, so when it finishes says nothing about them. A done walk that started inside the window
+ * reads as settling, and the scheduled pass walks it again once the window has passed. A deploy that
+ * takes longer to roll out is followed by `store-driver rebuild`, or by a person's attestation that
+ * it has finished. The default; the control plane reads `STORE_REBUILD_SETTLE_MINUTES`.
+ */
+export const STORE_REBUILD_SETTLE_MS = 60 * 60_000;
+/**
+ * Whether a walk proves every script current once done: it started after the settle window, or a
+ * person attested, when it started, that no control-plane deploy was rolling out.
+ */
+const settled = (rebuild, settleMs) => rebuild.attestedBy !== null || Date.parse(rebuild.startedAt) - Date.parse(rebuild.fingerprintSince) >= settleMs;
+const finishedMs = (rebuild) => Date.parse(rebuild.finishedAt ?? rebuild.updatedAt);
+/** How long a pass's lease on a driver's rebuild lives between progress writes. */
+const STORE_REBUILD_LEASE_TTL_MS = 10 * 60_000;
+/**
+ * The binding an unreadable vertical's store is walked under. A script's name and build take the
+ * vertical and the driver only, so the binding names nothing the upload uses.
+ */
+const UNNAMED_BINDING = 'UNNAMED';
+/** The cursor once the canary is walked: before every slug, after "not started" (null). */
+const CANARY_WALKED = '';
+const message = (e) => (e instanceof Error ? e.message : String(e));
+/** One pass of the walk over every driver, within `budget` scripts. */
+export async function runStoreFleetRebuild(deps) {
+    // One loader for the pass: a release's bytes are read once, however many scripts run it.
+    const read = storedReleaseLoader({ admin: deps.admin, actor: deps.actor, bytes: deps.bytes });
+    const cache = new Map();
+    const loadRelease = (input) => {
+        const key = `${input.driver}/${input.sha256 ?? ''}`;
+        let hit = cache.get(key);
+        if (!hit)
+            cache.set(key, (hit = read(input)));
+        return hit;
+    };
+    let budget = deps.budget;
+    const passes = [];
+    for (const registration of deps.drivers) {
+        // One driver's failure (its directory read, its lease write) never stops the others.
+        const pass = await rebuildDriver(deps, registration, loadRelease, budget).catch((e) => ({ driver: registration.driver, outcome: 'error', walked: [], uploaded: [], error: message(e) }));
+        budget -= pass.walked.length;
+        passes.push(pass);
+    }
+    return passes;
+}
+async function rebuildDriver(deps, registration, loadRelease, budget) {
+    const { admin, actor, fingerprint } = deps;
+    const driver = registration.driver;
+    const walked = [];
+    const uploaded = [];
+    const result = (outcome, rebuild) => ({ driver, outcome, walked, uploaded, ...(rebuild ? { rebuild } : {}) });
+    const stood = await admin.storeDriverRebuild(actor, driver);
+    const doneHere = stood?.fingerprint === fingerprint && stood.state === 'done';
+    const now = (deps.now ?? Date.now)();
+    const settleMs = deps.settleMs ?? STORE_REBUILD_SETTLE_MS;
+    const restart = doneHere && (deps.reverify === 'always' || now >= nextAutomaticWalk(stood, settleMs));
+    if (doneHere && !restart && deps.attestedBy === undefined)
+        return result('current', stood);
+    if (budget <= 0)
+        return result('paused', stood);
+    const holder = ulid();
+    const claimed = await admin.claimStoreDriverRebuild(actor, {
+        driver,
+        fingerprint,
+        holder,
+        ttlMs: STORE_REBUILD_LEASE_TTL_MS,
+        restart,
+        ...(deps.attestedBy !== undefined ? { attestedBy: deps.attestedBy } : {}),
+    });
+    if (!claimed)
+        return result('busy', stood);
+    let rebuild = claimed;
+    let cursor = claimed.cursor;
+    const failures = new Map(claimed.failures.map((f) => [f.vertical, f.detail]));
+    const advance = async (done, release = false) => {
+        const list = [...failures].map(([vertical, detail]) => ({ vertical, detail }));
+        // A done walk that started after the settle window (or was attested) proves every script current.
+        const verified = done && settled(rebuild, settleMs);
+        rebuild = await admin.advanceStoreDriverRebuild(actor, { driver, fingerprint, holder, cursor, failures: list, done, verified, ttlMs: STORE_REBUILD_LEASE_TTL_MS, release });
+    };
+    // The lease is dropped once, however the walk ends; the end of a whole walk drops it itself.
+    let leaseDropped = false;
+    let held = false;
+    try {
+        held = await admin.holdStoreDriver(actor, { driver, holder, mode: 'shared', ttlMs: PUSH_HOLD_TTL_MS });
+        if (!held)
+            return result('held', rebuild);
+        const declared = await admin.storeDriverDeclarations(actor, driver);
+        const byVertical = new Map(declared.verticals.map((v) => [v.vertical, v.stores.map((s) => ({ binding: s.binding, driver }))]));
+        // A vertical whose stores cannot be named may still run a script of this driver. Its name is the
+        // vertical's and the driver's alone, so it is walked by name: rebuilt if it exists, and left
+        // alone if not (no script, nothing that accepts a key). Only a failure to do that is a failure.
+        for (const vertical of declared.unreadable) {
+            if (!byVertical.has(vertical))
+                byVertical.set(vertical, [{ binding: UNNAMED_BINDING, driver }]);
+        }
+        // A failure no longer to be had: the vertical declares the driver no more, and is readable.
+        for (const vertical of [...failures.keys()]) {
+            if (vertical !== SCOPE_STORE_CANARY && !byVertical.has(vertical))
+                failures.delete(vertical);
+        }
+        const ahead = aheadOf(cursor, [...byVertical.keys()].sort());
+        const walking = new Set(ahead);
+        const steps = [
+            ...ahead.map((key) => ({ key, walk: true })),
+            // The retries: a failure behind the cursor that can be rebuilt now.
+            ...[...failures.keys()]
+                .filter((key) => !walking.has(key) && (key === SCOPE_STORE_CANARY || byVertical.has(key)))
+                .map((key) => ({ key, walk: false })),
+        ];
+        for (const step of steps) {
+            if (walked.length >= budget)
+                break;
+            if (!(await admin.holdStoreDriver(actor, { driver, holder, mode: 'shared', ttlMs: PUSH_HOLD_TTL_MS, renew: true }))) {
+                // The hold lapsed, and a rollout may hold the driver now: walk nothing more. The lease may
+                // have lapsed with it, and then another pass owns the row; what was walked is recorded.
+                return result('held', rebuild);
+            }
+            walked.push(step.key);
+            try {
+                const up = step.key === SCOPE_STORE_CANARY
+                    ? (await deps.ensureCanary({ driver, existingOnly: true, loadRelease })).uploaded
+                    : await deps.ensure({ vertical: step.key, stores: byVertical.get(step.key), existingOnly: true, loadRelease }).then((r) => (r ? r.uploaded > 0 : true));
+                if (up)
+                    uploaded.push(step.key);
+                failures.delete(step.key);
+            }
+            catch (e) {
+                failures.set(step.key, message(e));
+                await admin
+                    .recordOpsFailure({
+                    actor,
+                    operation: 'store-rebuild',
+                    stage: 'store-upload',
+                    vertical: step.key === SCOPE_STORE_CANARY ? null : step.key,
+                    status: e instanceof ControlPlaneError ? e.status : 502,
+                    message: `store script rebuild of '${driver}' for ${step.key === SCOPE_STORE_CANARY ? 'its canary' : step.key}: ${message(e)}`,
+                })
+                    .catch(() => undefined);
+            }
+            if (step.walk)
+                cursor = step.key === SCOPE_STORE_CANARY ? CANARY_WALKED : step.key;
+            await advance(false);
+        }
+        const left = steps.length - walked.length;
+        const done = left === 0 && failures.size === 0;
+        leaseDropped = true;
+        await advance(done, true);
+        return result(left > 0 ? 'paused' : done ? 'done' : 'failing', rebuild);
+    }
+    finally {
+        if (held)
+            await admin.releaseStoreDriverHold(actor, { driver, holder }).catch(() => undefined);
+        if (!leaseDropped)
+            await advance(false, true).catch(() => undefined);
+    }
+}
+/** The scripts ahead of a walk's cursor: the canary before any vertical, then each slug after it. */
+function aheadOf(cursor, verticals) {
+    return [...(cursor === null ? [SCOPE_STORE_CANARY] : []), ...verticals.filter((v) => cursor === null || v > cursor)];
+}
+/**
+ * When the scheduled pass walks a done rebuild again: once its settle window has passed, for a walk
+ * that finished inside it; else a day after it finished.
+ */
+function nextAutomaticWalk(rebuild, settleMs) {
+    return settled(rebuild, settleMs) ? finishedMs(rebuild) + STORE_REBUILD_REVERIFY_MS : Date.parse(rebuild.fingerprintSince) + settleMs;
+}
+/**
+ * Whether every script of a driver is built from what the platform builds from now, and which may
+ * not be: while the rebuild is towards another fingerprint (or none ran yet), the canary and every
+ * vertical that declared the driver; while it runs, the ones ahead of its cursor and the ones that
+ * failed. A push or serve may have rebuilt some of them since, and the walk then finds them
+ * current, so `pending` may over-count, never under-count. A walk that is done but not settled (see
+ * {@link STORE_REBUILD_SETTLE_MS}) is not current: `settling` says when it is walked again. `nextWalk`
+ * is when the scheduled pass walks a done rebuild again. A walk under way at lists an earlier walk
+ * already proved is `current` and `reverifying` until a script it reaches fails.
+ */
+export function storeRebuildStatus(rebuild, fingerprint, declared, settleMs = STORE_REBUILD_SETTLE_MS) {
+    if (!rebuild || rebuild.fingerprint !== fingerprint)
+        return { current: false, pending: [SCOPE_STORE_CANARY, ...declared] };
+    if (rebuild.state === 'done') {
+        const nextWalk = new Date(nextAutomaticWalk(rebuild, settleMs)).toISOString();
+        return settled(rebuild, settleMs) ? { current: true, pending: [], nextWalk } : { current: false, pending: [], settling: nextWalk, nextWalk };
+    }
+    // A walk under way that re-checks a fleet a walk at these lists already proved current: still
+    // current, unless a script it reached is known to have failed.
+    if (rebuild.verifiedAt !== null && rebuild.failures.length === 0)
+        return { current: true, pending: [], reverifying: true };
+    const pending = new Set([...aheadOf(rebuild.cursor, declared), ...rebuild.failures.map((f) => f.vertical)]);
+    return { current: false, pending: [...pending] };
+}
+//# sourceMappingURL=store-rebuild.js.map

package/dist/store-rebuild.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store-rebuild.js","sourceRoot":"","sources":["../src/store-rebuild.ts"],"names":[],"mappings":"AAgCA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,EAAE,IAAI,EAAiF,MAAM,sBAAsB,CAAC;AAC3H,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAA0B,MAAM,qBAAqB,CAAC;AAKpG;;;;GAIG;AACH,MAAM,oCAAoC,GAAG,CAAC,CAAC;AAC/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,GAAG,CAAC;AAClD,MAAM,CAAC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AACrD,kHAAkH;AAClH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,WAAmB,EAAU,EAAE,CAChE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,oCAAoC,CAAC,CAAC,CAAC;AAC9E;;;;;GAKG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;AAC1D;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,MAAM,CAAC;AACnD;;;GAGG;AACH,MAAM,OAAO,GAAG,CAAC,OAAiC,EAAE,QAAgB,EAAE,EAAE,CACtE,OAAO,CAAC,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,QAAQ,CAAC;AAClH,MAAM,UAAU,GAAG,CAAC,OAAiC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;AAC9G,mFAAmF;AACnF,MAAM,0BAA0B,GAAG,EAAE,GAAG,MAAM,CAAC;AAE/C;;;GAGG;AACH,MAAM,eAAe,GAAG,SAAS,CAAC;AAElC,2FAA2F;AAC3F,MAAM,aAAa,GAAG,EAAE,CAAC;AAsDzB,MAAM,OAAO,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAE7E,uEAAuE;AACvE,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAA2B;IACpE,yFAAyF;IACzF,MAAM,IAAI,GAAG,mBAAmB,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9F,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmC,CAAC;IACzD,MAAM,WAAW,GAAgB,CAAC,KAAK,EAAE,EAAE;QACzC,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACpD,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,GAAG;YAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IACzB,MAAM,MAAM,GAA6B,EAAE,CAAC;IAC5C,KAAK,MAAM,YAAY,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACxC,qFAAqF;QACrF,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC,KAAK,CAC7E,CAAC,CAAU,EAA0B,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CACzI,CAAC;QACF,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,IAA2B,EAC3B,YAAqC,EACrC,WAAwB,EACxB,MAAc;IAEd,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IACnC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,CAAC,OAA0C,EAAE,OAAkC,EAA0B,EAAE,CACxH,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,KAAK,EAAE,WAAW,KAAK,WAAW,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,CAAC;IAC9E,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,uBAAuB,CAAC;IAC1D,MAAM,OAAO,GAAG,QAAQ,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,IAAI,iBAAiB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IACtG,IAAI,QAAQ,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC3F,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEhD,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC;IACtB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,uBAAuB,CAAC,KAAK,EAAE;QACzD,MAAM;QACN,WAAW;QACX,MAAM;QACN,KAAK,EAAE,0BAA0B;QACjC,OAAO;QACP,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1E,CAAC,CAAC;IACH,IAAI,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,IAAI,OAAO,GAAG,OAAO,CAAC;IACtB,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC5B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,KAAK,EAAE,IAAa,EAAE,OAAO,GAAG,KAAK,EAAE,EAAE;QACvD,MAAM,IAAI,GAAgC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5G,kGAAkG;QAClG,MAAM,QAAQ,GAAG,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACpD,OAAO,GAAG,MAAM,KAAK,CAAC,yBAAyB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,0BAA0B,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9K,CAAC,CAAC;IACF,6FAA6F;IAC7F,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAC;QACvG,IAAI,CAAC,IAAI;YAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/H,iGAAiG;QACjG,6FAA6F;QAC7F,gGAAgG;QAChG,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC3C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAClG,CAAC;QACD,4FAA4F;QAC5F,KAAK,MAAM,QAAQ,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC5C,IAAI,QAAQ,KAAK,kBAAkB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9F,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAqC;YAC9C,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,oEAAoE;YACpE,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;iBACpB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,kBAAkB,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;iBACzF,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACxC,CAAC;QACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM;gBAAE,MAAM;YACnC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpH,2FAA2F;gBAC3F,wFAAwF;gBACxF,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACjC,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,KAAK,kBAAkB;oBACxC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,QAAQ;oBACjF,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAE,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC3J,IAAI,EAAE;oBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,KAAK;qBACR,gBAAgB,CAAC;oBAChB,KAAK;oBACL,SAAS,EAAE,eAAe;oBAC1B,KAAK,EAAE,cAAc;oBACrB,QAAQ,EAAE,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG;oBAC3D,MAAM,EAAE,CAAC,YAAY,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG;oBACvD,OAAO,EAAE,4BAA4B,MAAM,SAAS,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE;iBAC/H,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;YACD,IAAI,IAAI,CAAC,IAAI;gBAAE,MAAM,GAAG,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YACnF,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;QAC/C,YAAY,GAAG,IAAI,CAAC;QACpB,MAAM,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC1B,OAAO,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACT,IAAI,IAAI;YAAE,MAAM,KAAK,CAAC,sBAAsB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/F,IAAI,CAAC,YAAY;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,qGAAqG;AACrG,SAAS,OAAO,CAAC,MAAqB,EAAE,SAA4B;IAClE,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;AACvH,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,OAAiC,EAAE,QAAgB;IAC5E,OAAO,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,QAAQ,CAAC;AACxI,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAA6C,EAC7C,WAAmB,EACnB,QAA2B,EAC3B,QAAQ,GAAW,uBAAuB;IAE1C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,WAAW,KAAK,WAAW;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,kBAAkB,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;IAC3H,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAC9E,OAAO,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;IAC/I,CAAC;IACD,8FAA8F;IAC9F,+DAA+D;IAC/D,IAAI,OAAO,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC3H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC5G,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,CAAC;AACnD,CAAC"}

package/dist/store-releases.d.ts

@@ -1,38 +1,38 @@
 /**
- * A store kind's releases: stored, canaried, rolled out and rolled back (D-63, #2300).
+ * A store driver's releases: stored, canaried, rolled out and rolled back (D-63, #2300).
  *
- * The bytes live outside the directory, content-addressed under `<kind>/<sha256>`, where the
- * sha256 is `storeKindReleaseDigest` of the bundle, computed here when it arrives and again by
+ * The bytes live outside the directory, content-addressed under `<driver>/<sha256>`, where the
+ * sha256 is `storeDriverReleaseDigest` of the bundle, computed here when it arrives and again by
  * the ensurer before every upload of it, restore and rollback included. The directory holds who
  * uploaded which release from which commit, every rollout, and the current good release
- * (`store-kind-releases.ts` in the kernel).
+ * (`store-driver-releases.ts` in the kernel).
  *
- * A rollout is one request across a fleet. It holds the kind exclusively for its whole length,
+ * A rollout is one request across a fleet. It holds the driver exclusively for its whole length,
  * renewing the hold before every upload, so no push or serve rebuilds a script between the
  * rollout's read and its upload, and two rollouts never interleave. It canaries the release on the
  * platform's own object first, and no vertical's script moves unless every reserved operation
- * answered there. Then it moves each vertical that ever declared the kind, at most
- * {@link ROLLOUT_VERTICALS_PER_REQUEST} uploads per request. A bigger fleet ends the request
- * `paused`, and posting the same rollout (or rollback) again continues it, walking past the scripts
- * already there. A request that stops on a failure ends `failed` or `diverged`. One request walks at
- * most {@link ROLLOUT_FLEET_MAX} verticals, a rollback included, until #2312.
+ * answered there. Then it moves each vertical that ever declared the driver, at most
+ * {@link ROLLOUT_VERTICALS_PER_REQUEST} uploads and {@link ROLLOUT_WALKS_PER_REQUEST} verticals
+ * walked per request. A bigger fleet ends the request `paused`, and posting the same rollout (or
+ * rollback) again continues it from the attempt's walk ledger (#2312), so a fleet of any size
+ * converges, a rollback included. A request that stops on a failure ends `failed` or `diverged`.
  */
-import { type PlatformActorId, type ReservedStoreOperation, type ScopeStoreNeed, type StoreKindRegistration } from '@substrat-run/contracts';
-import { type HostAdmin, type StoreKindRolloutAction, type StoreKindRolloutRecord } from '@substrat-run/kernel';
-import type { ScopeStoreEnsurerOptions, StoreKindReleaseArtifact } from './store-script.js';
-/** Where a release's bytes are kept: an object store keyed by kind and sha256. */
+import { type PlatformActorId, type ReservedStoreOperation, type ScopeStoreNeed, type StoreDriverRegistration } from '@substrat-run/contracts';
+import { type HostAdmin, type StoreDriverRolloutAction, type StoreDriverRolloutRecord } from '@substrat-run/kernel';
+import type { ScopeStoreEnsurerOptions, StoreDriverReleaseArtifact } from './store-script.js';
+/** Where a release's bytes are kept: an object store keyed by driver and sha256. */
 export interface StoreReleaseBytes {
     get(key: string): Promise<Uint8Array | undefined>;
     put(key: string, bytes: Uint8Array): Promise<void>;
 }
 /** The key a release's bytes are stored under. */
-export declare const storeReleaseKey: (kind: string, sha256: string) => string;
+export declare const storeReleaseKey: (driver: string, sha256: string) => string;
 /** How long a rollout's exclusive hold lives between renewals. */
 export declare const ROLLOUT_HOLD_TTL_MS: number;
 /**
- * How long a push's or a serve's shared hold lives. It is renewed before each kind's upload; one
- * kind's upload (a settings read and at most one upload per script) must fit inside it, or a
- * rollout could take the kind before that upload lands. The same lease as a rollout's.
+ * How long a push's or a serve's shared hold lives. It is renewed before each driver's upload; one
+ * driver's upload (a settings read and at most one upload per script) must fit inside it, or a
+ * rollout could take the driver before that upload lands. The same lease as a rollout's.
  */
 export declare const PUSH_HOLD_TTL_MS: number;
 /**
@@ -42,32 +42,33 @@ export declare const PUSH_HOLD_TTL_MS: number;
  */
 export declare const ROLLOUT_VERTICALS_PER_REQUEST = 100;
 /**
- * The most verticals one rollout walks. Every one costs a settings read and a hold renewal even
- * when it is already at the release, so a bigger fleet would outrun one invocation; it is refused
- * (409) rather than left unable to converge, and that blocks a rollback too. Raising it needs a
- * cursor on the rollout row: #2312.
+ * The verticals one rollout request walks. Every one costs a settings read and a hold renewal even
+ * when it is already at the release, so a request that reaches this many ends `paused` and the
+ * next continues from the walk ledger (#2312). It also bounds what a failure returns: an attempt
+ * that reached more ends `diverged` without returning any, and a rollback, paged the same way,
+ * converges them.
  */
-export declare const ROLLOUT_FLEET_MAX = 300;
+export declare const ROLLOUT_WALKS_PER_REQUEST = 300;
 type LoadRelease = ScopeStoreEnsurerOptions['loadRelease'];
 /**
  * A release bundle as uploaded or as stored, decoded and judged: the upload body's shape, the
- * decoded size cap, and the Python kind's module rule. Refusals are 422 (413 for the size).
+ * decoded size cap, and the Python driver's module rule. Refusals are 422 (413 for the size).
  */
-export declare function decodeStoreKindRelease(raw: unknown): Omit<StoreKindReleaseArtifact, 'sha256'>;
+export declare function decodeStoreDriverRelease(raw: unknown): Omit<StoreDriverReleaseArtifact, 'sha256'>;
 /**
  * Store a release's bytes and record them (D-63: bytes may be published before the pin lands).
  * The bytes go first, so a recorded release always has bytes; a retry of the same bundle writes
  * the same key and records nothing new.
  */
-export declare function storeKindRelease(input: {
+export declare function storeDriverRelease(input: {
     admin: HostAdmin;
     actor: PlatformActorId;
     bytes: StoreReleaseBytes;
-    kind: string;
+    driver: string;
     body: unknown;
-}): Promise<import("@substrat-run/kernel").StoreKindReleaseRecord>;
+}): Promise<import("@substrat-run/kernel").StoreDriverReleaseRecord>;
 /**
- * The ensurer's loader over the stored bytes: a named release, or the kind's current good one.
+ * The ensurer's loader over the stored bytes: a named release, or the driver's current good one.
  * Each answer carries the sha256 it was read as, which the ensurer holds the bytes to.
  */
 export declare function storedReleaseLoader(input: {
@@ -97,14 +98,14 @@ export declare function canaryStoreRelease(input: {
     scriptName: string;
     binding?: string;
 }): Promise<void>;
-export interface StoreKindRolloutDeps {
+export interface StoreDriverRolloutDeps {
     admin: HostAdmin;
     actor: PlatformActorId;
-    registration: StoreKindRegistration;
+    registration: StoreDriverRegistration;
     sha256: string;
-    action: StoreKindRolloutAction;
+    action: StoreDriverRolloutAction;
     /**
-     * Move one vertical's scripts of the kind to `to` (the push's ensurer, with `to`). Answers
+     * Move one vertical's scripts of the driver to `to` (the push's ensurer, with `to`). Answers
      * whether anything was uploaded: false for a script already there, built the same way.
      */
     ensure: (input: {
@@ -112,36 +113,46 @@ export interface StoreKindRolloutDeps {
         stores: readonly ScopeStoreNeed[];
         to: string;
     }) => Promise<boolean>;
-    /** Put `to` in the kind's canary script; answers its name. */
+    /** Put `to` in the driver's canary script; answers its name. */
     ensureCanary: (to: string) => Promise<{
         scriptName: string;
     }>;
     callCanary: CanaryCall;
     /** For a test: a smaller batch than {@link ROLLOUT_VERTICALS_PER_REQUEST}. */
     perRequest?: number;
+    /** For a test: fewer verticals walked per request than {@link ROLLOUT_WALKS_PER_REQUEST}. */
+    perRequestWalks?: number;
 }
-export interface StoreKindRolloutResult {
-    rollout: StoreKindRolloutRecord;
+export interface StoreDriverRolloutResult {
+    rollout: StoreDriverRolloutRecord;
     /** The verticals whose scripts this request uploaded. */
     moved: string[];
     /** False when verticals remain: post the same rollout again. */
     complete: boolean;
+    /** The verticals not walked yet; 0 once complete. */
+    remaining: number;
 }
 /**
- * Move every script of a kind to a stored, pinned release (D-63): hold, canary, then each
- * vertical. A failure after a vertical moved returns every vertical it reached to the release
- * the rollout started from, rehashing those bytes as it uploads them, and ends `failed`. It ends
- * `diverged` instead, naming what is where, when there is nothing to return to (a kind's first
- * rollout), when returning would lower the stored schema, or when a return fails: the next
- * rollout or rollback converges every script. The hold is renewed before every upload, the
- * returns included, and a hold that cannot be renewed stops everything at once: another holder
- * may be moving the same scripts, so nothing is returned either. The hold is released however it
- * ends.
+ * Move every script of a driver to a stored, pinned release (D-63): hold, canary, then each
+ * vertical. A failure after a vertical moved returns every vertical the attempt reached to the
+ * release the rollout started from, rehashing those bytes as it uploads them, and ends `failed`.
+ * It ends `diverged` instead, naming what is where, when there is nothing to return to (a driver's
+ * first rollout), when returning would lower the stored schema, when the attempt reached more
+ * verticals than one request returns, or when a return fails: the next rollout or rollback
+ * converges every script. The hold is renewed before every upload, the returns included, and a
+ * hold that cannot be renewed stops everything at once: another holder may be moving the same
+ * scripts, so nothing is returned either. The hold is released however it ends.
  *
- * Each request uploads at most `perRequest` scripts. A vertical already at the release is
- * walked past without counting, so posting the same rollout again continues where the last
- * request stopped, and a fleet of any size within {@link ROLLOUT_FLEET_MAX} converges.
+ * A fleet bigger than one request is paged (#2312). Each request walks at most
+ * {@link ROLLOUT_WALKS_PER_REQUEST} verticals and uploads at most `perRequest` scripts; one that runs out
+ * ends `paused`, and its end records what it walked in the attempt's ledger. Posting the same
+ * rollout (or rollback) again continues the attempt and walks only declarations the ledger does
+ * not hold, in (vertical, binding) order: a vertical declared between two requests, wherever it
+ * sorts, is walked by the next one, and one walked is never walked again. A request ends `good`
+ * only when the pending list it read under its exclusive hold was the whole of it and it walked
+ * every vertical on it: no push can create a script while the hold is held, so nothing declared
+ * can be left on another release.
  */
-export declare function runStoreKindRollout(deps: StoreKindRolloutDeps): Promise<StoreKindRolloutResult>;
+export declare function runStoreDriverRollout(deps: StoreDriverRolloutDeps): Promise<StoreDriverRolloutResult>;
 export {};
 //# sourceMappingURL=store-releases.d.ts.map

package/dist/store-releases.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"store-releases.d.ts","sourceRoot":"","sources":["../src/store-releases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,sBAAsB,EAC3B,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC3B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAoB,KAAK,SAAS,EAAE,KAAK,sBAAsB,EAAE,KAAK,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAElI,OAAO,KAAK,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAE5F,kFAAkF;AAClF,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAClD,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpD;AAED,kDAAkD;AAClD,eAAO,MAAM,eAAe,SAAU,MAAM,UAAU,MAAM,KAAG,MAA6B,CAAC;AAE7F,kEAAkE;AAClE,eAAO,MAAM,mBAAmB,QAAc,CAAC;AAC/C;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,QAAc,CAAC;AAC5C;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,MAAM,CAAC;AACjD;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC,KAAK,WAAW,GAAG,wBAAwB,CAAC,aAAa,CAAC,CAAC;AAY3D;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC,wBAAwB,EAAE,QAAQ,CAAC,CAqB7F;AAkBD;;;;GAIG;AACH,wBAAsB,gBAAgB,CAAC,KAAK,EAAE;IAC5C,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,eAAe,CAAC;IACvB,KAAK,EAAE,iBAAiB,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;CACf,kEAWA;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,KAAK,EAAE,iBAAiB,CAAA;CAAE,GAAG,WAAW,CAc9H;AAED,+DAA+D;AAC/D,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,EAAE,sBAAsB,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEtJ;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,YAAa,MAAM,KAAG,MAA6E,CAAC;AAIjI;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBzH;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,eAAe,CAAC;IACvB,YAAY,EAAE,qBAAqB,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,sBAAsB,CAAC;IAC/B;;;OAGG;IACH,MAAM,EAAE,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,SAAS,cAAc,EAAE,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACzG,8DAA8D;IAC9D,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9D,UAAU,EAAE,UAAU,CAAC;IACvB,8EAA8E;IAC9E,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,sBAAsB,CAAC;IAChC,yDAAyD;IACzD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,QAAQ,EAAE,OAAO,CAAC;CACnB;AAaD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,oBAAoB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAiErG"}
+{"version":3,"file":"store-releases.d.ts","sourceRoot":"","sources":["../src/store-releases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,sBAAsB,EAC3B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAC7B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAGL,KAAK,SAAS,EACd,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAE9B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE9F,oFAAoF;AACpF,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAClD,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpD;AAED,kDAAkD;AAClD,eAAO,MAAM,eAAe,WAAY,MAAM,UAAU,MAAM,KAAG,MAA+B,CAAC;AAEjG,kEAAkE;AAClE,eAAO,MAAM,mBAAmB,QAAc,CAAC;AAC/C;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,QAAc,CAAC;AAC5C;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,MAAM,CAAC;AACjD;;;;;;GAMG;AACH,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C,KAAK,WAAW,GAAG,wBAAwB,CAAC,aAAa,CAAC,CAAC;AAY3D;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC,0BAA0B,EAAE,QAAQ,CAAC,CAqBjG;AAkBD;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAC9C,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,eAAe,CAAC;IACvB,KAAK,EAAE,iBAAiB,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;CACf,oEAWA;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,KAAK,EAAE,iBAAiB,CAAA;CAAE,GAAG,WAAW,CAc9H;AAED,+DAA+D;AAC/D,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,EAAE,sBAAsB,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEtJ;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,YAAa,MAAM,KAAG,MAA6E,CAAC;AAIjI;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBzH;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,eAAe,CAAC;IACvB,YAAY,EAAE,uBAAuB,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,wBAAwB,CAAC;IACjC;;;OAGG;IACH,MAAM,EAAE,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,SAAS,cAAc,EAAE,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACzG,gEAAgE;IAChE,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9D,UAAU,EAAE,UAAU,CAAC;IACvB,8EAA8E;IAC9E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6FAA6F;IAC7F,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,OAAO,EAAE,wBAAwB,CAAC;IAClC,yDAAyD;IACzD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,QAAQ,EAAE,OAAO,CAAC;IAClB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;CACnB;AAaD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqE3G"}