@substrat-run/control-plane-api 0.147.0 → 0.149.0

package/dist/store-rebuild.js

@@ -0,0 +1,223 @@
+import { SCOPE_STORE_CANARY } from '@substrat-run/contracts';
+import { ControlPlaneError } from '@substrat-run/control-plane-client';
+import { ulid } from '@substrat-run/kernel';
+import { PUSH_HOLD_TTL_MS, storedReleaseLoader } from './store-releases.js';
+/**
+ * What one script of the walk costs in subrequests: the hold renewal, the script's settings read,
+ * the release's bytes (read once per release a pass meets, counted for every script to stay on the
+ * safe side), at most one upload, and the progress write.
+ */
+const STORE_REBUILD_SUBREQUESTS_PER_SCRIPT = 5;
+/**
+ * The subrequests a pass of the walk may spend by default. A Worker invocation gets 1000 on the
+ * paid plan; the scheduled pass shares its invocation with every other phase, so it takes half.
+ * A staff request runs alone and may take {@link STORE_REBUILD_REQUEST_SUBREQUESTS}.
+ */
+export const STORE_REBUILD_PASS_SUBREQUESTS = 500;
+export const STORE_REBUILD_REQUEST_SUBREQUESTS = 900;
+/** The scripts a pass with `subrequests` to spend may walk: {@link STORE_REBUILD_SUBREQUESTS_PER_SCRIPT} each. */
+export const storeRebuildBudget = (subrequests) => Math.max(1, Math.floor(subrequests / STORE_REBUILD_SUBREQUESTS_PER_SCRIPT));
+/**
+ * How long a `done` rebuild stands before a scheduled pass walks it again. A walk that is done
+ * proves every script was current when it read it; a control plane still serving the older inputs
+ * (a deploy rolling out gradually) can upload a script after that. Walking it again reads each
+ * script's settings and uploads only a script that is not current.
+ */
+export const STORE_REBUILD_REVERIFY_MS = 24 * 60 * 60_000;
+/**
+ * How long after a fingerprint first appears a walk is not yet proof (#2309 review). While a
+ * control-plane deploy rolls out, the version still building from the older inputs can push a
+ * script after the walk read it. Only a walk that STARTED this long after its fingerprint was first
+ * walked towards proves every script current: a walk that spans passes reads its first scripts when
+ * it starts, so when it finishes says nothing about them. A done walk that started inside the window
+ * reads as settling, and the scheduled pass walks it again once the window has passed. A deploy that
+ * takes longer to roll out is followed by `store-driver rebuild`, or by a person's attestation that
+ * it has finished. The default; the control plane reads `STORE_REBUILD_SETTLE_MINUTES`.
+ */
+export const STORE_REBUILD_SETTLE_MS = 60 * 60_000;
+/**
+ * Whether a walk proves every script current once done: it started after the settle window, or a
+ * person attested, when it started, that no control-plane deploy was rolling out.
+ */
+const settled = (rebuild, settleMs) => rebuild.attestedBy !== null || Date.parse(rebuild.startedAt) - Date.parse(rebuild.fingerprintSince) >= settleMs;
+const finishedMs = (rebuild) => Date.parse(rebuild.finishedAt ?? rebuild.updatedAt);
+/** How long a pass's lease on a driver's rebuild lives between progress writes. */
+const STORE_REBUILD_LEASE_TTL_MS = 10 * 60_000;
+/**
+ * The binding an unreadable vertical's store is walked under. A script's name and build take the
+ * vertical and the driver only, so the binding names nothing the upload uses.
+ */
+const UNNAMED_BINDING = 'UNNAMED';
+/** The cursor once the canary is walked: before every slug, after "not started" (null). */
+const CANARY_WALKED = '';
+const message = (e) => (e instanceof Error ? e.message : String(e));
+/** One pass of the walk over every driver, within `budget` scripts. */
+export async function runStoreFleetRebuild(deps) {
+    // One loader for the pass: a release's bytes are read once, however many scripts run it.
+    const read = storedReleaseLoader({ admin: deps.admin, actor: deps.actor, bytes: deps.bytes });
+    const cache = new Map();
+    const loadRelease = (input) => {
+        const key = `${input.driver}/${input.sha256 ?? ''}`;
+        let hit = cache.get(key);
+        if (!hit)
+            cache.set(key, (hit = read(input)));
+        return hit;
+    };
+    let budget = deps.budget;
+    const passes = [];
+    for (const registration of deps.drivers) {
+        // One driver's failure (its directory read, its lease write) never stops the others.
+        const pass = await rebuildDriver(deps, registration, loadRelease, budget).catch((e) => ({ driver: registration.driver, outcome: 'error', walked: [], uploaded: [], error: message(e) }));
+        budget -= pass.walked.length;
+        passes.push(pass);
+    }
+    return passes;
+}
+async function rebuildDriver(deps, registration, loadRelease, budget) {
+    const { admin, actor, fingerprint } = deps;
+    const driver = registration.driver;
+    const walked = [];
+    const uploaded = [];
+    const result = (outcome, rebuild) => ({ driver, outcome, walked, uploaded, ...(rebuild ? { rebuild } : {}) });
+    const stood = await admin.storeDriverRebuild(actor, driver);
+    const doneHere = stood?.fingerprint === fingerprint && stood.state === 'done';
+    const now = (deps.now ?? Date.now)();
+    const settleMs = deps.settleMs ?? STORE_REBUILD_SETTLE_MS;
+    const restart = doneHere && (deps.reverify === 'always' || now >= nextAutomaticWalk(stood, settleMs));
+    if (doneHere && !restart && deps.attestedBy === undefined)
+        return result('current', stood);
+    if (budget <= 0)
+        return result('paused', stood);
+    const holder = ulid();
+    const claimed = await admin.claimStoreDriverRebuild(actor, {
+        driver,
+        fingerprint,
+        holder,
+        ttlMs: STORE_REBUILD_LEASE_TTL_MS,
+        restart,
+        ...(deps.attestedBy !== undefined ? { attestedBy: deps.attestedBy } : {}),
+    });
+    if (!claimed)
+        return result('busy', stood);
+    let rebuild = claimed;
+    let cursor = claimed.cursor;
+    const failures = new Map(claimed.failures.map((f) => [f.vertical, f.detail]));
+    const advance = async (done, release = false) => {
+        const list = [...failures].map(([vertical, detail]) => ({ vertical, detail }));
+        // A done walk that started after the settle window (or was attested) proves every script current.
+        const verified = done && settled(rebuild, settleMs);
+        rebuild = await admin.advanceStoreDriverRebuild(actor, { driver, fingerprint, holder, cursor, failures: list, done, verified, ttlMs: STORE_REBUILD_LEASE_TTL_MS, release });
+    };
+    // The lease is dropped once, however the walk ends; the end of a whole walk drops it itself.
+    let leaseDropped = false;
+    let held = false;
+    try {
+        held = await admin.holdStoreDriver(actor, { driver, holder, mode: 'shared', ttlMs: PUSH_HOLD_TTL_MS });
+        if (!held)
+            return result('held', rebuild);
+        const declared = await admin.storeDriverDeclarations(actor, driver);
+        const byVertical = new Map(declared.verticals.map((v) => [v.vertical, v.stores.map((s) => ({ binding: s.binding, driver }))]));
+        // A vertical whose stores cannot be named may still run a script of this driver. Its name is the
+        // vertical's and the driver's alone, so it is walked by name: rebuilt if it exists, and left
+        // alone if not (no script, nothing that accepts a key). Only a failure to do that is a failure.
+        for (const vertical of declared.unreadable) {
+            if (!byVertical.has(vertical))
+                byVertical.set(vertical, [{ binding: UNNAMED_BINDING, driver }]);
+        }
+        // A failure no longer to be had: the vertical declares the driver no more, and is readable.
+        for (const vertical of [...failures.keys()]) {
+            if (vertical !== SCOPE_STORE_CANARY && !byVertical.has(vertical))
+                failures.delete(vertical);
+        }
+        const ahead = aheadOf(cursor, [...byVertical.keys()].sort());
+        const walking = new Set(ahead);
+        const steps = [
+            ...ahead.map((key) => ({ key, walk: true })),
+            // The retries: a failure behind the cursor that can be rebuilt now.
+            ...[...failures.keys()]
+                .filter((key) => !walking.has(key) && (key === SCOPE_STORE_CANARY || byVertical.has(key)))
+                .map((key) => ({ key, walk: false })),
+        ];
+        for (const step of steps) {
+            if (walked.length >= budget)
+                break;
+            if (!(await admin.holdStoreDriver(actor, { driver, holder, mode: 'shared', ttlMs: PUSH_HOLD_TTL_MS, renew: true }))) {
+                // The hold lapsed, and a rollout may hold the driver now: walk nothing more. The lease may
+                // have lapsed with it, and then another pass owns the row; what was walked is recorded.
+                return result('held', rebuild);
+            }
+            walked.push(step.key);
+            try {
+                const up = step.key === SCOPE_STORE_CANARY
+                    ? (await deps.ensureCanary({ driver, existingOnly: true, loadRelease })).uploaded
+                    : await deps.ensure({ vertical: step.key, stores: byVertical.get(step.key), existingOnly: true, loadRelease }).then((r) => (r ? r.uploaded > 0 : true));
+                if (up)
+                    uploaded.push(step.key);
+                failures.delete(step.key);
+            }
+            catch (e) {
+                failures.set(step.key, message(e));
+                await admin
+                    .recordOpsFailure({
+                    actor,
+                    operation: 'store-rebuild',
+                    stage: 'store-upload',
+                    vertical: step.key === SCOPE_STORE_CANARY ? null : step.key,
+                    status: e instanceof ControlPlaneError ? e.status : 502,
+                    message: `store script rebuild of '${driver}' for ${step.key === SCOPE_STORE_CANARY ? 'its canary' : step.key}: ${message(e)}`,
+                })
+                    .catch(() => undefined);
+            }
+            if (step.walk)
+                cursor = step.key === SCOPE_STORE_CANARY ? CANARY_WALKED : step.key;
+            await advance(false);
+        }
+        const left = steps.length - walked.length;
+        const done = left === 0 && failures.size === 0;
+        leaseDropped = true;
+        await advance(done, true);
+        return result(left > 0 ? 'paused' : done ? 'done' : 'failing', rebuild);
+    }
+    finally {
+        if (held)
+            await admin.releaseStoreDriverHold(actor, { driver, holder }).catch(() => undefined);
+        if (!leaseDropped)
+            await advance(false, true).catch(() => undefined);
+    }
+}
+/** The scripts ahead of a walk's cursor: the canary before any vertical, then each slug after it. */
+function aheadOf(cursor, verticals) {
+    return [...(cursor === null ? [SCOPE_STORE_CANARY] : []), ...verticals.filter((v) => cursor === null || v > cursor)];
+}
+/**
+ * When the scheduled pass walks a done rebuild again: once its settle window has passed, for a walk
+ * that finished inside it; else a day after it finished.
+ */
+function nextAutomaticWalk(rebuild, settleMs) {
+    return settled(rebuild, settleMs) ? finishedMs(rebuild) + STORE_REBUILD_REVERIFY_MS : Date.parse(rebuild.fingerprintSince) + settleMs;
+}
+/**
+ * Whether every script of a driver is built from what the platform builds from now, and which may
+ * not be: while the rebuild is towards another fingerprint (or none ran yet), the canary and every
+ * vertical that declared the driver; while it runs, the ones ahead of its cursor and the ones that
+ * failed. A push or serve may have rebuilt some of them since, and the walk then finds them
+ * current, so `pending` may over-count, never under-count. A walk that is done but not settled (see
+ * {@link STORE_REBUILD_SETTLE_MS}) is not current: `settling` says when it is walked again. `nextWalk`
+ * is when the scheduled pass walks a done rebuild again. A walk under way at lists an earlier walk
+ * already proved is `current` and `reverifying` until a script it reaches fails.
+ */
+export function storeRebuildStatus(rebuild, fingerprint, declared, settleMs = STORE_REBUILD_SETTLE_MS) {
+    if (!rebuild || rebuild.fingerprint !== fingerprint)
+        return { current: false, pending: [SCOPE_STORE_CANARY, ...declared] };
+    if (rebuild.state === 'done') {
+        const nextWalk = new Date(nextAutomaticWalk(rebuild, settleMs)).toISOString();
+        return settled(rebuild, settleMs) ? { current: true, pending: [], nextWalk } : { current: false, pending: [], settling: nextWalk, nextWalk };
+    }
+    // A walk under way that re-checks a fleet a walk at these lists already proved current: still
+    // current, unless a script it reached is known to have failed.
+    if (rebuild.verifiedAt !== null && rebuild.failures.length === 0)
+        return { current: true, pending: [], reverifying: true };
+    const pending = new Set([...aheadOf(rebuild.cursor, declared), ...rebuild.failures.map((f) => f.vertical)]);
+    return { current: false, pending: [...pending] };
+}
+//# sourceMappingURL=store-rebuild.js.map

package/dist/store-rebuild.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store-rebuild.js","sourceRoot":"","sources":["../src/store-rebuild.ts"],"names":[],"mappings":"AAgCA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,EAAE,IAAI,EAAiF,MAAM,sBAAsB,CAAC;AAC3H,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAA0B,MAAM,qBAAqB,CAAC;AAKpG;;;;GAIG;AACH,MAAM,oCAAoC,GAAG,CAAC,CAAC;AAC/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,GAAG,CAAC;AAClD,MAAM,CAAC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AACrD,kHAAkH;AAClH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,WAAmB,EAAU,EAAE,CAChE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,oCAAoC,CAAC,CAAC,CAAC;AAC9E;;;;;GAKG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;AAC1D;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,MAAM,CAAC;AACnD;;;GAGG;AACH,MAAM,OAAO,GAAG,CAAC,OAAiC,EAAE,QAAgB,EAAE,EAAE,CACtE,OAAO,CAAC,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,QAAQ,CAAC;AAClH,MAAM,UAAU,GAAG,CAAC,OAAiC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;AAC9G,mFAAmF;AACnF,MAAM,0BAA0B,GAAG,EAAE,GAAG,MAAM,CAAC;AAE/C;;;GAGG;AACH,MAAM,eAAe,GAAG,SAAS,CAAC;AAElC,2FAA2F;AAC3F,MAAM,aAAa,GAAG,EAAE,CAAC;AAsDzB,MAAM,OAAO,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAE7E,uEAAuE;AACvE,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAA2B;IACpE,yFAAyF;IACzF,MAAM,IAAI,GAAG,mBAAmB,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9F,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmC,CAAC;IACzD,MAAM,WAAW,GAAgB,CAAC,KAAK,EAAE,EAAE;QACzC,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACpD,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,GAAG;YAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IACzB,MAAM,MAAM,GAA6B,EAAE,CAAC;IAC5C,KAAK,MAAM,YAAY,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACxC,qFAAqF;QACrF,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC,KAAK,CAC7E,CAAC,CAAU,EAA0B,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CACzI,CAAC;QACF,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,IAA2B,EAC3B,YAAqC,EACrC,WAAwB,EACxB,MAAc;IAEd,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IACnC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,CAAC,OAA0C,EAAE,OAAkC,EAA0B,EAAE,CACxH,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,KAAK,EAAE,WAAW,KAAK,WAAW,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,CAAC;IAC9E,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,uBAAuB,CAAC;IAC1D,MAAM,OAAO,GAAG,QAAQ,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,IAAI,iBAAiB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IACtG,IAAI,QAAQ,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC3F,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEhD,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC;IACtB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,uBAAuB,CAAC,KAAK,EAAE;QACzD,MAAM;QACN,WAAW;QACX,MAAM;QACN,KAAK,EAAE,0BAA0B;QACjC,OAAO;QACP,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1E,CAAC,CAAC;IACH,IAAI,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,IAAI,OAAO,GAAG,OAAO,CAAC;IACtB,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC5B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,KAAK,EAAE,IAAa,EAAE,OAAO,GAAG,KAAK,EAAE,EAAE;QACvD,MAAM,IAAI,GAAgC,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5G,kGAAkG;QAClG,MAAM,QAAQ,GAAG,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACpD,OAAO,GAAG,MAAM,KAAK,CAAC,yBAAyB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,0BAA0B,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9K,CAAC,CAAC;IACF,6FAA6F;IAC7F,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAC;QACvG,IAAI,CAAC,IAAI;YAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/H,iGAAiG;QACjG,6FAA6F;QAC7F,gGAAgG;QAChG,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC3C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAClG,CAAC;QACD,4FAA4F;QAC5F,KAAK,MAAM,QAAQ,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC5C,IAAI,QAAQ,KAAK,kBAAkB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9F,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAqC;YAC9C,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,oEAAoE;YACpE,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;iBACpB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,kBAAkB,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;iBACzF,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACxC,CAAC;QACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM;gBAAE,MAAM;YACnC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpH,2FAA2F;gBAC3F,wFAAwF;gBACxF,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACjC,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,KAAK,kBAAkB;oBACxC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,QAAQ;oBACjF,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAE,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC3J,IAAI,EAAE;oBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,KAAK;qBACR,gBAAgB,CAAC;oBAChB,KAAK;oBACL,SAAS,EAAE,eAAe;oBAC1B,KAAK,EAAE,cAAc;oBACrB,QAAQ,EAAE,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG;oBAC3D,MAAM,EAAE,CAAC,YAAY,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG;oBACvD,OAAO,EAAE,4BAA4B,MAAM,SAAS,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE;iBAC/H,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;YACD,IAAI,IAAI,CAAC,IAAI;gBAAE,MAAM,GAAG,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YACnF,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;QAC/C,YAAY,GAAG,IAAI,CAAC;QACpB,MAAM,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC1B,OAAO,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACT,IAAI,IAAI;YAAE,MAAM,KAAK,CAAC,sBAAsB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/F,IAAI,CAAC,YAAY;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,qGAAqG;AACrG,SAAS,OAAO,CAAC,MAAqB,EAAE,SAA4B;IAClE,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;AACvH,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,OAAiC,EAAE,QAAgB;IAC5E,OAAO,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,QAAQ,CAAC;AACxI,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAA6C,EAC7C,WAAmB,EACnB,QAA2B,EAC3B,QAAQ,GAAW,uBAAuB;IAE1C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,WAAW,KAAK,WAAW;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,kBAAkB,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;IAC3H,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAC9E,OAAO,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;IAC/I,CAAC;IACD,8FAA8F;IAC9F,+DAA+D;IAC/D,IAAI,OAAO,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC3H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC5G,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,CAAC;AACnD,CAAC"}

package/dist/store-releases.d.ts

@@ -0,0 +1,158 @@
+/**
+ * A store driver's releases: stored, canaried, rolled out and rolled back (D-63, #2300).
+ *
+ * The bytes live outside the directory, content-addressed under `<driver>/<sha256>`, where the
+ * sha256 is `storeDriverReleaseDigest` of the bundle, computed here when it arrives and again by
+ * the ensurer before every upload of it, restore and rollback included. The directory holds who
+ * uploaded which release from which commit, every rollout, and the current good release
+ * (`store-driver-releases.ts` in the kernel).
+ *
+ * A rollout is one request across a fleet. It holds the driver exclusively for its whole length,
+ * renewing the hold before every upload, so no push or serve rebuilds a script between the
+ * rollout's read and its upload, and two rollouts never interleave. It canaries the release on the
+ * platform's own object first, and no vertical's script moves unless every reserved operation
+ * answered there. Then it moves each vertical that ever declared the driver, at most
+ * {@link ROLLOUT_VERTICALS_PER_REQUEST} uploads and {@link ROLLOUT_WALKS_PER_REQUEST} verticals
+ * walked per request. A bigger fleet ends the request `paused`, and posting the same rollout (or
+ * rollback) again continues it from the attempt's walk ledger (#2312), so a fleet of any size
+ * converges, a rollback included. A request that stops on a failure ends `failed` or `diverged`.
+ */
+import { type PlatformActorId, type ReservedStoreOperation, type ScopeStoreNeed, type StoreDriverRegistration } from '@substrat-run/contracts';
+import { type HostAdmin, type StoreDriverRolloutAction, type StoreDriverRolloutRecord } from '@substrat-run/kernel';
+import type { ScopeStoreEnsurerOptions, StoreDriverReleaseArtifact } from './store-script.js';
+/** Where a release's bytes are kept: an object store keyed by driver and sha256. */
+export interface StoreReleaseBytes {
+    get(key: string): Promise<Uint8Array | undefined>;
+    put(key: string, bytes: Uint8Array): Promise<void>;
+}
+/** The key a release's bytes are stored under. */
+export declare const storeReleaseKey: (driver: string, sha256: string) => string;
+/** How long a rollout's exclusive hold lives between renewals. */
+export declare const ROLLOUT_HOLD_TTL_MS: number;
+/**
+ * How long a push's or a serve's shared hold lives. It is renewed before each driver's upload; one
+ * driver's upload (a settings read and at most one upload per script) must fit inside it, or a
+ * rollout could take the driver before that upload lands. The same lease as a rollout's.
+ */
+export declare const PUSH_HOLD_TTL_MS: number;
+/**
+ * The verticals one rollout request moves. Each costs a settings read and at most one upload,
+ * plus a hold renewal, so a request stays well inside a Worker invocation's subrequest budget.
+ * A fleet bigger than this is moved over several requests (the rollout answers `complete: false`).
+ */
+export declare const ROLLOUT_VERTICALS_PER_REQUEST = 100;
+/**
+ * The verticals one rollout request walks. Every one costs a settings read and a hold renewal even
+ * when it is already at the release, so a request that reaches this many ends `paused` and the
+ * next continues from the walk ledger (#2312). It also bounds what a failure returns: an attempt
+ * that reached more ends `diverged` without returning any, and a rollback, paged the same way,
+ * converges them.
+ */
+export declare const ROLLOUT_WALKS_PER_REQUEST = 300;
+type LoadRelease = ScopeStoreEnsurerOptions['loadRelease'];
+/**
+ * A release bundle as uploaded or as stored, decoded and judged: the upload body's shape, the
+ * decoded size cap, and the Python driver's module rule. Refusals are 422 (413 for the size).
+ */
+export declare function decodeStoreDriverRelease(raw: unknown): Omit<StoreDriverReleaseArtifact, 'sha256'>;
+/**
+ * Store a release's bytes and record them (D-63: bytes may be published before the pin lands).
+ * The bytes go first, so a recorded release always has bytes; a retry of the same bundle writes
+ * the same key and records nothing new.
+ */
+export declare function storeDriverRelease(input: {
+    admin: HostAdmin;
+    actor: PlatformActorId;
+    bytes: StoreReleaseBytes;
+    driver: string;
+    body: unknown;
+}): Promise<import("@substrat-run/kernel").StoreDriverReleaseRecord>;
+/**
+ * The ensurer's loader over the stored bytes: a named release, or the driver's current good one.
+ * Each answer carries the sha256 it was read as, which the ensurer holds the bytes to.
+ */
+export declare function storedReleaseLoader(input: {
+    admin: HostAdmin;
+    actor: PlatformActorId;
+    bytes: StoreReleaseBytes;
+}): LoadRelease;
+/** The reserved calls a canary makes, at the canary script. */
+export type CanaryCall = (target: {
+    scriptName: string;
+    objectName: string;
+}, operation: ReservedStoreOperation, input?: unknown) => Promise<unknown>;
+/**
+ * A canary object's name. Each id comes from a mint of its own, so its 80 random bits are fresh
+ * from `crypto.getRandomValues`: the shared mint would make the second id the first plus one
+ * within a millisecond, and either id would then give the other away.
+ */
+export declare const canaryObjectName: (binding: string) => string;
+/**
+ * Every reserved operation, on a fresh object of the platform's own (D-63): its tenant and scope
+ * ids are new ulids, from the runtime's CSPRNG, never in the directory, and its vertical segment
+ * is no vertical's. Each answer is parsed by the call and asserted here; the first that is not
+ * what an object holding no data answers fails the canary. The object is wiped at the end.
+ */
+export declare function canaryStoreRelease(input: {
+    call: CanaryCall;
+    scriptName: string;
+    binding?: string;
+}): Promise<void>;
+export interface StoreDriverRolloutDeps {
+    admin: HostAdmin;
+    actor: PlatformActorId;
+    registration: StoreDriverRegistration;
+    sha256: string;
+    action: StoreDriverRolloutAction;
+    /**
+     * Move one vertical's scripts of the driver to `to` (the push's ensurer, with `to`). Answers
+     * whether anything was uploaded: false for a script already there, built the same way.
+     */
+    ensure: (input: {
+        vertical: string;
+        stores: readonly ScopeStoreNeed[];
+        to: string;
+    }) => Promise<boolean>;
+    /** Put `to` in the driver's canary script; answers its name. */
+    ensureCanary: (to: string) => Promise<{
+        scriptName: string;
+    }>;
+    callCanary: CanaryCall;
+    /** For a test: a smaller batch than {@link ROLLOUT_VERTICALS_PER_REQUEST}. */
+    perRequest?: number;
+    /** For a test: fewer verticals walked per request than {@link ROLLOUT_WALKS_PER_REQUEST}. */
+    perRequestWalks?: number;
+}
+export interface StoreDriverRolloutResult {
+    rollout: StoreDriverRolloutRecord;
+    /** The verticals whose scripts this request uploaded. */
+    moved: string[];
+    /** False when verticals remain: post the same rollout again. */
+    complete: boolean;
+    /** The verticals not walked yet; 0 once complete. */
+    remaining: number;
+}
+/**
+ * Move every script of a driver to a stored, pinned release (D-63): hold, canary, then each
+ * vertical. A failure after a vertical moved returns every vertical the attempt reached to the
+ * release the rollout started from, rehashing those bytes as it uploads them, and ends `failed`.
+ * It ends `diverged` instead, naming what is where, when there is nothing to return to (a driver's
+ * first rollout), when returning would lower the stored schema, when the attempt reached more
+ * verticals than one request returns, or when a return fails: the next rollout or rollback
+ * converges every script. The hold is renewed before every upload, the returns included, and a
+ * hold that cannot be renewed stops everything at once: another holder may be moving the same
+ * scripts, so nothing is returned either. The hold is released however it ends.
+ *
+ * A fleet bigger than one request is paged (#2312). Each request walks at most
+ * {@link ROLLOUT_WALKS_PER_REQUEST} verticals and uploads at most `perRequest` scripts; one that runs out
+ * ends `paused`, and its end records what it walked in the attempt's ledger. Posting the same
+ * rollout (or rollback) again continues the attempt and walks only declarations the ledger does
+ * not hold, in (vertical, binding) order: a vertical declared between two requests, wherever it
+ * sorts, is walked by the next one, and one walked is never walked again. A request ends `good`
+ * only when the pending list it read under its exclusive hold was the whole of it and it walked
+ * every vertical on it: no push can create a script while the hold is held, so nothing declared
+ * can be left on another release.
+ */
+export declare function runStoreDriverRollout(deps: StoreDriverRolloutDeps): Promise<StoreDriverRolloutResult>;
+export {};
+//# sourceMappingURL=store-releases.d.ts.map

package/dist/store-releases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store-releases.d.ts","sourceRoot":"","sources":["../src/store-releases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,sBAAsB,EAC3B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAC7B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAGL,KAAK,SAAS,EACd,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAE9B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE9F,oFAAoF;AACpF,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAClD,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpD;AAED,kDAAkD;AAClD,eAAO,MAAM,eAAe,WAAY,MAAM,UAAU,MAAM,KAAG,MAA+B,CAAC;AAEjG,kEAAkE;AAClE,eAAO,MAAM,mBAAmB,QAAc,CAAC;AAC/C;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,QAAc,CAAC;AAC5C;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,MAAM,CAAC;AACjD;;;;;;GAMG;AACH,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C,KAAK,WAAW,GAAG,wBAAwB,CAAC,aAAa,CAAC,CAAC;AAY3D;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC,0BAA0B,EAAE,QAAQ,CAAC,CAqBjG;AAkBD;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAC9C,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,eAAe,CAAC;IACvB,KAAK,EAAE,iBAAiB,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;CACf,oEAWA;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,KAAK,EAAE,iBAAiB,CAAA;CAAE,GAAG,WAAW,CAc9H;AAED,+DAA+D;AAC/D,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,EAAE,sBAAsB,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEtJ;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,YAAa,MAAM,KAAG,MAA6E,CAAC;AAIjI;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBzH;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,eAAe,CAAC;IACvB,YAAY,EAAE,uBAAuB,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,wBAAwB,CAAC;IACjC;;;OAGG;IACH,MAAM,EAAE,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,SAAS,cAAc,EAAE,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACzG,gEAAgE;IAChE,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9D,UAAU,EAAE,UAAU,CAAC;IACvB,8EAA8E;IAC9E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6FAA6F;IAC7F,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,OAAO,EAAE,wBAAwB,CAAC;IAClC,yDAAyD;IACzD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,QAAQ,EAAE,OAAO,CAAC;IAClB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;CACnB;AAaD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqE3G"}