@substrat-run/control-plane-api 0.147.0 → 0.149.0

package/dist/store-releases.js

@@ -0,0 +1,332 @@
+/**
+ * A store driver's releases: stored, canaried, rolled out and rolled back (D-63, #2300).
+ *
+ * The bytes live outside the directory, content-addressed under `<driver>/<sha256>`, where the
+ * sha256 is `storeDriverReleaseDigest` of the bundle, computed here when it arrives and again by
+ * the ensurer before every upload of it, restore and rollback included. The directory holds who
+ * uploaded which release from which commit, every rollout, and the current good release
+ * (`store-driver-releases.ts` in the kernel).
+ *
+ * A rollout is one request across a fleet. It holds the driver exclusively for its whole length,
+ * renewing the hold before every upload, so no push or serve rebuilds a script between the
+ * rollout's read and its upload, and two rollouts never interleave. It canaries the release on the
+ * platform's own object first, and no vertical's script moves unless every reserved operation
+ * answered there. Then it moves each vertical that ever declared the driver, at most
+ * {@link ROLLOUT_VERTICALS_PER_REQUEST} uploads and {@link ROLLOUT_WALKS_PER_REQUEST} verticals
+ * walked per request. A bigger fleet ends the request `paused`, and posting the same rollout (or
+ * rollback) again continues it from the attempt's walk ledger (#2312), so a fleet of any size
+ * converges, a rollback included. A request that stops on a failure ends `failed` or `diverged`.
+ */
+import { STORE_RELEASE_MAX_BYTES, scopeStoreCanaryObjectName, storeDriverReleaseDigest, storeDriverReleaseUpload, } from '@substrat-run/contracts';
+import { ControlPlaneError } from '@substrat-run/control-plane-client';
+import { createUlid, ulid, } from '@substrat-run/kernel';
+import { assertPythonDriverModules } from './store-entry.js';
+/** The key a release's bytes are stored under. */
+export const storeReleaseKey = (driver, sha256) => `${driver}/${sha256}`;
+/** How long a rollout's exclusive hold lives between renewals. */
+export const ROLLOUT_HOLD_TTL_MS = 10 * 60_000;
+/**
+ * How long a push's or a serve's shared hold lives. It is renewed before each driver's upload; one
+ * driver's upload (a settings read and at most one upload per script) must fit inside it, or a
+ * rollout could take the driver before that upload lands. The same lease as a rollout's.
+ */
+export const PUSH_HOLD_TTL_MS = 10 * 60_000;
+/**
+ * The verticals one rollout request moves. Each costs a settings read and at most one upload,
+ * plus a hold renewal, so a request stays well inside a Worker invocation's subrequest budget.
+ * A fleet bigger than this is moved over several requests (the rollout answers `complete: false`).
+ */
+export const ROLLOUT_VERTICALS_PER_REQUEST = 100;
+/**
+ * The verticals one rollout request walks. Every one costs a settings read and a hold renewal even
+ * when it is already at the release, so a request that reaches this many ends `paused` and the
+ * next continues from the walk ledger (#2312). It also bounds what a failure returns: an attempt
+ * that reached more ends `diverged` without returning any, and a rollback, paged the same way,
+ * converges them.
+ */
+export const ROLLOUT_WALKS_PER_REQUEST = 300;
+const decodeBase64 = (text) => {
+    let binary;
+    try {
+        binary = atob(text);
+    }
+    catch {
+        throw new ControlPlaneError(422, 'a release module is not base64');
+    }
+    return Uint8Array.from(binary, (ch) => ch.charCodeAt(0));
+};
+/**
+ * A release bundle as uploaded or as stored, decoded and judged: the upload body's shape, the
+ * decoded size cap, and the Python driver's module rule. Refusals are 422 (413 for the size).
+ */
+export function decodeStoreDriverRelease(raw) {
+    const parsed = storeDriverReleaseUpload.safeParse(raw);
+    if (!parsed.success) {
+        throw new ControlPlaneError(422, `not a store driver release: ${parsed.error.issues.map((i) => i.message).join('; ')}`);
+    }
+    let total = 0;
+    const modules = parsed.data.modules.map((module) => {
+        const content = decodeBase64(module.contentBase64);
+        total += content.byteLength;
+        if (total > STORE_RELEASE_MAX_BYTES) {
+            throw new ControlPlaneError(413, `a store driver release carries at most ${STORE_RELEASE_MAX_BYTES} module bytes`);
+        }
+        return { name: module.name, content };
+    });
+    const release = { sourceCommit: parsed.data.sourceCommit, entry: parsed.data.entry, modules };
+    try {
+        assertPythonDriverModules(release);
+    }
+    catch (e) {
+        throw new ControlPlaneError(422, e instanceof Error ? e.message : String(e));
+    }
+    return release;
+}
+/** The stored form: the upload body again, so one decoder reads both. */
+function encodeStoreDriverRelease(release) {
+    const base64 = (bytes) => {
+        let text = '';
+        for (let i = 0; i < bytes.length; i += 0x8000)
+            text += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+        return btoa(text);
+    };
+    return new TextEncoder().encode(JSON.stringify({
+        sourceCommit: release.sourceCommit,
+        entry: release.entry,
+        modules: release.modules.map((m) => ({ name: m.name, contentBase64: base64(m.content) })),
+    }));
+}
+/**
+ * Store a release's bytes and record them (D-63: bytes may be published before the pin lands).
+ * The bytes go first, so a recorded release always has bytes; a retry of the same bundle writes
+ * the same key and records nothing new.
+ */
+export async function storeDriverRelease(input) {
+    const release = decodeStoreDriverRelease(input.body);
+    const sha256 = await storeDriverReleaseDigest(release).catch((e) => {
+        throw new ControlPlaneError(422, e instanceof Error ? e.message : String(e));
+    });
+    const held = await input.admin.getStoreDriverRelease(input.actor, input.driver, sha256);
+    if (held && held.sourceCommit !== release.sourceCommit) {
+        throw new ControlPlaneError(409, `release ${sha256} of '${input.driver}' is stored as built from ${held.sourceCommit}`);
+    }
+    if (!held)
+        await input.bytes.put(storeReleaseKey(input.driver, sha256), encodeStoreDriverRelease(release));
+    return input.admin.recordStoreDriverRelease(input.actor, { driver: input.driver, sha256, sourceCommit: release.sourceCommit });
+}
+/**
+ * The ensurer's loader over the stored bytes: a named release, or the driver's current good one.
+ * Each answer carries the sha256 it was read as, which the ensurer holds the bytes to.
+ */
+export function storedReleaseLoader(input) {
+    return async ({ driver, sha256 }) => {
+        const wanted = sha256 ?? (await input.admin.storeDriverState(input.actor, driver)).current?.sha256;
+        if (!wanted)
+            return undefined;
+        const raw = await input.bytes.get(storeReleaseKey(driver, wanted));
+        if (!raw)
+            return undefined;
+        let json;
+        try {
+            json = JSON.parse(new TextDecoder().decode(raw));
+        }
+        catch {
+            throw new ControlPlaneError(422, `driver '${driver}' release ${wanted} is stored as something other than a release`);
+        }
+        return { ...decodeStoreDriverRelease(json), sha256: wanted };
+    };
+}
+/**
+ * A canary object's name. Each id comes from a mint of its own, so its 80 random bits are fresh
+ * from `crypto.getRandomValues`: the shared mint would make the second id the first plus one
+ * within a millisecond, and either id would then give the other away.
+ */
+export const canaryObjectName = (binding) => scopeStoreCanaryObjectName(createUlid()(), createUlid()(), binding);
+const tablesOf = (answer) => answer.tables;
+/**
+ * Every reserved operation, on a fresh object of the platform's own (D-63): its tenant and scope
+ * ids are new ulids, from the runtime's CSPRNG, never in the directory, and its vertical segment
+ * is no vertical's. Each answer is parsed by the call and asserted here; the first that is not
+ * what an object holding no data answers fails the canary. The object is wiped at the end.
+ */
+export async function canaryStoreRelease(input) {
+    const target = { scriptName: input.scriptName, objectName: canaryObjectName(input.binding ?? 'CANARY') };
+    const step = async (operation, check, body) => {
+        let answer;
+        try {
+            answer = await input.call(target, operation, body);
+        }
+        catch (e) {
+            throw new ControlPlaneError(502, `canary refused: ${operation} failed (${e instanceof Error ? e.message : String(e)})`);
+        }
+        const wrong = check(answer);
+        if (wrong)
+            throw new ControlPlaneError(502, `canary refused: ${operation} ${wrong}`);
+        return answer;
+    };
+    const ok = () => null;
+    await step('substrat.size', ok);
+    const exported = tablesOf(await step('substrat.export', (a) => (tablesOf(a).some((t) => t.rows.length > 0) ? 'exported rows from an object holding none' : null)));
+    await step('substrat.import', (a) => (a.tables === exported.length ? null : `imported ${a.tables} of ${exported.length} tables`), { tables: exported });
+    await step('substrat.export', (a) => (JSON.stringify(tablesOf(a)) === JSON.stringify(exported) ? null : 'did not export what it imported'));
+    await step('substrat.erase', (a) => (a.erased === 0 ? null : 'erased rows of a subject the object never held'), { subjectId: ulid() });
+    await step('substrat.wipe', ok);
+    await step('substrat.export', (a) => (tablesOf(a).some((t) => t.rows.length > 0) ? 'still holds rows after its wipe' : null));
+}
+const message = (e) => (e instanceof Error ? e.message : String(e));
+const schemaOf = (registration, sha256) => sha256 === null ? undefined : registration.releases.find((pin) => pin.sha256 === sha256)?.schema;
+/** The hold lapsed, or another holder ended the rollout: nothing more may be moved under it. */
+class HoldLost extends ControlPlaneError {
+    constructor(driver) {
+        super(409, `the hold on driver '${driver}' lapsed or its rollout was ended by another; post the rollout again to converge it`);
+    }
+}
+/**
+ * Move every script of a driver to a stored, pinned release (D-63): hold, canary, then each
+ * vertical. A failure after a vertical moved returns every vertical the attempt reached to the
+ * release the rollout started from, rehashing those bytes as it uploads them, and ends `failed`.
+ * It ends `diverged` instead, naming what is where, when there is nothing to return to (a driver's
+ * first rollout), when returning would lower the stored schema, when the attempt reached more
+ * verticals than one request returns, or when a return fails: the next rollout or rollback
+ * converges every script. The hold is renewed before every upload, the returns included, and a
+ * hold that cannot be renewed stops everything at once: another holder may be moving the same
+ * scripts, so nothing is returned either. The hold is released however it ends.
+ *
+ * A fleet bigger than one request is paged (#2312). Each request walks at most
+ * {@link ROLLOUT_WALKS_PER_REQUEST} verticals and uploads at most `perRequest` scripts; one that runs out
+ * ends `paused`, and its end records what it walked in the attempt's ledger. Posting the same
+ * rollout (or rollback) again continues the attempt and walks only declarations the ledger does
+ * not hold, in (vertical, binding) order: a vertical declared between two requests, wherever it
+ * sorts, is walked by the next one, and one walked is never walked again. A request ends `good`
+ * only when the pending list it read under its exclusive hold was the whole of it and it walked
+ * every vertical on it: no push can create a script while the hold is held, so nothing declared
+ * can be left on another release.
+ */
+export async function runStoreDriverRollout(deps) {
+    const { admin, actor, registration, sha256 } = deps;
+    const driver = registration.driver;
+    const holder = ulid();
+    if (!(await admin.holdStoreDriver(actor, { driver, holder, mode: 'exclusive', ttlMs: ROLLOUT_HOLD_TTL_MS }))) {
+        throw new ControlPlaneError(409, `driver '${driver}' is being rolled out or a vertical declaring it is being pushed; try again shortly`);
+    }
+    try {
+        const walkMax = deps.perRequestWalks ?? ROLLOUT_WALKS_PER_REQUEST;
+        const page = await admin.storeDriverRolloutPending(actor, { driver, sha256, action: deps.action, limit: walkMax });
+        if (page.unreadable.length > 0) {
+            throw new ControlPlaneError(409, `driver '${driver}' cannot be rolled out: ${page.unreadable.join(', ')} ha${page.unreadable.length === 1 ? 's a version' : 've versions'} whose stores cannot be named`);
+        }
+        const row = await admin.beginStoreDriverRollout(actor, { driver, sha256, action: deps.action, holder });
+        const end = (state, detail) => admin.finishStoreDriverRollout(actor, { driver, seq: row.seq, holder, state, detail });
+        const renew = async () => {
+            const held = await admin.holdStoreDriver(actor, { driver, holder, mode: 'exclusive', ttlMs: ROLLOUT_HOLD_TTL_MS, renew: true, rolloutSeq: row.seq });
+            if (!held)
+                throw new HoldLost(driver);
+        };
+        try {
+            await deps.ensureCanary(sha256).then(({ scriptName }) => canaryStoreRelease({ call: deps.callCanary, scriptName }));
+        }
+        catch (e) {
+            // No vertical's script moved in this request; the current release stays. A row the end
+            // cannot write (the hold lapsed) reads as diverged, which is what it then is.
+            await end('failed', message(e)).catch(() => undefined);
+            throw e;
+        }
+        const budget = deps.perRequest ?? ROLLOUT_VERTICALS_PER_REQUEST;
+        const moved = [];
+        // Every vertical this request reached. A request that pauses failed at none of them, so its
+        // paused end records exactly the verticals whose ensure answered.
+        const reached = [];
+        for (const target of page.verticals) {
+            if (moved.length >= budget)
+                break;
+            try {
+                await renew();
+                reached.push(target);
+                const stores = target.stores.map((s) => ({ binding: s.binding, driver }));
+                if (await deps.ensure({ vertical: target.vertical, stores, to: sha256 }))
+                    moved.push(target.vertical);
+            }
+            catch (e) {
+                const detail = e instanceof HoldLost
+                    ? { state: 'diverged', text: `Nothing was returned: ${reachedText(page.walked, reached)} may run ${sha256}.` }
+                    : await restore(deps, row, page.walked, reached, renew);
+                await end(detail.state, `${target.vertical}: ${message(e)}. ${detail.text}`).catch(() => undefined);
+                throw new ControlPlaneError(e instanceof ControlPlaneError ? e.status : 502, `${deps.action} of '${driver}' to ${sha256} stopped at ${target.vertical} (${message(e)}). ${detail.text}`);
+            }
+        }
+        const remaining = page.pending - reached.length;
+        if (remaining > 0) {
+            const rollout = await admin.finishStoreDriverRollout(actor, {
+                driver,
+                seq: row.seq,
+                holder,
+                state: 'paused',
+                detail: `this request walked ${reached.length} and moved ${moved.length}; ${remaining} vertical(s) not walked yet: post the same ${deps.action} again`,
+                walked: reached,
+            });
+            return { rollout, moved, complete: false, remaining };
+        }
+        return { rollout: await end('good', null), moved, complete: true, remaining: 0 };
+    }
+    finally {
+        await admin.releaseStoreDriverHold(actor, { driver, holder }).catch(() => undefined);
+    }
+}
+/** The verticals an attempt reached, for a refusal: earlier requests' count, then this one's names. */
+const reachedText = (before, reached) => {
+    const names = reached.map((t) => t.vertical).join(', ');
+    if (before === 0)
+        return names || 'no vertical';
+    return `${before} vertical(s) earlier requests of this attempt walked${names ? ` and ${names}` : ''}`;
+};
+/**
+ * Return every vertical a failed attempt reached to the release it started from, where that is
+ * allowed, renewing the hold before each upload: the ones its earlier requests walked (the
+ * ledger), then the ones this request reached. A hold that cannot be renewed stops the return.
+ * More than one request returns ({@link ROLLOUT_WALKS_PER_REQUEST}) returns nothing and ends diverged.
+ */
+async function restore(deps, row, before, reached, renew) {
+    const from = row.fromSha256;
+    const all = reachedText(before, reached);
+    if (before + reached.length === 0)
+        return { state: 'failed', text: 'No vertical moved.' };
+    if (from === null) {
+        return { state: 'diverged', text: `There is no earlier release to return to; ${all} may run ${deps.sha256}.` };
+    }
+    const leaving = schemaOf(deps.registration, deps.sha256);
+    const returning = schemaOf(deps.registration, from);
+    if (leaving === undefined || returning === undefined || returning < leaving) {
+        return {
+            state: 'diverged',
+            text: `Returning to ${from} would lower the stored schema (${leaving} to ${returning}), so ${all} may run ${deps.sha256}.`,
+        };
+    }
+    const max = deps.perRequestWalks ?? ROLLOUT_WALKS_PER_REQUEST;
+    if (before + reached.length > max) {
+        return {
+            state: 'diverged',
+            text: `The attempt reached ${before + reached.length} verticals, more than one request returns (${max}), so none was returned and they may run ${deps.sha256}: post a rollback, which converges every script on ${from} over as many requests as it takes.`,
+        };
+    }
+    const earlier = before === 0 ? [] : (await deps.admin.storeDriverRolloutWalked(deps.actor, { driver: row.driver, seq: row.seq, limit: max })).verticals;
+    const back = [...earlier, ...reached];
+    const stuck = [];
+    for (const [i, target] of back.entries()) {
+        try {
+            await renew();
+        }
+        catch {
+            return { state: 'diverged', text: `The hold was lost while returning; ${back.slice(i).map((t) => t.vertical).join(', ')} may run ${deps.sha256}.` };
+        }
+        try {
+            await deps.ensure({ vertical: target.vertical, stores: target.stores.map((s) => ({ binding: s.binding, driver: deps.registration.driver })), to: from });
+        }
+        catch (e) {
+            stuck.push(`${target.vertical} (${message(e)})`);
+        }
+    }
+    return stuck.length === 0
+        ? { state: 'failed', text: `${back.map((t) => t.vertical).join(', ')} returned to ${from}.` }
+        : { state: 'diverged', text: `Could not return ${stuck.join(', ')} to ${from}.` };
+}
+//# sourceMappingURL=store-releases.js.map

package/dist/store-releases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store-releases.js","sourceRoot":"","sources":["../src/store-releases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EACL,uBAAuB,EACvB,0BAA0B,EAC1B,wBAAwB,EACxB,wBAAwB,GAKzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,EACL,UAAU,EACV,IAAI,GAKL,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,yBAAyB,EAAE,MAAM,kBAAkB,CAAC;AAS7D,kDAAkD;AAClD,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAc,EAAE,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;AAEjG,kEAAkE;AAClE,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,GAAG,MAAM,CAAC;AAC/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,GAAG,MAAM,CAAC;AAC5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,GAAG,CAAC;AACjD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAI7C,MAAM,YAAY,GAAG,CAAC,IAAY,EAAc,EAAE;IAChD,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,gCAAgC,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,GAAY;IACnD,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACvD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,+BAA+B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1H,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACjD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACnD,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;QAC5B,IAAI,KAAK,GAAG,uBAAuB,EAAE,CAAC;YACpC,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,0CAA0C,uBAAuB,eAAe,CAAC,CAAC;QACrH,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;IACxC,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;IAC9F,IAAI,CAAC;QACH,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,yEAAyE;AACzE,SAAS,wBAAwB,CAAC,OAAmD;IACnF,MAAM,MAAM,GAAG,CAAC,KAAiB,EAAE,EAAE;QACnC,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,MAAM;YAAE,IAAI,IAAI,MAAM,CAAC,YAAY,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;QAC7G,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IACF,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAC7B,IAAI,CAAC,SAAS,CAAC;QACb,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;KAC1F,CAAC,CACH,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAMxC;IACC,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;QAC1E,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,qBAAqB,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxF,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC;QACvD,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,WAAW,MAAM,QAAQ,KAAK,CAAC,MAAM,6BAA6B,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;IAC1H,CAAC;IACD,IAAI,CAAC,IAAI;QAAE,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3G,OAAO,KAAK,CAAC,KAAK,CAAC,wBAAwB,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;AACjI,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA6E;IAC/G,OAAO,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE;QAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC;QACnG,IAAI,CAAC,MAAM;YAAE,OAAO,SAAS,CAAC;QAC9B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,WAAW,MAAM,aAAa,MAAM,8CAA8C,CAAC,CAAC;QACvH,CAAC;QACD,OAAO,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC/D,CAAC,CAAC;AACJ,CAAC;AAKD;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,OAAe,EAAU,EAAE,CAAC,0BAA0B,CAAC,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;AAEjI,MAAM,QAAQ,GAAG,CAAC,MAAe,EAAE,EAAE,CAAE,MAA0D,CAAC,MAAM,CAAC;AAEzG;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAiE;IACxG,MAAM,MAAM,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,gBAAgB,CAAC,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,EAAE,CAAC;IACzG,MAAM,IAAI,GAAG,KAAK,EAAE,SAAiC,EAAE,KAAyC,EAAE,IAAc,EAAE,EAAE;QAClH,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,mBAAmB,SAAS,YAAY,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1H,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,KAAK;YAAE,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,mBAAmB,SAAS,IAAI,KAAK,EAAE,CAAC,CAAC;QACrF,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IACF,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC;IACtB,MAAM,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnK,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAE,CAAwB,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAa,CAAwB,CAAC,MAAM,OAAO,QAAQ,CAAC,MAAM,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IACxM,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC;IAC5I,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAE,CAAwB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gDAAgD,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IAC/J,MAAM,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAChC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAChI,CAAC;AAgCD,MAAM,OAAO,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7E,MAAM,QAAQ,GAAG,CAAC,YAAqC,EAAE,MAAqB,EAAE,EAAE,CAChF,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE,MAAM,CAAC;AAEnG,gGAAgG;AAChG,MAAM,QAAS,SAAQ,iBAAiB;IACtC,YAAY,MAAc;QACxB,KAAK,CAAC,GAAG,EAAE,uBAAuB,MAAM,qFAAqF,CAAC,CAAC;IACjI,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAA4B;IACtE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACpD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC;IACtB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,mBAAmB,EAAE,CAAC,CAAC,EAAE,CAAC;QAC7G,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,WAAW,MAAM,qFAAqF,CAAC,CAAC;IAC3I,CAAC;IACD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,IAAI,yBAAyB,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,yBAAyB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACnH,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,iBAAiB,CACzB,GAAG,EACH,WAAW,MAAM,2BAA2B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,+BAA+B,CACxK,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,uBAAuB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACxG,MAAM,GAAG,GAAG,CAAC,KAAqC,EAAE,MAAqB,EAAE,EAAE,CAC3E,KAAK,CAAC,wBAAwB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACzF,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YACrJ,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QACtH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,uFAAuF;YACvF,8EAA8E;YAC9E,MAAM,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,CAAC,CAAC;QACV,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,6BAA6B,CAAC;QAChE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,4FAA4F;QAC5F,kEAAkE;QAClE,MAAM,OAAO,GAA+B,EAAE,CAAC;QAC/C,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,MAAM,IAAI,MAAM;gBAAE,MAAM;YAClC,IAAI,CAAC;gBACH,MAAM,KAAK,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;gBAC1E,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACxG,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,MAAM,GAAG,CAAC,YAAY,QAAQ;oBAClC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAmB,EAAE,IAAI,EAAE,yBAAyB,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,MAAM,GAAG,EAAE;oBACvH,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC1D,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBACpG,MAAM,IAAI,iBAAiB,CAAC,CAAC,YAAY,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,QAAQ,MAAM,QAAQ,MAAM,eAAe,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3L,CAAC;QACH,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAChD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,wBAAwB,CAAC,KAAK,EAAE;gBAC1D,MAAM;gBACN,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,MAAM;gBACN,KAAK,EAAE,QAAQ;gBACf,MAAM,EAAE,uBAAuB,OAAO,CAAC,MAAM,cAAc,KAAK,CAAC,MAAM,KAAK,SAAS,8CAA8C,IAAI,CAAC,MAAM,QAAQ;gBACtJ,MAAM,EAAE,OAAO;aAChB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QACxD,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACnF,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,CAAC,sBAAsB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED,uGAAuG;AACvG,MAAM,WAAW,GAAG,CAAC,MAAc,EAAE,OAA4C,EAAE,EAAE;IACnF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,IAAI,aAAa,CAAC;IAChD,OAAO,GAAG,MAAM,uDAAuD,KAAK,CAAC,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACxG,CAAC,CAAC;AAEF;;;;;GAKG;AACH,KAAK,UAAU,OAAO,CACpB,IAA4B,EAC5B,GAA6B,EAC7B,MAAc,EACd,OAA4C,EAC5C,KAA0B;IAE1B,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC;IAC5B,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;IAC1F,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,6CAA6C,GAAG,YAAY,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IACjH,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IACpD,IAAI,OAAO,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;QAC5E,OAAO;YACL,KAAK,EAAE,UAAU;YACjB,IAAI,EAAE,gBAAgB,IAAI,mCAAmC,OAAO,OAAO,SAAS,SAAS,GAAG,YAAY,IAAI,CAAC,MAAM,GAAG;SAC3H,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,IAAI,yBAAyB,CAAC;IAC9D,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAClC,OAAO;YACL,KAAK,EAAE,UAAU;YACjB,IAAI,EAAE,uBAAuB,MAAM,GAAG,OAAO,CAAC,MAAM,8CAA8C,GAAG,4CAA4C,IAAI,CAAC,MAAM,sDAAsD,IAAI,qCAAqC;SAC5P,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACxJ,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC;IACtC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,KAAK,EAAE,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,sCAAsC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACtJ,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3J,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,GAAG,EAAE;QAC7F,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,oBAAoB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,CAAC;AACtF,CAAC"}

package/dist/store-script.d.ts

@@ -0,0 +1,98 @@
+import { type ScopeStoreNeed, type StoreDriverRegistration } from '@substrat-run/contracts';
+import { type PythonStoreDriverRelease } from './store-entry.js';
+/**
+ * The binding that records which pinned release a store script runs: the release bundle's
+ * sha256 (D-63). In the entry's reserved prefix, so the driver's env never carries it.
+ */
+export declare const STORE_RELEASE_VAR = "SUBSTRAT_STORE_RELEASE";
+/**
+ * The binding that records what else the script was built from: a digest over every module the
+ * upload sent and its configuration. A changed entry, verifier, key list or flag changes it.
+ */
+export declare const STORE_BUILD_VAR = "SUBSTRAT_STORE_BUILD";
+/** One release of a driver's code, as the publish flow stores it (#2297). */
+export interface StoreDriverReleaseArtifact extends PythonStoreDriverRelease {
+    /** The commit the bundle was built from. A reviewer reads it; the upload checks the sha256. */
+    readonly sourceCommit: string;
+    /**
+     * The sha256 the loader read these bytes as (its storage key). When present, the bytes must
+     * hash to it, so a store that answers other bytes under a key is refused, not uploaded.
+     */
+    readonly sha256?: string;
+}
+export interface ScopeStoreEnsurerOptions {
+    accountId: string;
+    namespace: string;
+    apiToken: string;
+    /** The `caller` audience's public keys, raw Ed25519, base64url. A list, so a key can rotate. */
+    callerPublicKeys: readonly string[];
+    /** The `platform` audience's public keys, in the same form. Never one of the caller's. */
+    platformPublicKeys: readonly string[];
+    drivers?: readonly StoreDriverRegistration[];
+    /**
+     * A release's bytes. With `sha256`, that release: the one the script runs, or the one a
+     * rollout moves it to. Without it, the driver's current good release (#2300). Undefined when
+     * there is none.
+     */
+    loadRelease: (input: {
+        readonly driver: string;
+        readonly vertical: string;
+        readonly sha256?: string;
+    }) => Promise<StoreDriverReleaseArtifact | undefined>;
+}
+/**
+ * What the platform builds every store script from, as one digest (#2309): the entry, its
+ * verifier, the compatibility date and flags, and both key lists. It is the build digest of one
+ * fixed probe release, so it changes exactly when the platform's half of every script's build
+ * changes (a key added, dropped or revoked, an entry or verifier fix) and never when a driver's
+ * release does. The fleet rebuild converges every script on it.
+ */
+export declare function storePlatformFingerprint(keys: Pick<ScopeStoreEnsurerOptions, 'callerPublicKeys' | 'platformPublicKeys'>): Promise<string>;
+/**
+ * Put every (driver, vertical) store script a vertical's stores need in place, before the
+ * vertical's own upload (#2210, D-63). A push or a serve never moves a script to another
+ * release: only a rollout or a rollback does (#2300), and it says so with `to`.
+ *
+ * - No `to`, no script yet: it is uploaded at the driver's current good release.
+ * - No `to`, a script: it is rebuilt at the release it already runs, and uploaded again only when
+ *   what it was built from changed (the platform's entry, the verifier, a key list, a flag), so a
+ *   fix or a key rotation reaches it without a release moving.
+ * - `to`: the script is moved to that release, or left alone when it already runs it, built the
+ *   same way. Re-running a rollout therefore converges and uploads nothing twice.
+ *
+ * Every upload hashes the module bytes it is about to send and refuses (422) bytes whose
+ * sha256 the driver's registration does not pin, or that are not the release asked for. A driver
+ * with no release to run is refused (409). A script is never moved to a release whose pinned
+ * `schema` is lower than that of the release it runs (409): its objects may have migrated. The
+ * runtime's own failures are 502. Answers how many scripts were uploaded.
+ */
+export declare function createScopeStoreEnsurer(opts: ScopeStoreEnsurerOptions): (input: {
+    vertical: string;
+    stores: readonly ScopeStoreNeed[];
+    to?: string;
+    /** The fleet rebuild (#2309): a script that does not exist is left alone, never uploaded. */
+    existingOnly?: boolean;
+}) => Promise<{
+    uploaded: number;
+}>;
+/**
+ * Put a driver's release in the platform's canary script (D-63, #2300), where it is canaried
+ * before any vertical's script moves: `scopeStoreCanaryScriptName(driver)`, built and checked as a
+ * vertical's script is, except that it may move to a lower `schema`: every canary object is fresh
+ * and wiped, so none holds migrated data. With `existingOnly` instead of `to`, the fleet rebuild
+ * (#2309): the canary script is rebuilt at the release it runs, and left alone when there is none.
+ * Answers the script's name and whether it was uploaded.
+ */
+export declare function createStoreCanaryEnsurer(opts: ScopeStoreEnsurerOptions): (input: StoreCanaryTarget) => Promise<{
+    scriptName: string;
+    uploaded: boolean;
+}>;
+/** Where the canary script goes: to a release (a rollout), or rebuilt where it is (the fleet rebuild). */
+export type StoreCanaryTarget = {
+    driver: string;
+    to: string;
+} | {
+    driver: string;
+    existingOnly: true;
+};
+//# sourceMappingURL=store-script.d.ts.map

package/dist/store-script.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store-script.d.ts","sourceRoot":"","sources":["../src/store-script.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAE7B,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EAAwB,KAAK,wBAAwB,EAA0B,MAAM,kBAAkB,CAAC;AAE/G;;;GAGG;AACH,eAAO,MAAM,iBAAiB,2BAA6C,CAAC;AAE5E;;;GAGG;AACH,eAAO,MAAM,eAAe,yBAA2C,CAAC;AAExE,6EAA6E;AAC7E,MAAM,WAAW,0BAA2B,SAAQ,wBAAwB;IAC1E,+FAA+F;IAC/F,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,gGAAgG;IAChG,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,0FAA0F;IAC1F,kBAAkB,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,OAAO,CAAC,EAAE,SAAS,uBAAuB,EAAE,CAAC;IAC7C;;;;OAIG;IACH,WAAW,EAAE,CAAC,KAAK,EAAE;QACnB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;KAC1B,KAAK,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC,CAAC;CACvD;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,CAAC,wBAAwB,EAAE,kBAAkB,GAAG,oBAAoB,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAQ/I;AAYD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,wBAAwB,WAE/C;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,6FAA6F;IAC7F,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,KAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAMlC;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,wBAAwB,WAEhD,iBAAiB,KAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CAAC,CAW5F;AAED,0GAA0G;AAC1G,MAAM,MAAM,iBAAiB,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,IAAI,CAAA;CAAE,CAAC"}

package/dist/store-script.js

@@ -0,0 +1,188 @@
+import { SCOPE_STORE_CANARY, STORE_DRIVERS, scopeStoreCanaryScriptName, storeDriverReleaseDigest, } from '@substrat-run/contracts';
+import { ControlPlaneError } from '@substrat-run/control-plane-client';
+import { scopeStoreUploadPlan } from './deploy.js';
+import { STORE_OBJECT_CLASS, STORE_RESERVED_BINDING_PREFIX } from './store-entry.generated.js';
+import { withPythonStoreEntry } from './store-entry.js';
+/**
+ * The binding that records which pinned release a store script runs: the release bundle's
+ * sha256 (D-63). In the entry's reserved prefix, so the driver's env never carries it.
+ */
+export const STORE_RELEASE_VAR = `${STORE_RESERVED_BINDING_PREFIX}_RELEASE`;
+/**
+ * The binding that records what else the script was built from: a digest over every module the
+ * upload sent and its configuration. A changed entry, verifier, key list or flag changes it.
+ */
+export const STORE_BUILD_VAR = `${STORE_RESERVED_BINDING_PREFIX}_BUILD`;
+/**
+ * What the platform builds every store script from, as one digest (#2309): the entry, its
+ * verifier, the compatibility date and flags, and both key lists. It is the build digest of one
+ * fixed probe release, so it changes exactly when the platform's half of every script's build
+ * changes (a key added, dropped or revoked, an entry or verifier fix) and never when a driver's
+ * release does. The fleet rebuild converges every script on it.
+ */
+export async function storePlatformFingerprint(keys) {
+    const probe = 'driver_substrat_probe/probe.py';
+    return buildDigest(withPythonStoreEntry({ entry: probe, modules: [{ name: probe, content: new Uint8Array() }] }, { className: 'Probe', script: 'substrat-store-probe', callerPublicKeys: keys.callerPublicKeys, platformPublicKeys: keys.platformPublicKeys }));
+}
+/** Everything the upload sends, as one digest, so a script built from anything else differs. */
+async function buildDigest(script) {
+    const { modules, ...config } = script;
+    const named = modules.map((module) => ({ name: `module/${module.name}/${module.contentType}`, content: module.content }));
+    return storeDriverReleaseDigest({
+        entry: named[0].name,
+        modules: [...named, { name: 'config', content: new TextEncoder().encode(JSON.stringify(config)) }],
+    });
+}
+/**
+ * Put every (driver, vertical) store script a vertical's stores need in place, before the
+ * vertical's own upload (#2210, D-63). A push or a serve never moves a script to another
+ * release: only a rollout or a rollback does (#2300), and it says so with `to`.
+ *
+ * - No `to`, no script yet: it is uploaded at the driver's current good release.
+ * - No `to`, a script: it is rebuilt at the release it already runs, and uploaded again only when
+ *   what it was built from changed (the platform's entry, the verifier, a key list, a flag), so a
+ *   fix or a key rotation reaches it without a release moving.
+ * - `to`: the script is moved to that release, or left alone when it already runs it, built the
+ *   same way. Re-running a rollout therefore converges and uploads nothing twice.
+ *
+ * Every upload hashes the module bytes it is about to send and refuses (422) bytes whose
+ * sha256 the driver's registration does not pin, or that are not the release asked for. A driver
+ * with no release to run is refused (409). A script is never moved to a release whose pinned
+ * `schema` is lower than that of the release it runs (409): its objects may have migrated. The
+ * runtime's own failures are 502. Answers how many scripts were uploaded.
+ */
+export function createScopeStoreEnsurer(opts) {
+    const ensure = scriptEnsurer(opts);
+    return async (input) => {
+        const plan = scopeStoreUploadPlan({ vertical: input.vertical, stores: input.stores, drivers: opts.drivers });
+        let uploaded = 0;
+        for (const script of plan.scripts)
+            if (await ensure(script, input.to, { existingOnly: input.existingOnly }))
+                uploaded++;
+        return { uploaded };
+    };
+}
+/**
+ * Put a driver's release in the platform's canary script (D-63, #2300), where it is canaried
+ * before any vertical's script moves: `scopeStoreCanaryScriptName(driver)`, built and checked as a
+ * vertical's script is, except that it may move to a lower `schema`: every canary object is fresh
+ * and wiped, so none holds migrated data. With `existingOnly` instead of `to`, the fleet rebuild
+ * (#2309): the canary script is rebuilt at the release it runs, and left alone when there is none.
+ * Answers the script's name and whether it was uploaded.
+ */
+export function createStoreCanaryEnsurer(opts) {
+    const ensure = scriptEnsurer(opts);
+    return async (input) => {
+        const driver = (opts.drivers ?? STORE_DRIVERS).find((k) => k.driver === input.driver);
+        if (!driver)
+            throw new ControlPlaneError(403, `scope store driver '${input.driver}' is not registered`);
+        const scriptName = scopeStoreCanaryScriptName(driver.driver);
+        const uploaded = await ensure({ driver: driver.driver, vertical: SCOPE_STORE_CANARY, scriptName, className: driver.className, releases: driver.releases }, 'to' in input ? input.to : undefined, { anySchema: true, existingOnly: 'existingOnly' in input });
+        return { scriptName, uploaded };
+    };
+}
+function scriptEnsurer(opts) {
+    const scriptUrl = (name) => `https://api.cloudflare.com/client/v4/accounts/${opts.accountId}` +
+        `/workers/dispatch/namespaces/${opts.namespace}/scripts/${encodeURIComponent(name)}`;
+    return async (script, to, rule = {}) => {
+        const current = await recorded(scriptUrl(script.scriptName));
+        if (rule.existingOnly && !current.exists)
+            return false;
+        if (current.exists && !current.release) {
+            // Not an upload of this code. Which release it runs cannot be told, so it is not guessed.
+            throw new ControlPlaneError(409, `store script '${script.scriptName}' records no release`);
+        }
+        const wanted = to ?? current.release;
+        const release = await opts.loadRelease({
+            driver: script.driver,
+            vertical: script.vertical,
+            ...(wanted ? { sha256: wanted } : {}),
+        });
+        if (!release) {
+            throw new ControlPlaneError(409, wanted
+                ? `driver '${script.driver}' release ${wanted} is not stored`
+                : `driver '${script.driver}' has no release rolled out to '${script.vertical}'`);
+        }
+        // The bytes about to be sent, copied once and hashed now: never a hash taken when they
+        // arrived (D-63). The copy is what is built from and sent, so the hashed bytes are the
+        // sent bytes whatever the loader keeps or shares.
+        const loaded = {
+            entry: release.entry,
+            modules: release.modules.map((module) => ({ name: module.name, content: new Uint8Array(module.content) })),
+        };
+        const sha256 = await storeDriverReleaseDigest(loaded).catch((e) => {
+            throw new ControlPlaneError(422, e instanceof Error ? e.message : String(e));
+        });
+        const asked = wanted ?? release.sha256;
+        if (asked && sha256 !== asked) {
+            throw new ControlPlaneError(422, `driver '${script.driver}' release ${asked} loaded as other bytes (${sha256})`);
+        }
+        const pin = script.releases.find((p) => p.sha256 === sha256);
+        if (!pin) {
+            throw new ControlPlaneError(422, `driver '${script.driver}' release ${sha256} is not pinned in its registration`);
+        }
+        if (current.release && current.release !== sha256 && !rule.anySchema) {
+            const running = script.releases.find((p) => p.sha256 === current.release);
+            if (!running) {
+                throw new ControlPlaneError(409, `store script '${script.scriptName}' runs ${current.release}, which is not pinned, so its schema cannot be judged`);
+            }
+            if (pin.schema < running.schema) {
+                throw new ControlPlaneError(409, `store script '${script.scriptName}' runs schema ${running.schema}; release ${sha256} has schema ${pin.schema}, and a script never moves to a lower one. ` +
+                    `To undo a schema-raising rollout that stopped part way, converge it forward: post that rollout again`);
+            }
+        }
+        let built;
+        try {
+            built = withPythonStoreEntry(loaded, {
+                className: script.className,
+                script: script.scriptName,
+                callerPublicKeys: opts.callerPublicKeys,
+                platformPublicKeys: opts.platformPublicKeys,
+            });
+        }
+        catch (e) {
+            throw new ControlPlaneError(422, e instanceof Error ? e.message : String(e));
+        }
+        const build = await buildDigest(built);
+        if (current.exists && current.release === sha256 && current.build === build)
+            return false;
+        await put(scriptUrl(script.scriptName), script.scriptName, built, { sha256, build, first: !current.exists });
+        return true;
+    };
+    async function recorded(url) {
+        const res = await fetch(`${url}/settings`, { headers: { authorization: `Bearer ${opts.apiToken}` } });
+        if (res.status === 404)
+            return { exists: false };
+        if (!res.ok)
+            throw new ControlPlaneError(502, `store script settings failed (${res.status})`);
+        const body = (await res.json());
+        const text = (name) => body.result?.bindings?.find((b) => b.name === name)?.text;
+        return { exists: true, release: text(STORE_RELEASE_VAR), build: text(STORE_BUILD_VAR) };
+    }
+    async function put(url, scriptName, built, upload) {
+        const metadata = {
+            main_module: built.mainModule,
+            compatibility_date: built.compatibilityDate,
+            compatibility_flags: built.compatibilityFlags,
+            bindings: [
+                ...built.bindings,
+                { type: 'plain_text', name: STORE_RELEASE_VAR, text: upload.sha256 },
+                { type: 'plain_text', name: STORE_BUILD_VAR, text: upload.build },
+            ],
+            // The first upload declares the object class. Two pushes that both find no script both
+            // send this tag; the second is refused by the runtime (502), and a retry finds the script.
+            ...(upload.first ? { migrations: { new_tag: 'v1', new_sqlite_classes: [STORE_OBJECT_CLASS] } } : {}),
+        };
+        const form = new FormData();
+        form.set('metadata', new Blob([JSON.stringify(metadata)], { type: 'application/json' }), 'metadata.json');
+        for (const module of built.modules) {
+            // Sent as built, not copied again: every module is the ensurer's own copy or the entry's
+            // own encoding, so its buffer is an ArrayBuffer, and these are the bytes that were hashed.
+            form.set(module.name, new Blob([module.content], { type: module.contentType }), module.name);
+        }
+        const res = await fetch(url, { method: 'PUT', headers: { authorization: `Bearer ${opts.apiToken}` }, body: form });
+        if (!res.ok)
+            throw new ControlPlaneError(502, `store script upload failed (${res.status}) for '${scriptName}'`);
+    }
+}
+//# sourceMappingURL=store-script.js.map