@substrat-run/control-plane-api 0.170.0 → 0.172.0

package/dist/vertical-client.js

@@ -1,5 +1,5 @@
 import { z } from '@substrat-run/contracts';
-import { PREVIEW_CLIENT_PATH, attachmentRecord, projectedConnectionGrant, denialFilterParams, capabilityFilterParams, capabilityPage, capabilityRevokeAnswer, capabilityRevokeRequest, mintedPreviewClient, ownerSeat, ownerClaimLink, memberInviteLink, memberRemoval, scopeMembers, ownerTransferResult, previewClientClaim, retiredPreviewClients, peerGrantsEntry, peerSwitchOutcome, systemSwitchOutcome, lifecycleDelivery, systemScheduleEntry, switchedOffInUnit, exportedBatch, importResult, importCursorMoved, importState, substratError, CONNECTOR_ATTACHMENT_META_HEADER, CONNECTOR_ATTACHMENT_META_MAX_BYTES, CONNECTOR_ATTACHMENT_BUFFERED_MAX_BYTES, CONNECTOR_ATTACHMENT_STREAM_MAX_BYTES, connectorAttachmentMetaBytes, connectorAttachmentMetaRefusal, connectorAttachmentBufferedRefusal, connectorAttachmentSizeRefusal, CONNECTOR_ATTACHMENT_RECORD_HEADER, LOAD_STAMP_HEADER, PLATFORM_SECRET_HEADER, WRITE_REVISION_HEADER, mergeDumpPages, scopeDumpPage, } from '@substrat-run/contracts';
+import { CAPTURE_REFUSALS, PREVIEW_CLIENT_PATH, attachmentRecord, projectedConnectionGrant, denialFilterParams, capabilityFilterParams, capabilityPage, capabilityRevokeAnswer, capabilityRevokeRequest, mintedPreviewClient, ownerSeat, ownerClaimLink, memberInviteLink, memberRemoval, scopeMembers, ownerTransferResult, previewClientClaim, retiredPreviewClients, peerGrantsEntry, peerSwitchOutcome, systemSwitchOutcome, lifecycleDelivery, systemScheduleEntry, switchedOffInUnit, exportedBatch, importResult, importCursorMoved, importState, substratError, CONNECTOR_ATTACHMENT_META_HEADER, CONNECTOR_ATTACHMENT_META_MAX_BYTES, CONNECTOR_ATTACHMENT_BUFFERED_MAX_BYTES, CONNECTOR_ATTACHMENT_STREAM_MAX_BYTES, connectorAttachmentMetaBytes, connectorAttachmentMetaRefusal, connectorAttachmentBufferedRefusal, connectorAttachmentSizeRefusal, CONNECTOR_ATTACHMENT_RECORD_HEADER, LOAD_STAMP_HEADER, PLATFORM_SECRET_HEADER, WRITE_REVISION_HEADER, mergeDumpPages, scopeDumpChanges, scopeDumpPage, } from '@substrat-run/contracts';
 import { STREAM_IDLE_TIMEOUT_MS, STREAM_MAX_DURATION_MS, attachmentSha256, deadlineFetch, isSizedByteStream, knownLengthInit, runtimeSha256, scopeNotHeldRefusal, ulid, undrainedEventsOf, verifiedStream } from '@substrat-run/kernel';
 import { ControlPlaneError } from '@substrat-run/control-plane-client';
 /**
@@ -1170,9 +1170,12 @@ export class VerticalClient {
      * `exportScope` without. `'predates'` from a deployment built before the paged verbs (a 404, or
      * its own 501), which exports whole.
      */
-    async exportScopePage(scopeId, cursor, opts = {}) {
+    async exportScopePage(scopeId, cursor, 
+    /** `capture`: a copy's id, for a copy that catches up on writes (`exportScopeChanges`). */
+    opts = {}) {
         const base = this.options.baseUrl ?? 'https://vertical.invalid';
         const query = `scopeId=${encodeURIComponent(scopeId)}${opts.stamp ? '&stamp=1' : ''}` +
+            (opts.capture === undefined ? '' : `&capture=${encodeURIComponent(opts.capture)}`) +
             (cursor === null ? '' : `&cursor=${encodeURIComponent(cursor)}`);
         const answer = await withDeadline('export-page', PAGE_CALL_DEADLINE_MS, (signal) => this.routeAnswer('export-page', FENCED, () => this.options.fetch(`${base}/internal/export-page?${query}`, {
             headers: { [PLATFORM_SECRET_HEADER]: this.options.platformSecret },
@@ -1200,6 +1203,35 @@ export class VerticalClient {
     async discardStagedScope(scopeId, stageId) {
         await withDeadline('stage-discard', PAGE_CALL_DEADLINE_MS, (signal) => this.routeAnswer('stage-discard', FENCED, () => this.postRaw('/internal/stage/discard', { scopeId, stageId }, signal)));
     }
+    /**
+     * One batch of what `scopeId` took since the capture `copyId`'s first page (#2450). Batch `seq`
+     * acknowledges those before it, and the same `seq` asked again is served afresh, so a lost
+     * answer is retried safely. `'predates'` from a deployment that cannot catch a copy up.
+     */
+    async exportScopeChanges(scopeId, copyId, seq) {
+        const answer = await withDeadline('export-changes', PAGE_CALL_DEADLINE_MS, (signal) => this.routeAnswer('export-changes', FENCED, () => this.postRaw('/internal/export-changes', { scopeId, copyId, seq }, signal)));
+        if (answer === PREDATES)
+            return 'predates';
+        const changes = scopeDumpChanges.safeParse(answer);
+        if (!changes.success)
+            throw new ControlPlaneError(502, 'export-changes: the vertical answered something that is not a change batch');
+        return changes.data;
+    }
+    /** End the capture `copyId` of `scopeId` (#2450): a copy that stopped. Best effort, as a discard is. */
+    async endScopeCapture(scopeId, copyId) {
+        await withDeadline('capture-end', PAGE_CALL_DEADLINE_MS, (signal) => this.routeAnswer('capture-end', FENCED, () => this.postRaw('/internal/capture/end', { scopeId, copyId }, signal)));
+    }
+    /**
+     * Apply one batch of a capture's changes to the stage `stageId` here (#2450), numbered after its
+     * pages. `'predates'` from a deployment that cannot catch a copy up; a batch already applied
+     * answers `staged: false`.
+     */
+    async stageScopeChanges(scopeId, stageId, seq, tables) {
+        const answer = await withDeadline('stage-changes', PAGE_CALL_DEADLINE_MS, (signal) => this.routeAnswer('stage-changes', FENCED, () => this.postRaw('/internal/stage-changes', { scopeId, stageId, seq, tables }, signal)));
+        if (answer === PREDATES)
+            return 'predates';
+        return { staged: answer?.staged === true };
+    }
     /** A platform-authenticated JSON POST, its response unread, for `routeAnswer`. */
     postRaw(path, body, signal) {
         const base = this.options.baseUrl ?? 'https://vertical.invalid';
@@ -1761,6 +1793,12 @@ export class VerticalClient {
 }
 /** How many times a copy is restarted because the source took a write while it was read (#2450). */
 export const TRANSFER_ATTEMPTS = 3;
+/**
+ * How many change batches one attempt's catch-up reads before it gives up (#2450): a source that
+ * writes faster than the copy drains its log never reaches a final batch, and the copy then starts
+ * again like one torn at a quiet moment, and is refused once the attempts are spent.
+ */
+export const CATCH_UP_BATCHES = 50;
 /**
  * Move a scope's data out of `source` and into `dest`'s stage, a page at a time (#2450), so no
  * single call carries the whole scope: a Durable Object RPC message holds 32 MiB, and an isolate
@@ -1771,10 +1809,18 @@ export const TRANSFER_ATTEMPTS = 3;
  * before. A destination that predates them gets the pages merged into one dump, which is the old
  * ceiling, and only there.
  *
- * Each page is read in one turn of the source, so it is consistent on its own; the pages are one
- * consistent read only while the source's write revision stays where the first page read it. When
- * it moves the copy starts again, up to `TRANSFER_ATTEMPTS` times, and is then refused (409) with
- * nothing moved: a carry's later fences assume a copy that was true at one moment.
+ * Each page is read in one turn of the source, so it is consistent on its own. A carry's later
+ * fences assume a copy that was true at one moment, and there are two ways to get one. A source
+ * that can capture (#2450 step 3) logs every write from its first page on, and once the pages are
+ * staged the copy catches up on them in change batches until a final batch, read in the same turn
+ * as the log ran empty, names that moment. A source that cannot (an older deployment, or a table
+ * with a unique expression index, which no row trigger can key) is copied at a quiet moment: the
+ * pages are one read only while its write revision stays where the first page read it.
+ *
+ * Either way a copy that does not come true starts again, up to `TRANSFER_ATTEMPTS` times, and is
+ * then refused (409) with nothing moved: a quiet-moment copy the source wrote under, or a catch-up
+ * whose capture was lost (it expired, a migration moved the schema, a load replaced the store) or
+ * that did not reach a final batch in `CATCH_UP_BATCHES`.
  *
  * `beginTransfer` and `transferStep` are the same copy in resumable steps, for one that outlasts a
  * request; this runs them to the end.
@@ -1783,48 +1829,108 @@ export async function transferScope(source, dest,
 /** The scope read at `source`, and (a fork, #2450) the scope `dest` stages into, when another. */
 scope, opts = {}) {
     let progress = await beginTransfer({ source, dest }, scope, opts);
-    while ('state' in progress)
-        progress = await transferStep({ source, dest }, progress.state, opts);
+    while ('state' in progress) {
+        const { state } = progress;
+        try {
+            progress = await transferStep({ source, dest }, state, opts);
+        }
+        catch (e) {
+            // Nobody resumes a copy run in one request, so its stage and capture go with it. (A job's
+            // survive a fault, for the next step to resume.)
+            await abandonCopy({ source, dest }, state);
+            throw e;
+        }
+    }
     return progress.done;
 }
 /**
  * Start a copy (#2450): read the source's first page and stage it. Finished at once when the scope
  * fits that page (unless `alwaysStage`), when the source predates the paged verbs, or when the
  * destination does (its pages are then merged here, as the whole dump it takes).
+ *
+ * The first page asks the source to capture under the stage's id. A source that refuses (412: a
+ * table it cannot capture) is read again without one, and one that answers a plain page (it does
+ * not echo `captured`) has its capture ended: either way the copy is read at a quiet moment. A copy
+ * that catches up and fits its first page catches up here, so it is still finished at once.
  */
-export async function beginTransfer({ source, dest }, scope, opts = {}) {
+export async function beginTransfer(ends, scope, opts = {}) {
+    const { source, dest } = ends;
     const retry = opts.retry ?? ((run) => run());
     const stamp = opts.stamp ?? true;
     const from = scope.from;
     const into = scope.to ?? scope.from;
+    // The stage's id names the capture too, so whoever discards a stage ends the capture beside it.
+    const stageId = ulid();
+    const ask = opts.capture !== false && typeof source.exportScopeChanges === 'function';
+    /** The source refused to open the capture, so it holds none. */
+    let refused = false;
     // A client without the paged verbs (one a caller assembled from part of this one) is read as a
     // deployment that predates them.
-    const first = typeof source.exportScopePage === 'function'
-        ? await retry(() => source.exportScopePage(from, null, { stamp }))
-        : 'predates';
+    const first = typeof source.exportScopePage !== 'function'
+        ? 'predates'
+        : ask
+            ? await retry(() => source.exportScopePage(from, null, { stamp, capture: stageId })).catch((e) => {
+                if (!captureRefusal(e, 'expressionIndex'))
+                    throw e;
+                refused = true;
+                return retry(() => source.exportScopePage(from, null, { stamp }));
+            })
+            : await retry(() => source.exportScopePage(from, null, { stamp }));
     if (first === 'predates') {
         if (!stamp) {
             const tables = await retry(() => source.exportScope(from));
-            return { done: { dump: tables, loadStamp: null, revision: null, meta: metaOf([tables]) } };
+            return { done: { dump: tables, loadStamp: null, revision: null, meta: doneMeta(metaOf([tables])) } };
         }
         const whole = await retry(() => source.exportScopeStamped(from));
-        return { done: { dump: whole.tables, loadStamp: whole.loadStamp, revision: whole.revision, meta: metaOf([whole.tables]) } };
+        return { done: { dump: whole.tables, loadStamp: whole.loadStamp, revision: whole.revision, meta: doneMeta(metaOf([whole.tables])) } };
     }
+    const captured = ask && first.captured === true;
+    // A host that took the capture but predates the echo (#2761) still opened one: end it.
+    if (ask && !captured && !refused)
+        await endCapture(ends, from, stageId);
     if (first.next === null && !opts.alwaysStage) {
-        return { done: { dump: first.tables, loadStamp: first.loadStamp, revision: first.revision, meta: metaOf([first.tables]) } };
+        // One page is one read whatever the source takes next, so its capture is not needed.
+        if (captured)
+            await endCapture(ends, from, stageId);
+        return { done: { dump: first.tables, loadStamp: first.loadStamp, revision: first.revision, meta: doneMeta(metaOf([first.tables])) } };
+    }
+    let opened;
+    try {
+        opened = typeof dest.stageScopePage === 'function'
+            ? await retry(() => dest.stageScopePage(into, stageId, 0, first.tables))
+            : 'predates';
+    }
+    catch (e) {
+        if (captured)
+            await endCapture(ends, from, stageId);
+        throw e;
     }
-    const stageId = ulid();
-    const opened = typeof dest.stageScopePage === 'function'
-        ? await retry(() => dest.stageScopePage(into, stageId, 0, first.tables))
-        : 'predates';
     const state = {
         from, into, stamp, stageId, seq: 0, cursor: first.next,
         revision: first.revision, loadStamp: first.loadStamp,
         attempt: opts.attempt ?? 1, pages: 1, rows: rowsIn(first.tables), meta: metaOf([first.tables]),
+        ...(captured && opened !== 'predates' ? { capture: true, batch: 0 } : {}),
     };
-    if (opened !== 'predates')
-        return state.cursor === null ? { done: doneOf(state) } : { state };
-    // A destination that cannot stage takes the dump whole, so the pages are held here instead.
+    if (opened !== 'predates') {
+        if (state.cursor !== null)
+            return { state };
+        if (!state.capture)
+            return { done: doneOf(state) };
+        // Caught up here, within a step's budget: a source still writing is left for the next step.
+        const until = Date.now() + BEGIN_CATCH_UP_MS;
+        try {
+            return await transferStep(ends, state, { ...opts, until: () => Date.now() >= until });
+        }
+        catch (e) {
+            // Nothing names this copy yet, so nothing else would discard its stage or end its capture.
+            await abandonCopy(ends, state);
+            throw e;
+        }
+    }
+    // A destination that cannot stage takes the dump whole, so the pages are held here instead, read
+    // at a quiet moment: it cannot take a change batch either.
+    if (captured)
+        await endCapture(ends, from, stageId);
     const held = [first.tables];
     for (let cursor = first.next; cursor !== null;) {
         const at = cursor;
@@ -1832,11 +1938,11 @@ export async function beginTransfer({ source, dest }, scope, opts = {}) {
         if (page === 'predates')
             throw stoppedPaging(from);
         if (page.revision !== first.revision)
-            return restart({ source, dest }, state, opts, false);
+            return restart(ends, state, { ...opts, capture: false }, false);
         held.push(page.tables);
         cursor = page.next;
     }
-    return { done: { dump: mergeDumpPages(held), loadStamp: first.loadStamp, revision: first.revision, meta: metaOf(held) } };
+    return { done: { dump: mergeDumpPages(held), loadStamp: first.loadStamp, revision: first.revision, meta: doneMeta(metaOf(held)) } };
 }
 /**
  * Move the next pages of a copy (#2450), until it is staged whole or `until` says to stop (a job's
@@ -1845,17 +1951,32 @@ export async function beginTransfer({ source, dest }, scope, opts = {}) {
  *
  * `stopped` (a job being cancelled or forgotten): the copy starts nothing new. A torn copy then
  * discards its stage and answers where it was, opening no other: whoever stopped it settles it.
+ *
+ * Once a captured copy's pages are staged, the step catches up: each change batch the source
+ * serves is staged after the last page, until a final one. A batch is staged at the first number
+ * the stage has not taken (`staged: false` means it has), so a step resumed from a record older
+ * than the stage (a step that faulted after staging) applies the batch served now after the one
+ * staged then, and the newer rows win.
  */
-export async function transferStep({ source, dest }, state, opts = {}) {
+export async function transferStep(ends, state, opts = {}) {
+    const { source, dest } = ends;
     const retry = opts.retry ?? ((run) => run());
     let next = state;
     while (next.cursor !== null) {
         const at = next.cursor;
-        const page = await retry(() => source.exportScopePage(next.from, at, { stamp: next.stamp }));
+        const capture = next.capture ? { capture: next.stageId } : {};
+        const page = await retry(() => source.exportScopePage(next.from, at, { stamp: next.stamp, ...capture })).catch((e) => {
+            // A capture that is gone (ended, expired, its schema moved) cannot vouch for the pages.
+            if (next.capture && captureRefusal(e, 'gone', 'schemaMoved'))
+                return 'lost';
+            throw e;
+        });
+        if (page === 'lost')
+            return restart(ends, next, opts, true);
         if (page === 'predates')
             throw stoppedPaging(next.from);
-        if (page.revision !== next.revision)
-            return restart({ source, dest }, next, opts, true);
+        if (!next.capture && page.revision !== next.revision)
+            return restart(ends, next, opts, true);
         const seq = next.seq + 1;
         const staged = await retry(() => dest.stageScopePage(next.into, next.stageId, seq, page.tables));
         if (staged === 'predates') {
@@ -1867,18 +1988,71 @@ export async function transferStep({ source, dest }, state, opts = {}) {
             cursor: page.next,
             pages: next.pages + 1,
             rows: next.rows + rowsIn(page.tables),
-            meta: mergeDumpPages([next.meta, metaOf([page.tables])]),
+            meta: metaOf([next.meta, page.tables]),
         };
         if (next.cursor !== null && opts.until?.())
             return { state: next };
     }
+    while (next.capture) {
+        const batch = next.batch ?? 0;
+        // Stopped: no batch more, the first one included. A budget spent still lets one batch through.
+        if (opts.stopped?.())
+            return { state: next };
+        if (batch >= (opts.catchUpBatches ?? CATCH_UP_BATCHES))
+            return restart(ends, next, opts, true);
+        if (batch > 0 && opts.until?.())
+            return { state: next };
+        const at = next;
+        const changes = typeof source.exportScopeChanges !== 'function'
+            ? 'predates'
+            : await retry(() => source.exportScopeChanges(at.from, at.stageId, batch)).catch((e) => {
+                // Gone, or its schema moved: a final answer that was lost, asked again, is gone too.
+                if (captureRefusal(e, 'gone', 'schemaMoved'))
+                    return 'lost';
+                throw e;
+            });
+        if (changes === 'lost')
+            return restart(ends, next, opts, true);
+        // An end that cannot catch up: the copy is one read only if the source took no write since.
+        if (changes === 'predates')
+            return restart(ends, next, { ...opts, capture: false }, true);
+        let seq = next.seq + 1;
+        if (changes.tables.length > 0) {
+            for (;;) {
+                const staged = typeof dest.stageScopeChanges !== 'function'
+                    ? 'predates'
+                    : await retry(() => dest.stageScopeChanges(at.into, at.stageId, seq, changes.tables));
+                if (staged === 'predates')
+                    return restart(ends, next, { ...opts, capture: false }, true);
+                if (staged.staged)
+                    break;
+                seq++;
+            }
+        }
+        else {
+            seq = next.seq;
+        }
+        next = {
+            ...next,
+            seq,
+            batch: batch + 1,
+            rows: next.rows + rowsIn(changes.tables),
+            meta: foldMeta(next.meta, changes.tables),
+        };
+        if (changes.final) {
+            // The source ended the capture with this batch: the copy is true at the moment it names.
+            const { capture: _, batch: __, ...caught } = next;
+            return { done: doneOf({ ...caught, revision: changes.revision }) };
+        }
+    }
     return { done: doneOf(next) };
 }
 /** A torn copy: discard its stage and begin again, or refuse once the attempts are spent. */
 async function restart(ends, state, opts, staged) {
-    // Best effort: a stage nothing commits is replaced by the next one, and goes with a wipe.
+    // Best effort: a stage nothing commits is replaced by the next one, and goes with a wipe. (An
+    // unstaged copy holds no capture: it was ended when the destination turned out unable to stage.)
     if (staged)
-        await ends.dest.discardStagedScope(state.into, state.stageId).catch(() => undefined);
+        await abandonCopy(ends, state);
     // Stopped: a new stage would outlast whoever stopped this copy, so none is opened.
     if (opts.stopped?.())
         return { state };
@@ -1889,9 +2063,67 @@ async function restart(ends, state, opts, staged) {
     }
     return beginTransfer(ends, { from: state.from, to: state.into }, { ...opts, stamp: state.stamp, attempt: state.attempt + 1 });
 }
-const doneOf = (state) => ({ dump: { stageId: state.stageId }, loadStamp: state.loadStamp, revision: state.revision, meta: state.meta });
+/**
+ * End the capture `copyId` at the source (#2450), best effort: one left open expires on its own once
+ * nobody reads it. Every copy that stops (failed, cancelled, superseded or restarted) calls it.
+ */
+export async function endCapture(ends, from, copyId) {
+    await ends.source.endScopeCapture?.(from, copyId).catch(() => undefined);
+}
+/**
+ * Give a copy up (#2450), best effort: discard its stage at the destination and end its capture at
+ * the source, the two halves every copy that stops leaves behind. An end left out is skipped (one
+ * that could not be resolved); its half then waits for a wipe, or expires.
+ */
+export async function abandonCopy(ends, state) {
+    await ends.dest?.discardStagedScope(state.into, state.stageId).catch(() => undefined);
+    if (state.capture && ends.source)
+        await endCapture({ source: ends.source }, state.from, state.stageId);
+}
+/** How long a copy's first request catches up before it leaves the rest to a step, ms. */
+const BEGIN_CATCH_UP_MS = 20_000;
+/** Whether `e` is the source's capture refusing for one of `kinds` (`CAPTURE_REFUSALS`), not another 412. */
+const captureRefusal = (e, ...kinds) => e instanceof ControlPlaneError && e.status === 412 && kinds.some((k) => e.message.includes(CAPTURE_REFUSALS[k]));
+const doneOf = (state) => ({ dump: { stageId: state.stageId }, loadStamp: state.loadStamp, revision: state.revision, meta: doneMeta(state.meta) });
 const rowsIn = (tables) => tables.reduce((n, t) => n + t.rows.length, 0);
 const stoppedPaging = (scopeId) => new ControlPlaneError(502, `the source of scope ${scopeId} stopped answering dump pages part way; nothing was moved`);
-/** The `_substrat_meta` fragments of some dump pages, merged. */
-const metaOf = (pages) => mergeDumpPages(pages.map((page) => page.filter((t) => t.name === '_substrat_meta')));
+const META = '_substrat_meta';
+/**
+ * The `_substrat_meta` fragments of some dump pages, merged, each row still named by its rowid
+ * when every fragment carried them (a captured copy's), so a change batch can fold into it.
+ */
+const metaOf = (pages) => {
+    let held;
+    for (const t of pages.flat()) {
+        if (t.name !== META)
+            continue;
+        if (!held) {
+            held = { ...t, rows: [...t.rows], ...(t.rowids ? { rowids: [...t.rowids] } : {}) };
+            continue;
+        }
+        held.rows.push(...t.rows);
+        if (held.rowids && t.rowids)
+            held.rowids.push(...t.rowids);
+        else
+            delete held.rowids;
+    }
+    return held ? [held] : [];
+};
+/** `meta` with a change batch's `_substrat_meta` change applied: by rowid, or whole when it came whole. */
+const foldMeta = (meta, tables) => {
+    const change = tables.find((t) => t.name === META);
+    if (!change)
+        return meta;
+    const { deleted, ...table } = change;
+    if (!change.rowids)
+        return [table];
+    const held = meta[0];
+    const rows = new Map(held?.rowids ? held.rowids.map((id, i) => [id, held.rows[i]]) : []);
+    for (const id of deleted)
+        rows.delete(id);
+    change.rowids.forEach((id, i) => rows.set(id, change.rows[i]));
+    return [{ ...(held ?? table), rows: [...rows.values()], rowids: [...rows.keys()] }];
+};
+/** What a finished copy reports of `_substrat_meta`: its rows, the rowids left behind. */
+const doneMeta = (meta) => meta.map(({ rowids: _, ...t }) => t);
 //# sourceMappingURL=vertical-client.js.map