@substrat-run/control-plane-api 0.159.0 → 0.161.0

package/dist/platform-drain.js

@@ -1,6 +1,6 @@
-import { INERT_SCOPE_REASON, isPrimaryScope, lifecycleRefusal, platformIntentFailureMessage, ulid, } from '@substrat-run/kernel';
+import { INERT_SCOPE_REASON, isPrimaryScope, lifecycleRefusal, platformIntentFailureMessage, resolveScopeRecord, ulid, withheldOpsFailure, withheldStaffFailure, } from '@substrat-run/kernel';
 import { isTerminalDispatchFailure } from './provider-error.js';
-import { principalId as principalIdSchema, scopeId as scopeIdSchema, tenantId as tenantIdSchema, provisionSiblingPayload, archiveScopePayload, callsDeclares, peerInvokePayload, undeclaredCallMessage, provisionTenantPayload, setEntitlementsPayload, connectorDispatchPayload, CONNECTOR_DISPATCH_KIND_PREFIX, modelUsageLine, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, isUndecodablePlatformRequest, platformRequestId, sweepRunsPayload, } from '@substrat-run/contracts';
+import { principalId as principalIdSchema, scopeId as scopeIdSchema, tenantId as tenantIdSchema, provisionSiblingPayload, archiveScopePayload, callsDeclares, peerInvokePayload, undeclaredCallMessage, provisionTenantPayload, setEntitlementsPayload, connectorDispatchPayload, CONNECTOR_DISPATCH_KIND_PREFIX, modelUsageLine, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, isUndecodablePlatformRequest, platformRequestId, createTenantInput, errorCodeOf, sweepRunsPayload, neutralUpstreamMarkers, withoutUpstreamReference, } from '@substrat-run/contracts';
 import { ControlPlaneError } from '@substrat-run/control-plane-client';
 import { attributeFailure, terminalFailureNote } from './failure-attribution.js';
 import { connectionGrantsForScope } from './vertical-client.js';
@@ -32,6 +32,92 @@ const undecodedRefusal = (decodeError) => `not executed: the intent row could no
  * fault, short enough that the proposer learns the truth while it still matters.
  */
 export const MAX_PLATFORM_REQUEST_ATTEMPTS = 100;
+/**
+ * The ops-failure row of an intent the drain gave up on or settled `failed` (#570, #618). #2554:
+ * an intent's error is whatever its handler met, a provider's answer or another vertical's
+ * deployment among them, so the scope's tenant's row says only what became of the intent, under
+ * the platform's reference, and the error is staff's row under the same one (`withheldOpsFailure`).
+ */
+function intentFailure(ctx, kind, stage, outcome, raw, answered, reference) {
+    return withheldOpsFailure({
+        operation: `intent.${kind}`,
+        stage,
+        tenantId: ctx.tenantId,
+        scopeId: ctx.scopeId,
+        vertical: ctx.vertical,
+        version: ctx.versionId ?? null,
+        // The attribution the settle journals (#841) rides onto the fleet row too (#1233).
+        origin: outcome.failure?.origin ?? null,
+        code: outcome.failure?.code ?? null,
+    }, raw, answered, reference);
+}
+/** The stage of staff's row for a withheld failure on a pass that left the intent pending (#2556). */
+const WITHHELD_TRANSIENT_STAGE = 'transient';
+/**
+ * How long staff's row for a withheld transient failure stands for the same intent (#2556). Every
+ * pass of a stuck intent journals the same reference meanwhile, and files nothing: the staff digest
+ * mails every row since its watermark, so a row per pass would repeat one stuck intent every quarter
+ * hour. A condition that lasts is filed again after this, under a new reference.
+ */
+export const WITHHELD_REFILE_MS = 24 * 60 * 60_000;
+/** The stage of staff's note that a withheld transient reference could not be looked up (#2556). */
+const WITHHELD_LOOKUP_STAGE = 'transient-lookup';
+/** How a journaled withheld sentence ends: the platform's reference staff's row is filed under. */
+const JOURNALED_REFERENCE = /^(.*) \(reference ([0-9A-HJKMNP-TV-Z]{26})\)$/s;
+/**
+ * How staff's row for a withheld transient failure opens: the intent, and the sentence its scope was
+ * told. A sentence may quote the payload (a peer call's operation, a tenant slug), so the opening is
+ * stored after any upstream marker, which is read from the raw text alone, and neutralised as the
+ * row stores it (`withUpstreamReference`), which is also the form the next pass compares.
+ */
+const withheldTransientOpening = (id, withheld) => neutralUpstreamMarkers(platformIntentFailureMessage(id, `pending (${withheld}): `));
+/**
+ * The reference a withheld failure on a pass that left the intent pending is journaled under
+ * (#2556), keyed on the intent AND the sentence: a handler that gets one step further on a later
+ * pass is a different failure, with its own row. The previous settle journaled `<sentence>
+ * (reference R)` in the intent's `lastError`; when that sentence is this pass's, R is reused with
+ * nothing filed, once one ledger read by R confirms staff's row for this intent and sentence stands
+ * in the drained scope within `WITHHELD_REFILE_MS`. Otherwise a new reference is minted and staff's
+ * row filed under it, with no tenant, in the drained scope, so a subject erasure of the intent
+ * reaches it; the row opens with the sentence (`withheldTransientOpening`), which is what the next
+ * pass matches, never the raw text, whose upstream handles vary per call. A ledger that cannot be
+ * read files anew, since a second row is the safer error than a reference nothing resolves, and
+ * leaves staff a note saying the earlier reference went unconfirmed.
+ */
+async function withheldTransientReference(ctx, kind, id, previousError, withheld, outcome, opts) {
+    const operation = `intent.${kind}`;
+    const opening = withheldTransientOpening(id, withheld);
+    const entry = {
+        operation,
+        scopeId: ctx.scopeId,
+        vertical: ctx.vertical,
+        version: ctx.versionId ?? null,
+        origin: outcome.failure?.origin ?? null,
+        code: outcome.failure?.code ?? null,
+    };
+    const journaled = JOURNALED_REFERENCE.exec(previousError ?? '');
+    const previous = journaled?.[1] === withheld ? journaled[2] : null;
+    if (previous !== null && opts?.listFailures) {
+        const since = new Date((opts.now?.() ?? new Date()).getTime() - WITHHELD_REFILE_MS).toISOString();
+        try {
+            const rows = await opts.listFailures({ scopeId: ctx.scopeId, operation, reference: previous, since, limit: 10 });
+            const filed = rows.some((r) => r.tenantId === null && r.stage === WITHHELD_TRANSIENT_STAGE && withoutUpstreamReference(r.message).startsWith(opening));
+            if (filed)
+                return previous;
+        }
+        catch (e) {
+            // Filed anew below; the note is staff's, and names only the reference and the ledger's own error.
+            opts.recordFailure?.(withheldStaffFailure({ ...entry, stage: WITHHELD_LOOKUP_STAGE }, platformIntentFailureMessage(id, `pending: reference ${previous} could not be confirmed in the failure ledger, so this pass files anew: ${e instanceof Error ? e.message : String(e)}`), ulid()));
+        }
+    }
+    const reference = ulid();
+    opts?.recordFailure?.(withheldStaffFailure({ ...entry, stage: WITHHELD_TRANSIENT_STAGE }, outcome.error ?? 'unknown', reference, opening));
+    return reference;
+}
+/** What a withheld failure's settle journals: its origin and code, never a permission read from the text. */
+function withheldAttribution(failure) {
+    return failure ? { ...failure, permission: null } : null;
+}
 /**
  * Drain one scope's pending platform intents: list them from the vertical, dispatch each to the
  * handler for its `kind`, and settle the outcome back in the vertical. An unknown kind settles
@@ -118,30 +204,30 @@ export async function drainScopePlatformRequests(client, ctx, handlers, opts) {
         // lands an ops-failure row (#559) so a six-day retry loop is an operator's headline,
         // not a spine-table archaeology find.
         const maxAttempts = opts?.maxAttempts ?? MAX_PLATFORM_REQUEST_ATTEMPTS;
+        // #2556: a withheld error is journaled as its sentence under the platform's reference, the
+        // reference staff's row is filed under (terminal and give-up: the pair below; a pass that
+        // leaves the intent pending: `withheldTransientReference`).
+        const withheld = outcome.withheld;
+        let journaled = outcome.error ?? null;
         if (outcome.status === 'pending' && attempts + 1 >= maxAttempts) {
+            const gaveUp = `gave up after ${attempts + 1} drain attempts`;
             outcome = {
                 status: 'failed',
                 result: outcome.result,
-                error: `gave up after ${attempts + 1} drain attempts — last error: ${outcome.error ?? 'unknown'}`,
+                error: `${gaveUp} — last error: ${outcome.error ?? 'unknown'}`,
                 // The give-up inherits the last pass's attribution rather than inventing one: what
                 // refused a hundred times is what refused on the last attempt.
                 failure: outcome.failure,
+                ...(withheld === undefined ? {} : { withheld }),
             };
-            opts?.recordFailure?.({
-                operation: `intent.${kind}`,
-                stage: 'attempt-ceiling',
-                tenantId: ctx.tenantId,
-                scopeId: ctx.scopeId,
-                vertical: ctx.vertical,
-                version: ctx.versionId ?? null,
-                // The attribution the settle journals (#841) rides onto the fleet row too —
-                // until now the drain had {origin, code} in hand and dropped it here (#1233).
-                origin: outcome.failure?.origin ?? null,
-                code: outcome.failure?.code ?? null,
-                message: platformIntentFailureMessage(id, outcome.error ?? 'unknown'),
-            });
+            const reference = ulid();
+            journaled = withheld === undefined ? outcome.error : `${gaveUp} — last error: ${withheld} (reference ${reference})`;
+            opts?.recordFailure?.(intentFailure(ctx, kind, 'attempt-ceiling', outcome, platformIntentFailureMessage(id, outcome.error ?? 'unknown'), (r) => platformIntentFailureMessage(id, `${gaveUp} (reference ${r})`), reference));
         }
         else if (outcome.status === 'failed' && !inert) {
+            const reference = ulid();
+            if (withheld !== undefined)
+                journaled = `${withheld} (reference ${reference})`;
             // An inert settle (#2005) is expected, not an operator's headline; the settle below
             // still journals its reason and attribution.
             //
@@ -149,17 +235,11 @@ export async function drainScopePlatformRequests(client, ctx, handlers, opts) {
             // over, nobody is coming back to it, and until now its only trace was a `last_error`
             // column in the vertical's own DO. An unknown kind, a refused payload, a provider's 4xx:
             // each is an operator's headline for the same reason a give-up is.
-            opts?.recordFailure?.({
-                operation: `intent.${kind}`,
-                stage: 'terminal',
-                tenantId: ctx.tenantId,
-                scopeId: ctx.scopeId,
-                vertical: ctx.vertical,
-                version: ctx.versionId ?? null,
-                origin: outcome.failure?.origin ?? null,
-                code: outcome.failure?.code ?? null,
-                message: platformIntentFailureMessage(id, `failed: ${outcome.error ?? 'unknown'}`),
-            });
+            opts?.recordFailure?.(intentFailure(ctx, kind, 'terminal', outcome, platformIntentFailureMessage(id, `failed: ${outcome.error ?? 'unknown'}`), (r) => platformIntentFailureMessage(id, `failed (reference ${r})`), reference));
+        }
+        else if (outcome.status === 'pending' && withheld !== undefined) {
+            const previousError = isUndecodablePlatformRequest(request) ? null : request.lastError;
+            journaled = `${withheld} (reference ${await withheldTransientReference(ctx, kind, id, previousError, withheld, outcome, opts)})`;
         }
         // No special case for a refused row: every row that reaches here has an id the settle
         // route accepts (#1637 left the rest out above), so its settle is refused for the same
@@ -170,8 +250,8 @@ export async function drainScopePlatformRequests(client, ctx, handlers, opts) {
         await client.settlePlatformRequest(ctx.tenantId, ctx.scopeId, id, {
             status,
             result: outcome.result,
-            lastError: outcome.error ?? null,
-            failure: outcome.failure ?? null,
+            lastError: journaled,
+            failure: withheld === undefined ? (outcome.failure ?? null) : withheldAttribution(outcome.failure),
             ...(outcome.event ? { event: outcome.event } : {}),
         });
         report[status]++;
@@ -223,6 +303,7 @@ export async function provisionSiblingScope(deps, input) {
         slug: input.slug,
         name: input.name,
         vertical: parent.vertical,
+        verticalSource: 'copy',
         jurisdiction: parent.jurisdiction ?? undefined,
     });
     const created = await adoptServingForProvisioning(host, actor, await admin.getScopeRecord(actor, input.tenantId, input.scopeId));
@@ -273,11 +354,23 @@ export async function provisionSiblingScope(deps, input) {
  */
 export function provisionSiblingHandler(deps) {
     return async (ctx, request) => {
-        const payload = provisionSiblingPayload.parse(request.payload);
+        const parsed = provisionSiblingPayload.safeParse(request.payload);
+        if (!parsed.success)
+            return malformedPayload('provision-sibling', parsed.error);
+        const payload = parsed.data;
         const prior = request.result;
-        const scopeId = prior && typeof prior === 'object' && 'scopeId' in prior
-            ? scopeIdSchema.parse(prior.scopeId)
-            : scopeIdSchema.parse(ulid());
+        // The sibling's own input, checked before `provisionSiblingScope` writes its row, by the rules
+        // that write applies: one it would refuse settles here and leaves no row behind.
+        const checked = checkedBeforeWrites(() => {
+            const scopeId = prior && typeof prior === 'object' && 'scopeId' in prior
+                ? scopeIdSchema.parse(prior.scopeId)
+                : scopeIdSchema.parse(ulid());
+            resolveScopeRecord({ tenantId: ctx.tenantId, scopeId, slug: payload.slug, name: payload.name });
+            return scopeId;
+        });
+        if (!checked.ok)
+            return checked.refusal;
+        const scopeId = checked.value;
         try {
             const r = await provisionSiblingScope(deps, {
                 tenantId: ctx.tenantId,
@@ -292,12 +385,58 @@ export function provisionSiblingHandler(deps) {
                 : { status: 'failed', result: { scopeId }, error: r.error };
         }
         catch (e) {
+            // After the writes have begun, a throw is classified as it always was: a vertical's refusal
+            // is transient, and anything else leaves the request pending. A refusal settled terminal here
+            // could leave a half-provisioned sibling that no later pass finishes.
             if (e instanceof ControlPlaneError)
                 return { status: 'pending', result: { scopeId }, error: e.message };
             throw e;
         }
     };
 }
+/** The attribution of a request refused for its own input: ours, and the input's to fix. */
+const INPUT_REFUSED = { origin: 'platform', code: 'validation_failed', permission: null };
+/**
+ * #2556: a throw whose text is not the drained scope's to read (`PlatformRequestOutcome.withheld`),
+ * kept pending as the drain's own catch would keep it, attributed as that catch would, and
+ * journaled as `sentence`.
+ */
+function withheldThrow(e, sentence, result) {
+    return {
+        status: 'pending',
+        ...(result === undefined ? {} : { result }),
+        error: e instanceof Error ? e.message : String(e),
+        failure: attributeFailure(e),
+        withheld: sentence,
+    };
+}
+/** A payload that does not parse, settled `failed` at once: it never will parse. */
+function malformedPayload(kind, error) {
+    const fields = error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ');
+    return { status: 'failed', error: `${kind} payload is malformed: ${fields}`, failure: INPUT_REFUSED };
+}
+/**
+ * A request's input, checked before its handler writes anything: `check` parses and resolves, and
+ * performs no I/O. A `validation_failed` from it (a parse failure reads as one) refuses the input
+ * itself, which is refused the same way on every pass, so it settles `failed` at once rather than
+ * spending attempts. This is the ONLY place such a refusal is terminal: once a handler has begun
+ * writing, a throw keeps its old classification, so a parse failing on a stored row mid-provision
+ * leaves the request pending to be finished, not a half-written tenant or scope settled for good.
+ * Anything else `check` throws is rethrown.
+ */
+function checkedBeforeWrites(check) {
+    try {
+        return { ok: true, value: check() };
+    }
+    catch (e) {
+        if (errorCodeOf(e) !== 'validation_failed')
+            throw e;
+        return {
+            ok: false,
+            refusal: { status: 'failed', error: e instanceof Error ? e.message : String(e), failure: INPUT_REFUSED },
+        };
+    }
+}
 /**
  * The `archive-scope` platform-request handler — archives a sibling scope named in the intent.
  * The scope being drained proves the tenant; the target must be under that same tenant and run the
@@ -425,17 +564,25 @@ export function peerInvokeHandler(deps) {
         if (!client) {
             return { status: 'failed', error: `no deployment serves '${payload.vertical}' in this tenant` };
         }
-        const result = await client.verticalInvoke({
-            // `tenantSlug`: the directory's, for the tenant this intent was drained in (#2252), so a
-            // pinned caller meets a bare `peers` entry at the door (`peerCallerAt`).
-            caller: { vertical: ctx.vertical, scope: ctx.scopeId, tenantSlug: tenant.slug },
-            tenantId: ctx.tenantId,
-            scopeId: resolution.instance.scopeId,
-            operation: payload.operation,
-            ...(payload.input === undefined ? {} : { input: payload.input }),
-            // The intent id: a redelivery reaches the recorded response instead of running twice.
-            idempotencyKey: request.id,
-        });
+        let result;
+        try {
+            result = await client.verticalInvoke({
+                // `tenantSlug`: the directory's, for the tenant this intent was drained in (#2252), so a
+                // pinned caller meets a bare `peers` entry at the door (`peerCallerAt`).
+                caller: { vertical: ctx.vertical, scope: ctx.scopeId, tenantSlug: tenant.slug },
+                tenantId: ctx.tenantId,
+                scopeId: resolution.instance.scopeId,
+                operation: payload.operation,
+                ...(payload.input === undefined ? {} : { input: payload.input }),
+                // The intent id: a redelivery reaches the recorded response instead of running twice.
+                idempotencyKey: request.id,
+            });
+        }
+        catch (e) {
+            // #2556: the target's answer is its deployment's text, possibly another builder's code, so
+            // the caller's journal says only which call did not complete. Transient, as it always was.
+            return withheldThrow(e, `the call to '${payload.vertical}' (operation '${payload.operation}') did not complete at its deployment`);
+        }
         return { status: 'done', result: { invoked: payload.operation, result: result ?? null } };
     };
 }
@@ -492,14 +639,39 @@ async function admitManager(deps, ctx, requestedKeys, targetVertical) {
  */
 export function provisionTenantHandler(deps) {
     return async (ctx, request) => {
-        const payload = provisionTenantPayload.parse(request.payload);
+        // Terminal, like the other refusals here: a payload that does not parse never will, and a
+        // throw would leave it pending and retried every pass until the attempt ceiling.
+        const parsed = provisionTenantPayload.safeParse(request.payload);
+        if (!parsed.success)
+            return malformedPayload('provision-tenant', parsed.error);
+        const payload = parsed.data;
         const admitted = await admitManager(deps, ctx, payload.entitlements.map((e) => e.key), payload.instance.vertical);
         if ('refusal' in admitted)
             return admitted.refusal;
+        // The whole input, checked BEFORE anything is written, by the same rules the writes below
+        // apply: an input they would refuse settles here and creates nothing, rather than leaving a
+        // tenant and its entitlements behind a scope that is never provisioned. After the capability
+        // check above, so a manager that may not provision at all is told that (#455), not what is
+        // wrong with a payload it could not have sent anyway.
+        const checked = checkedBeforeWrites(() => {
+            const tenantId = tenantIdSchema.parse(payload.tenant.id);
+            const scopeId = scopeIdSchema.parse(payload.instance.scopeId);
+            createTenantInput.parse({ id: tenantId, slug: payload.tenant.slug, name: payload.tenant.name, provisionedByTenant: ctx.tenantId });
+            const scopeInput = {
+                tenantId,
+                scopeId,
+                slug: payload.instance.slug,
+                name: payload.instance.name,
+                vertical: payload.instance.vertical,
+            };
+            resolveScopeRecord(scopeInput);
+            return { tenantId, scopeId, scopeInput };
+        });
+        if (!checked.ok)
+            return checked.refusal;
+        const { tenantId, scopeId, scopeInput } = checked.value;
         const { host, actor } = deps;
         const admin = host.admin;
-        const tenantId = tenantIdSchema.parse(payload.tenant.id);
-        const scopeId = scopeIdSchema.parse(payload.instance.scopeId);
         try {
             await admin.createTenant(actor, {
                 id: tenantId,
@@ -515,49 +687,54 @@ export function provisionTenantHandler(deps) {
             const message = e instanceof Error ? e.message : String(e);
             // Both adapters fail closed on a slug owned by a DIFFERENT tenant id with this
             // phrasing — a retry can never converge, so settle terminal instead of pending.
+            // The text names only the payload's own slug (#2556), so it is journaled as it is.
             if (message.includes('slugs are unique'))
                 return { status: 'failed', error: message };
             throw e;
         }
-        // Granted BEFORE provisionInstance so the #310 projection carries them on day one.
-        for (const grant of payload.entitlements) {
-            await admin.grantEntitlement(actor, tenantId, grant.key, { plan: grant.plan });
-        }
-        await host.provisionScope(actor, {
-            tenantId,
-            scopeId,
-            slug: payload.instance.slug,
-            name: payload.instance.name,
-            vertical: payload.instance.vertical,
-        });
-        const created = await adoptServingForProvisioning(host, actor, await admin.getScopeRecord(actor, tenantId, scopeId));
-        const vertical = created ? await deps.resolveVerticalForScope(created) : undefined;
-        if (!vertical) {
-            return {
-                status: 'failed',
-                result: { tenantId, scopeId },
-                error: `no deployment is bound for vertical '${payload.instance.vertical}'`,
-            };
-        }
-        const entitlements = await admin.listEntitlements(actor, tenantId);
-        // #301: a NEW tenant's first install is exactly the minting path — the stores are
-        // per (tenant, target vertical), keyed by the PAYLOAD's vertical, not the manager's.
-        const tenantStores = await collectTenantStoreHandles({
-            host,
-            actor,
-            slug: payload.instance.vertical,
-            tenantId,
-            patchBindings: deps.patchScriptBindings,
-        });
-        // #473: same blob-store minting on the manager-driven provision-tenant path.
-        await collectBlobStoreHandles({
-            host,
-            actor,
-            slug: payload.instance.vertical,
-            tenantId,
-            patchBindings: deps.patchScriptBindings,
-        });
+        // #2556: from here every write and call concerns the tenant being provisioned and the
+        // payload vertical's deployment, neither of them the manager's, so a throw's text is withheld
+        // from the manager's journal: it learns the step, and staff read the text under the reference.
+        let step = 'granting its entitlements';
         try {
+            // Granted BEFORE provisionInstance so the #310 projection carries them on day one.
+            for (const grant of payload.entitlements) {
+                await admin.grantEntitlement(actor, tenantId, grant.key, { plan: grant.plan });
+            }
+            // Past the first write, so a throw keeps its old classification (pending, retried): the
+            // tenant exists, and a later pass is what finishes it.
+            step = 'provisioning its scope';
+            await host.provisionScope(actor, scopeInput);
+            const created = await adoptServingForProvisioning(host, actor, await admin.getScopeRecord(actor, tenantId, scopeId));
+            const vertical = created ? await deps.resolveVerticalForScope(created) : undefined;
+            if (!vertical) {
+                return {
+                    status: 'failed',
+                    result: { tenantId, scopeId },
+                    error: `no deployment is bound for vertical '${payload.instance.vertical}'`,
+                };
+            }
+            step = 'collecting its stores';
+            const entitlements = await admin.listEntitlements(actor, tenantId);
+            // #301: a NEW tenant's first install is exactly the minting path — the stores are
+            // per (tenant, target vertical), keyed by the PAYLOAD's vertical, not the manager's.
+            const tenantStores = await collectTenantStoreHandles({
+                host,
+                actor,
+                slug: payload.instance.vertical,
+                tenantId,
+                patchBindings: deps.patchScriptBindings,
+            });
+            // #473: same blob-store minting on the manager-driven provision-tenant path.
+            await collectBlobStoreHandles({
+                host,
+                actor,
+                slug: payload.instance.vertical,
+                tenantId,
+                patchBindings: deps.patchScriptBindings,
+            });
+            // A refusal from the vertical is transient (`pending`), as is anything else thrown here.
+            step = `provisioning its '${payload.instance.vertical}' instance`;
             // #1674: a re-drain reuses the proposed ids, so this can re-seat a wiped, switched-off scope.
             await reconcileThenReassert(admin, actor, { tenantId, scopeId }, (carry) => vertical.provisionInstance({
                 tenantId,
@@ -570,6 +747,7 @@ export function provisionTenantHandler(deps) {
                 ...carry,
             }));
             if (payload.config && Object.keys(payload.config).length) {
+                step = `configuring its '${payload.instance.vertical}' instance`;
                 try {
                     await vertical.configureInstance({
                         tenantId,
@@ -584,14 +762,12 @@ export function provisionTenantHandler(deps) {
                         throw e;
                 }
             }
+            step = 'activating its scope';
+            await admin.activateScope(actor, tenantId, scopeId);
         }
         catch (e) {
-            if (e instanceof ControlPlaneError) {
-                return { status: 'pending', result: { tenantId, scopeId }, error: e.message };
-            }
-            throw e;
+            return withheldThrow(e, `provisioning tenant '${payload.tenant.slug}' did not complete while ${step}`, { tenantId, scopeId });
         }
-        await admin.activateScope(actor, tenantId, scopeId);
         return { status: 'done', result: { tenantId, scopeId } };
     };
 }
@@ -664,34 +840,38 @@ export function setEntitlementsHandler(deps) {
         if (!scope) {
             return { status: 'failed', error: `unknown scope for tenant: (${tenantId}, ${scopeId})` };
         }
-        const target = new Map(payload.entitlements.map((e) => [e.key, e.plan]));
-        for (const key of admitted.declared) {
-            if (target.has(key)) {
-                await admin.grantEntitlement(actor, tenantId, key, { plan: target.get(key) });
+        // #2556: the writes and the reconcile concern the managed tenant and its scope's deployment,
+        // neither of them the manager's, so a throw's text is withheld from the manager's journal.
+        let step = 'reconciling its entitlements';
+        try {
+            const target = new Map(payload.entitlements.map((e) => [e.key, e.plan]));
+            for (const key of admitted.declared) {
+                if (target.has(key)) {
+                    await admin.grantEntitlement(actor, tenantId, key, { plan: target.get(key) });
+                }
+                else {
+                    await admin.revokeEntitlement(actor, tenantId, key);
+                }
             }
-            else {
-                await admin.revokeEntitlement(actor, tenantId, key);
+            const vertical = await deps.resolveVerticalForScope(scope);
+            if (!vertical) {
+                // Named by the payload's own ids, not by what the managed tenant's scope runs (#2556).
+                return { status: 'failed', error: `no deployment is bound for scope ${scopeId} of tenant ${tenantId}` };
             }
-        }
-        const vertical = await deps.resolveVerticalForScope(scope);
-        if (!vertical) {
-            return {
-                status: 'failed',
-                error: `no deployment is bound for vertical '${scope.vertical ?? 'none'}'`,
-            };
-        }
-        const entitlements = await admin.listEntitlements(actor, tenantId);
-        const identityLinks = (await admin.listIdentityLinks(actor, tenantId)).map(({ tenantId: _tenantId, ...link }) => link);
-        // #592: every reconcile re-delivers the tenant's live connection grants, whoever
-        // triggered it — this drain-side reconcile must not silently wipe or omit them.
-        const connectionGrants = connectionGrantsForScope(await admin.listConnectionGrants(actor, tenantId), scope.vertical, scopeId);
-        // #687: sealing keys ride every reconcile too, for the same reason grants do — a
-        // drain-side reconcile must not leave a scope unable to seal to a connector it could
-        // seal to a moment ago.
-        const connectionKeys = scope.vertical
-            ? await admin.connectionSealingKeys(tenantId, scope.vertical)
-            : [];
-        try {
+            step = 'gathering what its instance is reconciled with';
+            const entitlements = await admin.listEntitlements(actor, tenantId);
+            const identityLinks = (await admin.listIdentityLinks(actor, tenantId)).map(({ tenantId: _tenantId, ...link }) => link);
+            // #592: every reconcile re-delivers the tenant's live connection grants, whoever
+            // triggered it — this drain-side reconcile must not silently wipe or omit them.
+            const connectionGrants = connectionGrantsForScope(await admin.listConnectionGrants(actor, tenantId), scope.vertical, scopeId);
+            // #687: sealing keys ride every reconcile too, for the same reason grants do — a
+            // drain-side reconcile must not leave a scope unable to seal to a connector it could
+            // seal to a moment ago.
+            const connectionKeys = scope.vertical
+                ? await admin.connectionSealingKeys(tenantId, scope.vertical)
+                : [];
+            // A refusal from the vertical is transient (`pending`), as is anything else thrown here.
+            step = 'reconciling its instance';
             await reconcileThenReassert(admin, actor, { tenantId, scopeId }, (carry) => vertical.reconcileInstance({
                 tenantId,
                 scopeId,
@@ -703,9 +883,7 @@ export function setEntitlementsHandler(deps) {
             }));
         }
         catch (e) {
-            if (e instanceof ControlPlaneError)
-                return { status: 'pending', error: e.message };
-            throw e;
+            return withheldThrow(e, `setting the entitlements of tenant ${tenantId} did not complete while ${step}`);
         }
         return { status: 'done', result: { tenantId, plan: payload.plan } };
     };