@intx/hub-sessions 0.3.0 → 0.4.0

package/dist/workflow-allocation-service.js

@@ -1,120 +1,201 @@
-import { createSidecarAllocationStore, createWorkflowRunLaunchSpecStore, getAncestorChain, resolveSourcesByOfferingIds, } from "@intx/db";
-import { grant, tenant as tenantTable, workflowRun } from "@intx/db/schema";
-import { eq, inArray } from "drizzle-orm";
+import { type } from "arktype";
+import { sha256 } from "@intx/crypto";
+import { createSidecarAllocationStore, createWorkflowProbeStore, createWorkflowRunLaunchSpecStore, resolveTenantSidecarCapabilityPolicies, resolveSourcesByOfferingIds, } from "@intx/db";
+import { grant, workflowRun } from "@intx/db/schema";
+import { eq } from "drizzle-orm";
 import { generateId } from "@intx/hub-common";
-import { hexEncode, TenantConfig, } from "@intx/types";
-import { deriveRunAddress, deriveRunAgentId } from "@intx/workflow-deploy";
-import { resolveEffectiveSidecarPlacement, } from "./sidecar-allocation/index.js";
+import { hexEncode, SidecarCapabilityRule, } from "@intx/types";
+import { approvalItemsFromSet, approvalSetFromItems, buildInertProjectionStepSources, deriveRunAddress, deriveRunAgentId, } from "@intx/workflow-deploy";
+import { DestroySidecarResult, EnsureSidecarResult, } from "./sidecar-allocation/index.js";
 import { SessionLaunchError, } from "./session-service.js";
-export class ExclusiveWorkflowPlacementError extends Error {
+import { buildReferencedWorkflowSourcePins } from "./workflow-source-pins.js";
+import { DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, runSidecarOperation, } from "./sidecar-allocation/operation.js";
+export class WorkflowProvisioningError extends Error {
     code;
     constructor(code, message) {
         super(message);
-        this.name = "ExclusiveWorkflowPlacementError";
+        this.name = "WorkflowProvisioningError";
         this.code = code;
     }
 }
 function randomAllocationId() {
     return `sal_${hexEncode(crypto.getRandomValues(new Uint8Array(16)))}`;
 }
-/** Resolve the tenant-inherited sidecar placement into one launch decision. */
-export async function resolveWorkflowSidecarPlacement(db, tenantId) {
-    const tenantIds = await getAncestorChain(db, tenantId);
-    const rows = await db.query.tenant.findMany({
-        where: inArray(tenantTable.id, tenantIds),
-        columns: { id: true, config: true },
-    });
-    const configsByTenantId = new Map(rows.map((row) => [
-        row.id,
-        row.config === null
-            ? TenantConfig.assert({})
-            : TenantConfig.assert(row.config),
-    ]));
-    const tenantConfigs = tenantIds.map((id) => {
-        const config = configsByTenantId.get(id);
-        if (config === undefined) {
-            throw new Error(`Tenant ${id} disappeared while resolving sidecar placement`);
-        }
-        return config;
+function randomSidecarId() {
+    return `sc_${hexEncode(crypto.getRandomValues(new Uint8Array(16)))}`;
+}
+function randomToken() {
+    return `intx_sc_${hexEncode(crypto.getRandomValues(new Uint8Array(32)))}`;
+}
+function selectProvisioner(selection) {
+    if (selection.ok)
+        return selection.provisioner;
+    throw new WorkflowProvisioningError("provisioner_no_match", formatProvisionerMismatches(selection.mismatches));
+}
+function parseEnsureResult(value) {
+    const result = EnsureSidecarResult(value);
+    if (result instanceof type.errors) {
+        throw new Error(`Sidecar provisioner returned an invalid ensure result: ${result.summary}`);
+    }
+    return result;
+}
+function parseDestroyResult(value) {
+    const result = DestroySidecarResult(value);
+    if (result instanceof type.errors) {
+        throw new Error(`Sidecar provisioner returned an invalid destroy result: ${result.summary}`);
+    }
+    return result;
+}
+function createProvisionedHarnessConfig(args) {
+    const deploymentAddress = deriveRunAddress({
+        runId: args.anchorRunId,
+        domain: args.deploymentDomain,
     });
-    return resolveEffectiveSidecarPlacement({ tenantConfigs });
+    return {
+        sessionId: args.sessionId,
+        agentId: deriveRunAgentId({ runId: args.anchorRunId }),
+        tenantId: args.tenantId,
+        principalId: args.sourceAuthorityPrincipalId,
+        agentAddress: deploymentAddress,
+        systemPrompt: "",
+        tools: [],
+        grants: [],
+        sources: args.sources,
+        defaultSource: args.defaultSource,
+    };
 }
-export function createWorkflowAllocationService({ db, plugins, preparedDeployer, credentialCipher, allocationRouter, createAllocationId = randomAllocationId, now = () => new Date(), }) {
+export function createWorkflowAllocationService({ db, deploymentPlugins, probePlugins, preparedDeployer, credentialCipher, probeCapabilityRules = [], allocationRouter, hubWebSocketUrl, createAllocationId = randomAllocationId, createSidecarId = randomSidecarId, createToken = randomToken, connectTimeoutMs = 120_000, operationTimeoutMs = DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, now = () => new Date(), }) {
     const allocationStore = createSidecarAllocationStore(db);
+    const probeStore = createWorkflowProbeStore(db);
     const launchSpecStore = createWorkflowRunLaunchSpecStore(db);
-    async function prepareExclusiveDeployment(args) {
-        // Fail closed before any probe: exclusive placement has no meaning without a
-        // provisioner to stand up its dedicated sidecar, and the shared-capacity
-        // probe below is wasted work if no provisioner exists. (In-tree this is
-        // always null -- exclusive is dormant -- so a live prepare never runs here.)
-        const provisioner = plugins.getDefaultProvisioner();
-        if (provisioner === null) {
-            throw new ExclusiveWorkflowPlacementError("exclusive_provisioner_unavailable", "No default sidecar provisioner is configured for exclusive workflows");
+    const validatedProbeCapabilityRules = SidecarCapabilityRule.array()(probeCapabilityRules);
+    if (validatedProbeCapabilityRules instanceof type.errors) {
+        throw new Error(`Invalid workflow probe capability rules: ${validatedProbeCapabilityRules.summary}`);
+    }
+    const configuredProbeCapabilityRules = validatedProbeCapabilityRules.map((rule) => ({ ...rule }));
+    if (connectTimeoutMs <= 0) {
+        throw new Error("connectTimeoutMs must be positive");
+    }
+    if (!Number.isSafeInteger(operationTimeoutMs) || operationTimeoutMs <= 0) {
+        throw new Error("operationTimeoutMs must be a positive integer");
+    }
+    function matchingProvisioner(probe) {
+        const provisioner = probePlugins.getProvisioner(probe.provisionerId);
+        return provisioner !== null &&
+            provisioner.apiVersion === probe.provisionerApiVersion &&
+            provisioner.bindingFingerprint === probe.provisionerBindingFingerprint
+            ? provisioner
+            : null;
+    }
+    async function beginProbeRelease(probe, failure) {
+        if (probe.status === "releasing")
+            return probe;
+        return probeStore.transition(probe.id, ["pending", "provisioning", "probing"], "releasing", {
+            ...(failure !== undefined
+                ? {
+                    failureCode: failure.code,
+                    failureMessage: failure.message,
+                }
+                : {}),
+            now: now(),
+        });
+    }
+    const probeCleanupTasks = new Map();
+    function finishProbeRelease(probe, finalStatus) {
+        const existing = probeCleanupTasks.get(probe.id);
+        if (existing !== undefined)
+            return existing;
+        const task = cleanUpProbe(probe, finalStatus);
+        probeCleanupTasks.set(probe.id, task);
+        const settled = () => {
+            if (probeCleanupTasks.get(probe.id) === task)
+                probeCleanupTasks.delete(probe.id);
+        };
+        void task.then(settled, settled);
+        return task;
+    }
+    async function cleanUpProbe(probe, finalStatus) {
+        const releasing = await probeStore.get(probe.id);
+        if (releasing?.status === "succeeded" || releasing?.status === "failed")
+            return;
+        if (releasing?.status !== "releasing") {
+            throw new Error(`Workflow probe ${probe.id} is not releasing`);
         }
-        if (args.sourceOfferingIds.length === 0) {
-            throw new ExclusiveWorkflowPlacementError("invalid_source_offerings", "Exclusive workflow deployment requires at least one catalog offering");
+        const sidecarId = releasing.sidecarId;
+        if (sidecarId !== null) {
+            const provisioner = matchingProvisioner(releasing);
+            if (provisioner === null) {
+                throw new Error(`Cannot clean up workflow probe ${releasing.id}: provisioner binding ${releasing.provisionerId} is unavailable`);
+            }
+            const destroyed = parseDestroyResult(await runSidecarOperation("Probe destroy", operationTimeoutMs, (signal) => provisioner.destroy({
+                signal,
+                allocationId: releasing.id,
+                generation: releasing.generation,
+                sidecarId,
+                ...(releasing.externalRef !== null
+                    ? { externalRef: releasing.externalRef }
+                    : {}),
+            })));
+            if (destroyed.kind === "rejected") {
+                throw new Error(`Provisioner ${provisioner.id} rejected probe cleanup: ${destroyed.message}`);
+            }
+            allocationRouter.disconnectAllocation({
+                allocationId: releasing.id,
+                generation: releasing.generation,
+            });
         }
-        if (!args.sourceOfferingIds.includes(args.defaultSourceOfferingId)) {
-            throw new ExclusiveWorkflowPlacementError("invalid_source_offerings", "The default source offering must be present in the source offering chain");
+        const completed = await probeStore.transition(releasing.id, ["releasing"], finalStatus, { now: now() });
+        if (completed === null) {
+            throw new Error(`Workflow probe ${releasing.id} changed before cleanup completed`);
         }
-        const sourceCheck = await resolveSourcesByOfferingIds(db, args.tenantId, args.sourceOfferingIds, credentialCipher);
-        if (!sourceCheck.ok) {
-            throw new ExclusiveWorkflowPlacementError("source_offering_unavailable", `Catalog offering ${sourceCheck.offeringId} cannot be used by the deployment authority`);
-        }
-        // Probe + gate + freeze the code-sourced definition ONCE, on shared
-        // capacity, at request time. The freeze is sidecar-agnostic (a wire hash
-        // over the inert projection plus the resolved closure), so the frozen bundle
-        // deploys later to the dedicated allocation with no re-probe. A non-approval
-        // surfaces as a `WorkflowDefinitionInvalidError` from the deployer, which the
-        // route maps to a 409.
-        const approved = await preparedDeployer.installAndApproveWorkflowSource({
-            source: args.source,
-            entry: args.entry,
-            ...(args.pin !== undefined ? { pin: args.pin } : {}),
-            definitionAssetId: args.definitionAssetId,
+        allocationRouter.retireAllocation({
+            allocationId: releasing.id,
+            generation: releasing.generation,
+        });
+    }
+    async function releaseProbe(probe, finalStatus, failure) {
+        const releasing = await beginProbeRelease(probe, failure);
+        if (releasing === null)
+            return;
+        await finishProbeRelease(releasing, finalStatus);
+    }
+    async function createDeployment(args) {
+        const { request, approved, provisioner, probe, adoptProbe } = args;
+        const probeSidecarId = probe.sidecarId;
+        if (adoptProbe && probeSidecarId === null) {
+            throw new Error(`Workflow probe ${probe.id} has no sidecar to adopt`);
+        }
+        const allocationId = adoptProbe ? probe.id : createAllocationId();
+        const createdAt = now();
+        const deploymentAddress = deriveRunAddress({
+            runId: request.anchorRunId,
+            domain: request.deploymentDomain,
         });
-        if (!approved.approval.ok) {
-            // `installAndApproveWorkflowSource` already fails closed on a non-approval;
-            // restate the narrowing so the frozen bundle below reads the ok arm.
-            throw new Error("prepareExclusiveDeployment: install did not yield an approved definition");
-        }
-        // Capture the narrowed values before the transaction: TS drops the
-        // `approval.ok` narrowing inside the async callback below.
-        const definitionId = approved.approval.definitionId;
         const frozenApprovalBundle = {
-            source: args.source,
-            entry: args.entry,
+            source: request.source,
+            entry: request.entry,
             projection: approved.projection,
             closure: approved.closure,
             approvedWireHash: approved.approval.approvedWireHash,
-            approvedGrants: [...approved.approval.approvedGrants],
+            approvedGrants: [
+                ...approvalItemsFromSet(approved.approval.approvedSurface),
+            ],
         };
-        const allocationId = createAllocationId();
-        const createdAt = now();
-        const deploymentAddress = deriveRunAddress({
-            runId: args.anchorRunId,
-            domain: args.deploymentDomain,
-        });
         await db.transaction(async (tx) => {
-            // The freeze already ensured (create-if-absent) the definition row keyed by
-            // the approved wire hash and returned its id; anchor to THAT row rather
-            // than re-ensuring, so the anchor's definition is exactly the one approved.
             await tx.insert(workflowRun).values({
-                id: args.anchorRunId,
-                tenantId: args.tenantId,
-                anchorRunId: args.anchorRunId,
-                definitionId,
+                id: request.anchorRunId,
+                tenantId: request.tenantId,
+                anchorRunId: request.anchorRunId,
+                definitionId: approved.approval.definitionId,
                 address: deploymentAddress,
-                // Born "deployed": the anchor is live but pre-trigger. The first trigger
-                // flips it to "running" (see `anchorWithPrincipal`).
                 status: "deployed",
                 createdAt,
             });
             await tx.insert(grant).values({
                 id: generateId("grant"),
-                tenantId: args.tenantId,
-                principalId: args.sourceAuthorityPrincipalId,
-                resource: `workflow-run:${args.anchorRunId}`,
+                tenantId: request.tenantId,
+                principalId: request.sourceAuthorityPrincipalId,
+                resource: `workflow-run:${request.anchorRunId}`,
                 action: "read",
                 effect: "allow",
                 origin: "creator",
@@ -122,38 +203,227 @@ export function createWorkflowAllocationService({ db, plugins, preparedDeployer,
                 updatedAt: createdAt,
             });
             await launchSpecStore.create({
-                anchorRunId: args.anchorRunId,
-                sessionId: args.sessionId,
-                deploymentDomain: args.deploymentDomain,
-                sourceAuthorityPrincipalId: args.sourceAuthorityPrincipalId,
+                anchorRunId: request.anchorRunId,
+                sessionId: request.sessionId,
+                deploymentDomain: request.deploymentDomain,
+                sourceAuthorityPrincipalId: request.sourceAuthorityPrincipalId,
                 frozenApprovalBundle,
-                sourceOfferingIds: [...args.sourceOfferingIds],
-                defaultSourceOfferingId: args.defaultSourceOfferingId,
-                deployContent: args.deployContent,
-                ...(args.toolPackagePins !== undefined
-                    ? { toolPackagePins: [...args.toolPackagePins] }
+                sourceOfferingIds: [...request.sourceOfferingIds],
+                defaultSourceOfferingId: request.defaultSourceOfferingId,
+                deployContent: request.deployContent,
+                ...(request.toolPackagePins !== undefined
+                    ? { toolPackagePins: [...request.toolPackagePins] }
                     : {}),
                 createdAt,
             }, tx);
-            await allocationStore.createPending({
-                id: allocationId,
-                anchorRunId: args.anchorRunId,
-                tenantId: args.tenantId,
-                provisionerId: provisioner.id,
-                provisionerApiVersion: provisioner.apiVersion,
-                provisionerBindingFingerprint: provisioner.bindingFingerprint,
-                placement: args.placement,
-                now: createdAt,
-            }, tx);
+            if (adoptProbe) {
+                if (probeSidecarId === null) {
+                    throw new Error(`Workflow probe ${probe.id} lost its sidecar`);
+                }
+                await allocationStore.createAdopted({
+                    id: allocationId,
+                    anchorRunId: request.anchorRunId,
+                    tenantId: request.tenantId,
+                    provisionerId: provisioner.id,
+                    provisionerApiVersion: provisioner.apiVersion,
+                    provisionerBindingFingerprint: provisioner.bindingFingerprint,
+                    sidecarId: probeSidecarId,
+                    generation: probe.generation,
+                    ...(probe.externalRef !== null
+                        ? { externalRef: probe.externalRef }
+                        : {}),
+                    connectDeadline: new Date(createdAt.getTime() + connectTimeoutMs),
+                    now: createdAt,
+                }, tx);
+                const completed = await probeStore.transition(probe.id, ["probing"], "succeeded", { now: createdAt, tx });
+                if (completed === null) {
+                    throw new Error(`Workflow probe ${probe.id} changed before adoption`);
+                }
+            }
+            else {
+                await allocationStore.createPending({
+                    id: allocationId,
+                    anchorRunId: request.anchorRunId,
+                    tenantId: request.tenantId,
+                    provisionerId: provisioner.id,
+                    provisionerApiVersion: provisioner.apiVersion,
+                    provisionerBindingFingerprint: provisioner.bindingFingerprint,
+                    now: createdAt,
+                }, tx);
+            }
         });
+        if (adoptProbe) {
+            allocationRouter.disconnectAllocation({
+                allocationId: probe.id,
+                generation: probe.generation,
+            });
+        }
         return {
-            anchorRunId: args.anchorRunId,
+            anchorRunId: request.anchorRunId,
             deploymentAddress,
             allocationId,
             status: "pending",
         };
     }
-    async function deployReadyAllocation(allocation) {
+    async function prepareProvisionedDeployment(args) {
+        if (args.sourceOfferingIds.length === 0) {
+            throw new WorkflowProvisioningError("invalid_source_offerings", "Provisioned workflow deployment requires at least one catalog offering");
+        }
+        if (!args.sourceOfferingIds.includes(args.defaultSourceOfferingId)) {
+            throw new WorkflowProvisioningError("invalid_source_offerings", "The default source offering must be present in the source offering chain");
+        }
+        const [sourceCheck, tenantPolicies] = await Promise.all([
+            resolveSourcesByOfferingIds(db, args.tenantId, args.sourceOfferingIds, credentialCipher),
+            resolveTenantSidecarCapabilityPolicies(db, args.tenantId),
+        ]);
+        if (!sourceCheck.ok) {
+            throw new WorkflowProvisioningError("source_offering_unavailable", `Catalog offering ${sourceCheck.offeringId} cannot be used by the deployment authority`);
+        }
+        const defaultSource = sourceCheck.sources.find((source) => source.id === args.defaultSourceOfferingId);
+        if (defaultSource === undefined) {
+            throw new WorkflowProvisioningError("invalid_source_offerings", `Default offering ${args.defaultSourceOfferingId} was not resolved for deployment ${args.anchorRunId}`);
+        }
+        const probeProvisioner = selectProvisioner(await probePlugins.selectProvisioner({
+            tenantPolicies,
+            probeRules: configuredProbeCapabilityRules,
+            workflowRules: [],
+        }));
+        const probeId = createAllocationId();
+        let probe = await probeStore.create({
+            id: probeId,
+            tenantId: args.tenantId,
+            definitionAssetId: args.definitionAssetId,
+            source: args.source,
+            entry: args.entry,
+            ...(args.pin !== undefined ? { pin: args.pin } : {}),
+            provisionerId: probeProvisioner.id,
+            provisionerApiVersion: probeProvisioner.apiVersion,
+            provisionerBindingFingerprint: probeProvisioner.bindingFingerprint,
+            now: now(),
+        });
+        try {
+            const token = createToken();
+            const sidecarId = createSidecarId();
+            const bound = await probeStore.bindSidecar({
+                probeId,
+                sidecarId,
+                tokenHashSha256: await sha256(token),
+                now: now(),
+            });
+            if (bound === null) {
+                throw new Error(`Workflow probe ${probeId} changed before provisioning`);
+            }
+            probe = bound;
+            const allocationTarget = {
+                allocationId: probe.id,
+                generation: probe.generation,
+            };
+            allocationRouter.fenceAllocation(probe.id, probe.generation);
+            const ensured = parseEnsureResult(await runSidecarOperation("Probe ensure", operationTimeoutMs, (signal) => probeProvisioner.ensure({
+                signal,
+                allocationId: probe.id,
+                generation: probe.generation,
+                tenantId: probe.tenantId,
+                // The provisioner contract treats this as an opaque owner id. A
+                // probe has no workflow run, so its own id is the honest owner.
+                anchorRunId: probe.id,
+                sidecarId,
+                token,
+                hubWebSocketUrl,
+            })));
+            if (ensured.kind === "rejected") {
+                throw new WorkflowProvisioningError(ensured.code, ensured.message);
+            }
+            const probing = await probeStore.markProbing({
+                probeId,
+                ...(ensured.externalRef !== undefined
+                    ? { externalRef: ensured.externalRef }
+                    : {}),
+                now: now(),
+            });
+            if (probing === null) {
+                throw new Error(`Workflow probe ${probeId} changed after provisioning`);
+            }
+            probe = probing;
+            await allocationRouter.waitForAllocatedSidecar(allocationTarget, connectTimeoutMs);
+            let resultRecorded = false;
+            const approved = await preparedDeployer.installAndApproveWorkflowSource({
+                source: args.source,
+                entry: args.entry,
+                ...(args.pin !== undefined ? { pin: args.pin } : {}),
+                definitionAssetId: args.definitionAssetId,
+                allocationTarget,
+                onProbeResult: async (result) => {
+                    if ((await probeStore.recordResult(probeId, result, now())) === null) {
+                        throw new Error(`Workflow probe ${probeId} changed before result persistence`);
+                    }
+                    resultRecorded = true;
+                },
+            });
+            if (!resultRecorded) {
+                throw new Error(`Workflow probe ${probeId} completed without persisting its result`);
+            }
+            if (!approved.approval.ok) {
+                throw new Error("prepareProvisionedDeployment: install did not yield an approved definition");
+            }
+            const narrowed = {
+                ...approved,
+                approval: approved.approval,
+            };
+            const config = createProvisionedHarnessConfig({
+                tenantId: args.tenantId,
+                anchorRunId: args.anchorRunId,
+                deploymentDomain: args.deploymentDomain,
+                sessionId: args.sessionId,
+                sourceAuthorityPrincipalId: args.sourceAuthorityPrincipalId,
+                sources: sourceCheck.sources,
+                defaultSource: defaultSource.id,
+            });
+            buildInertProjectionStepSources({
+                projection: approved.projection,
+                config,
+                operatorApprovals: approved.approval.approvedSurface,
+            });
+            await buildReferencedWorkflowSourcePins({
+                projection: approved.projection,
+                config,
+                operatorApprovals: approved.approval.approvedSurface,
+            });
+            const deploymentProvisioner = selectProvisioner(await deploymentPlugins.selectProvisioner({
+                tenantPolicies,
+                workflowRules: approved.projection.sidecarPlacement?.capabilities ?? [],
+            }));
+            const adoptProbe = deploymentProvisioner.id === probeProvisioner.id &&
+                deploymentProvisioner.apiVersion === probeProvisioner.apiVersion &&
+                deploymentProvisioner.bindingFingerprint ===
+                    probeProvisioner.bindingFingerprint;
+            if (!adoptProbe) {
+                await releaseProbe(probe, "succeeded");
+            }
+            return await createDeployment({
+                request: args,
+                approved: narrowed,
+                provisioner: deploymentProvisioner,
+                probe,
+                adoptProbe,
+            });
+        }
+        catch (error) {
+            try {
+                await releaseProbe(probe, "failed", {
+                    code: "probe_failed",
+                    message: error instanceof Error ? error.message : String(error),
+                });
+            }
+            catch {
+                // Preserve the primary probe/provisioning error. The durable
+                // `releasing` row remains discoverable for periodic cleanup.
+            }
+            throw error;
+        }
+    }
+    async function deployReadyAllocation(allocation, reconciliation) {
+        reconciliation.signal.throwIfAborted();
         if (allocation.status !== "allocated" ||
             allocation.ensureAcceptedGeneration !== allocation.generation) {
             throw new Error(`Allocation ${allocation.id} generation ${String(allocation.generation)} is not accepted for deployment`);
@@ -182,15 +452,16 @@ export function createWorkflowAllocationService({ db, plugins, preparedDeployer,
             throw new Error(`Allocation ${allocation.id} has no workflow launch specification`);
         }
         // The frozen bundle deploys verbatim -- no re-probe. Rehydrate the approval
-        // hand-off from it: the approved grant set becomes a `Set`, and the frozen
-        // definition id is the anchor's own (set at prepare time from this freeze).
+        // hand-off from it: the persisted flat item list is partitioned back into
+        // the gate's `ApprovalSet`, and the frozen definition id is the anchor's own
+        // (set at prepare time from this freeze).
         const bundle = spec.frozenApprovalBundle;
         const approved = {
             approval: {
                 ok: true,
                 definitionId: anchor.definitionId,
                 approvedWireHash: bundle.approvedWireHash,
-                approvedGrants: new Set(bundle.approvedGrants),
+                approvedSurface: approvalSetFromItems(bundle.approvedGrants),
                 projection: bundle.projection,
             },
             projection: bundle.projection,
@@ -207,33 +478,77 @@ export function createWorkflowAllocationService({ db, plugins, preparedDeployer,
         if (defaultSource === undefined) {
             throw new Error(`Default offering ${spec.defaultSourceOfferingId} was not resolved for allocation ${allocation.id}`);
         }
-        const deploymentAddress = deriveRunAddress({
-            runId: allocation.anchorRunId,
-            domain: spec.deploymentDomain,
-        });
-        const config = {
-            sessionId: spec.sessionId,
-            agentId: deriveRunAgentId({ runId: allocation.anchorRunId }),
+        const config = createProvisionedHarnessConfig({
             tenantId: allocation.tenantId,
-            principalId: spec.sourceAuthorityPrincipalId,
-            agentAddress: deploymentAddress,
-            systemPrompt: "",
-            tools: [],
-            grants: [],
+            anchorRunId: allocation.anchorRunId,
+            deploymentDomain: spec.deploymentDomain,
+            sessionId: spec.sessionId,
+            sourceAuthorityPrincipalId: spec.sourceAuthorityPrincipalId,
             sources: resolved.sources,
             defaultSource: defaultSource.id,
-        };
+        });
+        reconciliation.signal.throwIfAborted();
         return preparedDeployer.deployPreparedCodeSourcedWorkflow({
+            reconciliation,
             tenantId: allocation.tenantId,
             anchorRunId: allocation.anchorRunId,
             deploymentDomain: spec.deploymentDomain,
-            agentAddress: deploymentAddress,
+            agentAddress: config.agentAddress,
             source: bundle.source,
             approved,
             config,
             allocationTarget,
-            ...(credentialCipher !== undefined ? { credentialCipher } : {}),
+            credentialCipher,
         });
     }
-    return { prepareExclusiveDeployment, deployReadyAllocation };
+    function cleanupFinalStatus(probe) {
+        return probe.failureCode === null ? "succeeded" : "failed";
+    }
+    async function reconcileReleasingProbes() {
+        const failures = [];
+        for (const probe of await probeStore.listReleasing()) {
+            try {
+                await releaseProbe(probe, cleanupFinalStatus(probe));
+            }
+            catch (error) {
+                failures.push(error);
+            }
+        }
+        if (failures.length > 0) {
+            throw new AggregateError(failures, "Failed to clean up releasing workflow probes");
+        }
+    }
+    async function initialize() {
+        const failures = [];
+        for (const probe of await probeStore.listActive()) {
+            try {
+                await beginProbeRelease(probe, probe.status === "releasing"
+                    ? undefined
+                    : {
+                        code: "probe_interrupted",
+                        message: "Hub restarted before the workflow probe completed",
+                    });
+            }
+            catch (error) {
+                failures.push(error);
+            }
+        }
+        if (failures.length > 0) {
+            throw new AggregateError(failures, "Failed to mark interrupted workflow probes for cleanup");
+        }
+    }
+    return {
+        initialize,
+        reconcileReleasingProbes,
+        prepareProvisionedDeployment,
+        deployReadyAllocation,
+    };
+}
+function formatProvisionerMismatches(mismatches) {
+    const details = Object.entries(mismatches)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .flatMap(([provisionerId, failures]) => failures.map(({ capability, expected, actual }) => `${provisionerId}: ${capability} must be ${expected}, was ${actual}`));
+    return details.length === 0
+        ? "No sidecar provisioners are registered"
+        : `No sidecar provisioner satisfies the deployment (${details.join("; ")})`;
 }