@intx/hub-sessions 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/agent-repo.d.ts +14 -2
  2. package/dist/agent-repo.js +17 -4
  3. package/dist/agent-state-kind.js +14 -63
  4. package/dist/asset-service.js +14 -10
  5. package/dist/credential-push.d.ts +48 -4
  6. package/dist/credential-push.js +138 -6
  7. package/dist/event-collector-registry.d.ts +2 -1
  8. package/dist/event-collector-registry.js +38 -9
  9. package/dist/event-collector.d.ts +11 -1
  10. package/dist/event-collector.js +36 -3
  11. package/dist/hub-session-lookups.d.ts +1 -1
  12. package/dist/hub-session-lookups.js +68 -72
  13. package/dist/hub-session-orchestrator.d.ts +2 -3
  14. package/dist/hub-session-orchestrator.js +13 -12
  15. package/dist/index.d.ts +7 -6
  16. package/dist/index.js +7 -6
  17. package/dist/reconciliation-scheduler.d.ts +14 -0
  18. package/dist/reconciliation-scheduler.js +55 -0
  19. package/dist/repo-store/index.d.ts +1 -0
  20. package/dist/repo-store/index.js +1 -0
  21. package/dist/repo-store/user-principal-gate.d.ts +26 -0
  22. package/dist/repo-store/user-principal-gate.js +78 -0
  23. package/dist/session-service.d.ts +66 -121
  24. package/dist/session-service.js +444 -411
  25. package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
  26. package/dist/sidecar-allocation/capability-policy.js +124 -0
  27. package/dist/sidecar-allocation/contracts.d.ts +29 -6
  28. package/dist/sidecar-allocation/contracts.js +7 -2
  29. package/dist/sidecar-allocation/index.d.ts +4 -3
  30. package/dist/sidecar-allocation/index.js +3 -2
  31. package/dist/sidecar-allocation/operation.d.ts +10 -0
  32. package/dist/sidecar-allocation/operation.js +54 -0
  33. package/dist/sidecar-allocation/plugin-registry.d.ts +16 -3
  34. package/dist/sidecar-allocation/plugin-registry.js +36 -12
  35. package/dist/sidecar-allocation/reconciler.d.ts +16 -4
  36. package/dist/sidecar-allocation/reconciler.js +486 -92
  37. package/dist/skill-kind.js +8 -62
  38. package/dist/substrate.d.ts +1 -1
  39. package/dist/substrate.js +1 -1
  40. package/dist/workflow-allocation-service.d.ts +21 -15
  41. package/dist/workflow-allocation-service.js +440 -125
  42. package/dist/workflow-dispatch-service.d.ts +4 -2
  43. package/dist/workflow-dispatch-service.js +89 -26
  44. package/dist/workflow-kind.d.ts +12 -0
  45. package/dist/workflow-kind.js +17 -60
  46. package/dist/workflow-probe-gate.d.ts +99 -27
  47. package/dist/workflow-probe-gate.js +196 -21
  48. package/dist/workflow-run-kind.d.ts +112 -19
  49. package/dist/workflow-run-kind.js +626 -210
  50. package/dist/workflow-run-restore.d.ts +1 -0
  51. package/dist/workflow-run-restore.js +5 -1
  52. package/dist/workflow-source-pins.d.ts +8 -0
  53. package/dist/workflow-source-pins.js +14 -0
  54. package/dist/ws/index.d.ts +1 -1
  55. package/dist/ws/index.js +1 -1
  56. package/dist/ws/pending-tracker.d.ts +93 -0
  57. package/dist/ws/pending-tracker.js +132 -0
  58. package/dist/ws/sidecar-events.d.ts +43 -29
  59. package/dist/ws/sidecar-events.js +0 -2
  60. package/dist/ws/sidecar-handler.d.ts +122 -85
  61. package/dist/ws/sidecar-handler.js +925 -878
  62. package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
  63. package/dist/ws/sidecar-handler.test-helpers.js +95 -0
  64. package/dist/ws/sidecar-token-authenticator.js +37 -23
  65. package/package.json +13 -13
  66. package/dist/sidecar-allocation/placement-policy.d.ts +0 -11
  67. package/dist/sidecar-allocation/placement-policy.js +0 -21
@@ -1,21 +1,20 @@
1
1
  import { type } from "arktype";
2
- import { and, eq } from "drizzle-orm";
2
+ import { and, eq, isNull } from "drizzle-orm";
3
3
  import { getLogger } from "@intx/log";
4
- import { assembleMessage, assembleSignedContent, createDetachedSignatureFromProvider, } from "@intx/mime";
5
- import { buildCredentialDelivery, listAssetsForTenant, } from "@intx/db";
6
- import { grant as grantTable, sidecarAllocation as sidecarAllocationTable, workflowDefinition as workflowDefinitionTable, workflowRun as workflowRunTable, } from "@intx/db/schema";
7
- import { base64Encode, hexEncode } from "@intx/types";
8
- import { generateId } from "@intx/hub-common";
4
+ import { buildCredentialDelivery, createSidecarAllocationStore, listAssetsForTenant, resolveInferenceMaterials, } from "@intx/db";
5
+ import { workflowDefinition as workflowDefinitionTable, workflowRun as workflowRunTable, } from "@intx/db/schema";
6
+ import { hexEncode } from "@intx/types";
9
7
  import { sessionAsset as sessionAssetTable } from "@intx/db/schema";
10
8
  import { AssetRegistrySource, HttpRegistrySource, ManifestInvalidError, createClosureResolver, } from "@intx/tool-packaging";
11
9
  import { ToolPackageManifest, } from "@intx/types/tool-packages";
12
- import { computeWireDefinitionHash } from "@intx/types/wire-definition-hash";
13
- import { buildInertProjectionStepSources, deriveRunAddress, enumerateInertOnTriggerBodies, pickStepInferenceSource, WorkflowDefinitionInvalidError, } from "@intx/workflow-deploy";
10
+ import { buildInertProjectionStepSources, collectAgentBearingStepIds, deriveRunAddress, WorkflowDefinitionInvalidError, } from "@intx/workflow-deploy";
14
11
  import { DEFAULT_ASSET_REF, } from "./asset-service.js";
12
+ import { isDeployFrameFailure } from "./ws/sidecar-handler.js";
15
13
  import { buildSourceAssetMounts, } from "./workflow-closure-resolution.js";
16
14
  import { restoreWorkflowRunToAllocation } from "./workflow-run-restore.js";
17
15
  import { committedReadsToSourceTree } from "./committed-source-tree.js";
18
16
  import { installAndApproveWorkflowDefinition, } from "./workflow-probe-gate.js";
17
+ import { buildReferencedWorkflowSourcePins } from "./workflow-source-pins.js";
19
18
  const logger = getLogger(["interchange", "hub", "session-service"]);
20
19
  export class SessionLaunchError extends Error {
21
20
  /** Which phase failed: "write", "provision", "pack", or "start". */
@@ -102,7 +101,8 @@ export function bridgeOrchestratorDeployContent(content) {
102
101
  * closure reaches the wire surface via `sendAgentDeploy` with a `workflow`
103
102
  * field structurally matching the `AgentDeployFrame.workflow` schema.
104
103
  */
105
- export async function sendMultiStepDeployFrame(args) {
104
+ export async function sendMultiStepDeployFrame(args, signal, beforeSend) {
105
+ signal?.throwIfAborted();
106
106
  const workflow = {
107
107
  // The deploy frame carries no inline definition: the sidecar evaluates the
108
108
  // pinned code closure from `sourceRef` and re-verifies it against
@@ -121,16 +121,7 @@ export async function sendMultiStepDeployFrame(args) {
121
121
  ? { assets: [...args.assets] }
122
122
  : {}),
123
123
  };
124
- // A prepared exclusive deploy routes its frame to the dedicated allocation; a
125
- // shared deploy sends it on the shared router. The frozen projection/hash/pin
126
- // ride verbatim in both cases -- only the transport differs.
127
- if (args.allocationTarget !== undefined) {
128
- if (args.sidecarAllocationRouter === undefined) {
129
- throw new Error("Exclusive deployment routing is not configured");
130
- }
131
- return args.sidecarAllocationRouter.sendAgentDeployToAllocation(args.allocationTarget, args.agentAddress, args.config, workflow);
132
- }
133
- return args.sidecarRouter.sendAgentDeploy(args.agentAddress, args.config, workflow);
124
+ return args.sidecarAllocationRouter.sendAgentDeployToAllocation(args.allocationTarget, args.agentAddress, args.config, workflow, signal, beforeSend);
134
125
  }
135
126
  function isAssetDeployArgs(args) {
136
127
  return args.source.kind === "asset";
@@ -145,8 +136,16 @@ function isAssetDeployArgs(args) {
145
136
  * closure, so the child re-verify over the inert projection matches the gate's
146
137
  * freeze.
147
138
  *
148
- * Credential MATERIAL for the definition's tenant-owned bindings is resolved
149
- * here (`buildCredentialDelivery`) and delivered to the child on the frame.
139
+ * Credential MATERIAL rides ONE `CredentialDelivery` delivered to the child on
140
+ * the frame, unioned from three rails and deduped by credentialId: tool bindings
141
+ * (grant-scoped, resolved here via `buildCredentialDelivery`); every top-level
142
+ * inference source; and every inline body step's inference source. Both inference
143
+ * rails are resolved HERE from the DB under the tenant-ownership authority
144
+ * (`resolveInferenceMaterials`), so the deploy is self-contained -- no caller
145
+ * pre-supplies material, and a spawned body child finds its secret in the cell.
146
+ * The merge is a post-authz union of already-cleared material (tool material is
147
+ * grant-scoped, inference material is tenant-ownership-scoped), never a shared
148
+ * authz check.
150
149
  * Credential GRANT enforcement is a SEPARATE layer: the `credential:{id}` /
151
150
  * `use` grant the runtime gate checks is minted per-run by run-grant
152
151
  * materialization into `runs/<runId>/grants.json`, not carried on this frame --
@@ -156,14 +155,17 @@ function isAssetDeployArgs(args) {
156
155
  * A gate outcome that did not approve cannot deploy: an unapproved `approval`
157
156
  * fails closed here rather than shipping an unfrozen definition.
158
157
  *
159
- * This emits the source-ref deploy frame ONLY -- it does NOT write the anchor
160
- * `workflow_run` row. `deployCodeSourcedWorkflow` wraps it with the shared-path
161
- * INSERT; the prepared exclusive path wraps it with an UPDATE-under-lock of the
162
- * anchor row that already exists from prepare time. It returns the frozen
163
- * definition id so each wrapper writes the same content-addressed identity the
164
- * gate persisted.
158
+ * This does the READ-ONLY preparation ONLY: it runs the guards, resolves
159
+ * credential material, pins the body sources, and builds the asset mounts, then
160
+ * returns the frozen definition id and the assembled send args. It emits NO
161
+ * frame and writes NO row, so it has no side effect to unwind. The ordinary path
162
+ * (`deployCodeSourcedWorkflow`) sequences prepare -> INSERT anchor -> emit so
163
+ * the anchor is visible before the frame spawns the child; `emitSourceRefDeployFrame`
164
+ * composes prepare -> emit for the prepared provisioned path, whose anchor row
165
+ * already exists from prepare time. It returns the frozen definition id so each
166
+ * caller writes the same content-addressed identity the gate persisted.
165
167
  */
166
- async function emitSourceRefDeployFrame(args) {
168
+ async function prepareSourceRefDeploy(args) {
167
169
  const { approval, projection, closure } = args.approved;
168
170
  if (!approval.ok) {
169
171
  throw new Error(`deployCodeSourcedWorkflow: refusing to deploy an unapproved workflow (gate reason: ${approval.reason})`);
@@ -224,11 +226,12 @@ async function emitSourceRefDeployFrame(args) {
224
226
  }
225
227
  credentials = delivery.delivery;
226
228
  }
227
- // Pin per-step inference sources for the projection's inline onTrigger bodies.
229
+ // Pin per-step inference sources for the projection's inline trigger bodies
230
+ // -- onTrigger sections and childWorkflow children, enumerated transitively.
228
231
  // The hub holds only the frozen inert projection, so it enumerates the inline
229
232
  // bodies from the wire form and resolves each body step's source through the
230
233
  // same resolver + operator-approval gate the top-level steps use
231
- // (`pickStepInferenceSource` against `approval.approvedGrants`). Each body's
234
+ // (`pickStepInferenceSource` against `approval.approvedSurface`). Each body's
232
235
  // wire hash is recomputed from the inert body verbatim, so a body child's
233
236
  // re-verify over the re-evaluated closure clears the same barrier a top-level
234
237
  // re-verify does. The pinned sources ride OUTSIDE the hash; their trust comes
@@ -244,43 +247,74 @@ async function emitSourceRefDeployFrame(args) {
244
247
  // rather than reading it off disk, so no body workflow.json is staged (see the
245
248
  // staging loop in workflow-host-wiring.ts and the anti-fallback guard in
246
249
  // workflow-host run-child.ts).
247
- const referencedDefinitions = await Promise.all(enumerateInertOnTriggerBodies(projection).map(async (body) => {
248
- const sources = {};
249
- for (const bodyStepId of body.definition.stepOrder) {
250
- // Agent-bearing body steps run inference and need a source pinned
251
- // through the approval gate. A non-agent body step (sleep,
252
- // awaitSignal) declares no preference and runs no inference, so it
253
- // advertises no `inference.source` grant the gate could approve --
254
- // but the deploy frame's coverage contract still requires a source
255
- // entry for EVERY body step. Pin the deploy's default source as an
256
- // inert placeholder for such a step: the body child resolves a
257
- // step's source only when that step invokes inference, so this entry
258
- // is never read, which is why it needs no operator approval.
259
- const preferred = body.preferredByStep[bodyStepId] ?? null;
260
- if (preferred === null) {
261
- const placeholder = args.config.sources.find((s) => s.id === args.config.defaultSource);
262
- if (placeholder === undefined) {
263
- throw new WorkflowDefinitionInvalidError(body.ref, `non-agent body step ${bodyStepId} needs an inert placeholder source, but the deploy config carries no defaultSource entry to pin`);
264
- }
265
- sources[bodyStepId] = [placeholder];
266
- continue;
250
+ const referencedDefinitions = await buildReferencedWorkflowSourcePins({
251
+ projection,
252
+ config: args.config,
253
+ operatorApprovals: approval.approvedSurface,
254
+ });
255
+ // Assemble the ONE credential delivery. Its `materials` cover three rails, each
256
+ // authorized upstream on its own terms, deduped by credentialId into one cell:
257
+ // - tool bindings, grant-scoped through `buildCredentialDelivery` above;
258
+ // - the inference source pinned to each top-level step that can actually
259
+ // invoke inference, tenant-owned;
260
+ // - the same for each inline body step (onTrigger/childWorkflow bodies
261
+ // pinned above), tenant-owned.
262
+ // Which steps those are is read from the hash-covered projection, never from
263
+ // the pinned map: every step carries a pin because the wire shape demands one,
264
+ // but a step that cannot issue a request has no use for a secret. Delivering
265
+ // one anyway decrypts a tenant credential, seals it to the sidecar, and
266
+ // re-delivers it on every reconnect, on behalf of a step that never makes a
267
+ // call.
268
+ // The inference rails are resolved HERE from the DB under the tenant-ownership
269
+ // authority, so this deploy is self-contained: a direct deploy (a test) that
270
+ // seeds the credentials in the DB -- rather than pre-supplying material -- still
271
+ // fills the cell, and a spawned body finds its secret rather than failing closed
272
+ // at resolve time. Precedence on a shared credentialId is tool material first
273
+ // (grant-scoped), then the inference material (tenant-ownership-scoped): the
274
+ // first material for an id wins. Inference sources carry NO binding descriptor
275
+ // -- they reference their credential by id directly.
276
+ const materials = new Map();
277
+ for (const material of credentials?.materials ?? []) {
278
+ materials.set(material.credentialId, material);
279
+ }
280
+ const inferenceCredentialIds = new Set();
281
+ const addAgentBearingCredentials = (definition, pinned, context) => {
282
+ for (const stepId of collectAgentBearingStepIds({ definition, context })) {
283
+ const stepSources = pinned[stepId];
284
+ if (stepSources === undefined) {
285
+ throw new Error(`${context}step ${stepId} can invoke inference but carries no pinned source`);
286
+ }
287
+ for (const source of stepSources) {
288
+ inferenceCredentialIds.add(source.credentialId);
267
289
  }
268
- sources[bodyStepId] = [
269
- pickStepInferenceSource({
270
- preferred,
271
- stepId: bodyStepId,
272
- workflowId: body.ref,
273
- config: args.config,
274
- operatorApprovals: approval.approvedGrants,
275
- }),
276
- ];
277
290
  }
278
- return {
279
- definition: body.definition,
280
- sources,
281
- approvedWireHash: await computeWireDefinitionHash(body.definition),
282
- };
283
- }));
291
+ };
292
+ addAgentBearingCredentials(projection, args.sources, "deployCodeSourcedWorkflow: ");
293
+ for (const body of referencedDefinitions) {
294
+ addAgentBearingCredentials(body.definition, body.sources, `deployCodeSourcedWorkflow body ${body.definition.id}: `);
295
+ }
296
+ if (inferenceCredentialIds.size > 0) {
297
+ if (args.credentialCipher === undefined) {
298
+ throw new Error("deployCodeSourcedWorkflow: pinned inference sources reference credentials " +
299
+ "but no credentialCipher was supplied to resolve them");
300
+ }
301
+ const inferenceMaterials = await resolveInferenceMaterials(args.db, args.tenantId, inferenceCredentialIds, args.credentialCipher);
302
+ for (const material of inferenceMaterials) {
303
+ if (!materials.has(material.credentialId)) {
304
+ materials.set(material.credentialId, material);
305
+ }
306
+ }
307
+ }
308
+ // A delivery with tool bindings always carries their material, so an empty map
309
+ // means no rail contributed anything -- send no delivery. (Tool bindings never
310
+ // produce a descriptor without a material, so bindings-without-materials cannot
311
+ // occur.)
312
+ const credentialDelivery = materials.size > 0
313
+ ? {
314
+ bindings: credentials?.bindings ?? [],
315
+ materials: [...materials.values()],
316
+ }
317
+ : undefined;
284
318
  // An asset-sourced pin's `kind:"asset"` closure entries read from source
285
319
  // assets the sidecar cannot fetch itself; deliver them inline on the frame so
286
320
  // the sidecar checks them out into its durable per-deployment source store. A
@@ -288,58 +322,238 @@ async function emitSourceRefDeployFrame(args) {
288
322
  const assets = isAssetDeployArgs(args)
289
323
  ? await buildSourceAssetMounts(closure, args.resolveAttachment)
290
324
  : [];
291
- const result = await sendMultiStepDeployFrame({
325
+ const sendArgs = {
292
326
  lineage: "source-ref",
293
- sidecarRouter: args.sidecarRouter,
294
- ...(args.sidecarAllocationRouter !== undefined
295
- ? { sidecarAllocationRouter: args.sidecarAllocationRouter }
296
- : {}),
297
- ...(args.allocationTarget !== undefined
298
- ? { allocationTarget: args.allocationTarget }
299
- : {}),
327
+ sidecarAllocationRouter: args.sidecarAllocationRouter,
328
+ allocationTarget: args.allocationTarget,
300
329
  agentAddress: args.agentAddress,
301
330
  config: args.config,
302
331
  sources: args.sources,
303
332
  approvedWireHash: approval.approvedWireHash,
304
333
  sourceRef: { source: args.source, closure },
305
- ...(credentials !== undefined ? { credentials } : {}),
334
+ ...(credentialDelivery !== undefined
335
+ ? { credentials: credentialDelivery }
336
+ : {}),
306
337
  ...(referencedDefinitions.length > 0 ? { referencedDefinitions } : {}),
307
338
  ...(assets.length > 0 ? { assets } : {}),
308
- });
309
- return { publicKey: result.publicKey, definitionId: approval.definitionId };
339
+ };
340
+ return { definitionId: approval.definitionId, sendArgs };
341
+ }
342
+ /**
343
+ * Prepare then emit the source-ref deploy frame, for the prepared provisioned
344
+ * path whose anchor `workflow_run` row already exists (inserted at prepare
345
+ * time). It emits the frame but does NOT touch the anchor row: the caller
346
+ * (`deployPreparedCodeSourcedWorkflow`) stamps the acked key under the
347
+ * allocation-ownership lock. A tagged `DeployFrameFailure` is converted to the
348
+ * `SessionLaunchError` disposition the allocation reconciler consumes, while
349
+ * untagged preparation errors remain safe same-generation retries. The ordinary
350
+ * path does NOT use this wrapper -- it must interleave the anchor INSERT between
351
+ * prepare and emit, so it drives `prepareSourceRefDeploy` and
352
+ * `sendMultiStepDeployFrame` directly.
353
+ */
354
+ // The non-secret projection of a delivery, persisted on the anchor run so the
355
+ // reconnect resync can re-resolve current materials. Secrets never land here.
356
+ function credentialRefsFromDelivery(delivery) {
357
+ return {
358
+ credentialIds: delivery.materials.map((material) => material.credentialId),
359
+ bindings: delivery.bindings,
360
+ };
361
+ }
362
+ async function emitSourceRefDeployFrame(args, reconciliation) {
363
+ const { signal, leaseId } = reconciliation;
364
+ signal.throwIfAborted();
365
+ const { definitionId, sendArgs } = await prepareSourceRefDeploy(args);
366
+ signal.throwIfAborted();
367
+ const allocationStore = createSidecarAllocationStore(args.db);
368
+ const initialization = {
369
+ allocationId: sendArgs.allocationTarget.allocationId,
370
+ generation: sendArgs.allocationTarget.generation,
371
+ anchorRunId: args.anchorRunId,
372
+ tenantId: args.tenantId,
373
+ leaseId,
374
+ signal,
375
+ };
376
+ let previousPublicKey;
377
+ try {
378
+ const result = await sendMultiStepDeployFrame(sendArgs, signal, async () => {
379
+ const reserved = await allocationStore.beginInitialization(initialization);
380
+ if (reserved === null) {
381
+ throw new Error("Allocation no longer permits this initialization attempt");
382
+ }
383
+ previousPublicKey = reserved.previousPublicKey;
384
+ });
385
+ return {
386
+ publicKey: result.publicKey,
387
+ definitionId,
388
+ ...(sendArgs.credentials !== undefined
389
+ ? { credentialRefs: credentialRefsFromDelivery(sendArgs.credentials) }
390
+ : {}),
391
+ };
392
+ }
393
+ catch (cause) {
394
+ if (!isDeployFrameFailure(cause))
395
+ throw cause;
396
+ // Only a confirmed reservation gives us the key to restore. An ambiguous
397
+ // reservation response leaves its durable marker for conservative cleanup.
398
+ // A confirmed unsent attempt can roll back even after lease cancellation.
399
+ if (!cause.frameSent && previousPublicKey !== undefined) {
400
+ try {
401
+ const cleared = await allocationStore.clearUnsentInitialization({
402
+ ...initialization,
403
+ previousPublicKey,
404
+ });
405
+ if (cleared)
406
+ args.onUnsentInitializationCleared(previousPublicKey);
407
+ }
408
+ catch (error) {
409
+ logger.warn `Could not clear unsent initialization for ${initialization.allocationId}: ${error instanceof Error ? error.message : String(error)}`;
410
+ }
411
+ }
412
+ throw new SessionLaunchError("start", cause, cause.frameSent);
413
+ }
310
414
  }
311
415
  /**
312
- * The single public composition entrypoint for a SHARED code-sourced (npm)
313
- * deploy: emit the source-ref frame, then INSERT the deployment's anchor
314
- * `workflow_run` row -- the deployment's first-class record that owns its
315
- * routing address and public key. Run-grant materialization keys off this row
316
- * (address + live status), so WITHOUT it no per-run grants (tool, capability, OR
317
- * credential) ever materialize for a source-ref deployment. Born "deployed"
318
- * (live but pre-trigger): the first trigger's materialization flips it to
319
- * "running" via `anchorWithPrincipal`'s guarded update, which a row born
320
- * "running" would skip. Its `anchorRunId` equals its own id, so the anchor
321
- * references itself. The deployer read grant is deferred to the production
322
- * route, which carries the authenticated deployer principal; this stays a
323
- * single insert with no grant row to pair atomically.
416
+ * A direct allocation-bound composition entrypoint for tests and low-level
417
+ * callers: prepare, INSERT the deployment's anchor `workflow_run` row, THEN emit
418
+ * the source-ref frame. The anchor row is the deployment's first-class record
419
+ * that owns its routing address and public key. Run-grant materialization keys
420
+ * off this row (address + live status), so WITHOUT it no per-run grants (tool,
421
+ * capability, OR credential) ever materialize for a source-ref deployment. Born
422
+ * "deployed" (live but pre-trigger) with a null public key: the first trigger's
423
+ * materialization flips it to "running" via `anchorWithPrincipal`'s guarded
424
+ * update, which a row born "running" would skip. Its `anchorRunId` equals its
425
+ * own id, so the anchor references itself. The deployer read grant is deferred
426
+ * to the production route, which carries the authenticated deployer principal.
427
+ *
428
+ * ORDERING IS LOAD-BEARING. The anchor row must be committed and visible to the
429
+ * pack-receipt connection BEFORE the frame reaches the wire: the frame spawns
430
+ * the child, whose first events pack races the ack back, and
431
+ * `receiveWorkflowRunPack` fails closed on a missing live anchor. Emitting first
432
+ * (the previous order) rejected that first pack and never bootstrapped the log.
433
+ * This works because `args.db` is the autocommit handle (`DB["db"]`, which the
434
+ * type forbids from being a transaction) and the INSERT is NOT wrapped in a
435
+ * transaction with the emit -- so the row is durably visible the instant the
436
+ * INSERT statement returns. Do NOT relax `db` to a transaction executor or wrap
437
+ * anchor+emit in one transaction to make them atomic: that reopens the race.
324
438
  *
325
- * The prepared exclusive path does NOT use this wrapper: its anchor row already
326
- * exists from prepare time, so it wraps `emitSourceRefDeployFrame` with an
327
- * UPDATE-under-allocation-lock instead of this INSERT.
439
+ * On emit failure the anchor row is rolled back or fenced by the `frameSent`
440
+ * evidence from the transport. `leakedAgent: false` (safe to fully roll back) is
441
+ * the STRONG claim and is made only on positive proof the frame never reached
442
+ * the wire (`isDeployFrameFailure && frameSent === false`); every other failure
443
+ * -- a sent-but-unacked frame OR any untagged error -- is treated as
444
+ * possibly-live: the anchor is fenced `deployed` -> `failed` and the error is
445
+ * `leakedAgent: true`.
446
+ *
447
+ * The prepared provisioned path does NOT use this composition: its anchor row
448
+ * already exists from prepare time, so it drives `emitSourceRefDeployFrame` and
449
+ * an UPDATE-under-allocation-lock instead.
328
450
  */
329
451
  export async function deployCodeSourcedWorkflow(args) {
330
- const { publicKey, definitionId } = await emitSourceRefDeployFrame(args);
331
- await args.db.insert(workflowRunTable).values({
332
- id: args.anchorRunId,
333
- tenantId: args.tenantId,
334
- anchorRunId: args.anchorRunId,
335
- definitionId,
336
- address: args.agentAddress,
337
- publicKey,
338
- status: "deployed",
339
- createdAt: new Date(),
340
- });
452
+ const { definitionId, sendArgs } = await prepareSourceRefDeploy(args);
453
+ // INSERT the anchor before the frame. A collision or DB error here spawned
454
+ // nothing (no frame went out), so it is a clean, non-leaking failure.
455
+ try {
456
+ await args.db.insert(workflowRunTable).values({
457
+ id: args.anchorRunId,
458
+ tenantId: args.tenantId,
459
+ anchorRunId: args.anchorRunId,
460
+ definitionId,
461
+ address: args.agentAddress,
462
+ publicKey: null,
463
+ status: "deployed",
464
+ createdAt: new Date(),
465
+ // Persist the non-secret shape of the delivery so the reconnect resync
466
+ // can re-resolve current materials for these ids. Secrets never land here.
467
+ ...(sendArgs.credentials !== undefined
468
+ ? { credentialRefs: credentialRefsFromDelivery(sendArgs.credentials) }
469
+ : {}),
470
+ });
471
+ }
472
+ catch (cause) {
473
+ throw new SessionLaunchError("start", cause, false);
474
+ }
475
+ let publicKey;
476
+ try {
477
+ const result = await sendMultiStepDeployFrame(sendArgs);
478
+ publicKey = result.publicKey;
479
+ }
480
+ catch (cause) {
481
+ if (isDeployFrameFailure(cause) && cause.frameSent === false) {
482
+ // Positive proof the frame never reached the wire: nothing spawned, so
483
+ // fully roll the anchor back. The guard (`deployed`, null key) is a
484
+ // tripwire on the `frameSent: false` contract -- a 0-row delete means the
485
+ // row advanced or vanished, so the contract lied and a child may be live;
486
+ // surface that loudly and refuse to claim it is safe to roll back.
487
+ const deleted = await args.db
488
+ .delete(workflowRunTable)
489
+ .where(and(eq(workflowRunTable.id, args.anchorRunId), eq(workflowRunTable.anchorRunId, args.anchorRunId), eq(workflowRunTable.tenantId, args.tenantId), eq(workflowRunTable.status, "deployed"), isNull(workflowRunTable.publicKey)))
490
+ .returning({ id: workflowRunTable.id });
491
+ if (deleted.length === 0) {
492
+ logger.error `anchor-before-frame rollback found no deployed/null-key row for ${args.anchorRunId} after a frameSent:false failure; the never-sent contract was violated and a child may be live`;
493
+ throw new SessionLaunchError("start", cause, true);
494
+ }
495
+ throw new SessionLaunchError("start", cause, false);
496
+ }
497
+ // A sent-but-unacked frame, OR any untagged/unexpected error: no positive
498
+ // proof of a clean send, so treat the agent as possibly-live. Fence the
499
+ // anchor `deployed` -> `failed` (guarded so a self-flip to "running" by a
500
+ // trigger that already landed is left alone). Do NOT delete: a live child
501
+ // needs the anchor to bootstrap.
502
+ const flipped = await args.db
503
+ .update(workflowRunTable)
504
+ .set({ status: "failed" })
505
+ .where(and(eq(workflowRunTable.id, args.anchorRunId), eq(workflowRunTable.anchorRunId, args.anchorRunId), eq(workflowRunTable.tenantId, args.tenantId), eq(workflowRunTable.status, "deployed"), isNull(workflowRunTable.publicKey)))
506
+ .returning({ id: workflowRunTable.id });
507
+ if (flipped.length === 0) {
508
+ // The anchor already advanced past deployed -- a trigger flipped it to
509
+ // "running", so the deploy actually succeeded and the run is progressing
510
+ // despite the ack failure. Leave it; the leaked-agent disposition still
511
+ // holds because the frame was (or may have been) sent.
512
+ logger.warn `anchor-before-frame: anchor ${args.anchorRunId} already advanced past deployed on an unacked/failed emit; the agent is live and the run is progressing despite the ack failure`;
513
+ }
514
+ else {
515
+ logger.warn `anchor-before-frame: fenced anchor ${args.anchorRunId} deployed->failed on an unacked/failed emit; the agent may be leaked but the run is dead`;
516
+ }
517
+ throw new SessionLaunchError("start", cause, true);
518
+ }
519
+ // Emit succeeded: stamp the acked key. No status guard -- the key is a fact
520
+ // regardless of whether the pack-ack race already flipped the row to
521
+ // "running", and skipping the stamp there would strand a live run with a null
522
+ // key. A 0-row update is an anomaly (nothing should remove a deployed anchor
523
+ // on the success path), but the deploy succeeded, so log it rather than
524
+ // failing a live run.
525
+ const stamped = await args.db
526
+ .update(workflowRunTable)
527
+ .set({ publicKey })
528
+ .where(and(eq(workflowRunTable.id, args.anchorRunId), eq(workflowRunTable.anchorRunId, args.anchorRunId), eq(workflowRunTable.tenantId, args.tenantId)))
529
+ .returning({ id: workflowRunTable.id });
530
+ if (stamped.length === 0) {
531
+ logger.error `anchor-before-frame: anchor ${args.anchorRunId} vanished before its public key could be stamped on a successful deploy`;
532
+ }
341
533
  return { publicKey };
342
534
  }
535
+ /** Resolve deferred sender mail after claiming the previous initializer's lease. */
536
+ export async function recoverSenderDeploy(args) {
537
+ const { allocation, reconciliation } = args;
538
+ reconciliation.signal.throwIfAborted();
539
+ // A proven-unsent clear may still restore the previous key after this claim.
540
+ // Do not fail its mail using the claim's stale marker. Cleanup rechecks under
541
+ // the allocation lock; an advanced fence settles failure, while a rolled-back
542
+ // attempt is resolved by its caller or the next claim's completed key.
543
+ if (allocation.initializationLeaseId !== undefined)
544
+ return;
545
+ // Claiming the lease prevents the previous attempt from publishing. Its
546
+ // marker and key now distinguish a committed initialization from failure,
547
+ // even before the worker reconnects or the old response arrives.
548
+ const anchor = await args.db.query.workflowRun.findFirst({
549
+ where: eq(workflowRunTable.id, allocation.anchorRunId),
550
+ columns: { publicKey: true },
551
+ });
552
+ reconciliation.signal.throwIfAborted();
553
+ args.sidecarRouter.noteSenderDeploySettled({ allocationId: allocation.id, generation: allocation.generation }, anchor !== undefined && anchor.publicKey !== null
554
+ ? { recorded: anchor.publicKey }
555
+ : { failed: "Previous deployment initialization did not complete" });
556
+ }
343
557
  export function createSessionService(deps) {
344
558
  const { sidecarRouter, sidecarAllocationRouter, agentRepoStore, assetService, db, toolPackageRegistries, } = deps;
345
559
  if (assetService !== undefined && db === undefined) {
@@ -350,7 +564,7 @@ export function createSessionService(deps) {
350
564
  }
351
565
  function requireAllocationRouter() {
352
566
  if (sidecarAllocationRouter === undefined) {
353
- throw new Error("Exclusive deployment routing is not configured");
567
+ throw new Error("Provisioned deployment routing is not configured");
354
568
  }
355
569
  return sidecarAllocationRouter;
356
570
  }
@@ -455,12 +669,7 @@ export function createSessionService(deps) {
455
669
  // route is held only for the pack window and dropped in the `finally`.
456
670
  if (stageOnly) {
457
671
  try {
458
- if (params.allocationTarget === undefined) {
459
- sidecarRouter.bindStepRoute(agentAddress);
460
- }
461
- else {
462
- await requireAllocationRouter().bindAllocatedStepRoute(params.allocationTarget, agentAddress);
463
- }
672
+ await requireAllocationRouter().bindAllocatedStepRoute(params.allocationTarget, agentAddress);
464
673
  }
465
674
  catch (err) {
466
675
  throw new SessionLaunchError("provision", err, false);
@@ -474,12 +683,7 @@ export function createSessionService(deps) {
474
683
  // exist before the pack applies.
475
684
  try {
476
685
  if (stageOnly) {
477
- if (params.allocationTarget === undefined) {
478
- await sidecarRouter.sendProvisionStep(agentAddress, config);
479
- }
480
- else {
481
- await requireAllocationRouter().sendProvisionStepToAllocation(params.allocationTarget, agentAddress, config);
482
- }
686
+ await requireAllocationRouter().sendProvisionStepToAllocation(params.allocationTarget, agentAddress, config);
483
687
  }
484
688
  else {
485
689
  // Every caller supplies `stageOnly`. A deploy without it has no
@@ -500,26 +704,17 @@ export function createSessionService(deps) {
500
704
  // the orphaned repo. This is an acceptable minor leak on the exceptional
501
705
  // staging-failure path, not a live-path cost.
502
706
  try {
503
- if (params.allocationTarget === undefined) {
504
- await sidecarRouter.sendPack(agentAddress, pack, ref, commitSha);
505
- }
506
- else {
507
- await requireAllocationRouter().sendPackToAllocation(params.allocationTarget, agentAddress, pack, ref, commitSha);
508
- }
707
+ await requireAllocationRouter().sendPackToAllocation(params.allocationTarget, agentAddress, pack, ref, commitSha);
509
708
  }
510
709
  catch (err) {
511
- if (!stageOnly && params.allocationTarget === undefined) {
512
- await attemptCleanup(agentAddress, "pack", err);
513
- }
514
- throw new SessionLaunchError("pack", err, !stageOnly && params.allocationTarget !== undefined);
710
+ throw new SessionLaunchError("pack", err, !stageOnly);
515
711
  }
516
712
  // Phase 2b: Asset-pack fan-out. For each attached asset, build a
517
713
  // pack, reserve the manifest row, then send the pack. The manifest
518
714
  // reservation MUST happen before the pack send: if the sidecar acks
519
715
  // but the row is missing, the session has materialization without
520
- // a recorded manifest. An allocated replacement may reuse the exact
521
- // row its predecessor recorded; ordinary launches still require a new
522
- // row. If reservation fails, the pack send must not happen.
716
+ // a recorded manifest. An allocated replacement may reuse the exact row
717
+ // its predecessor recorded. If reservation fails, no pack is sent.
523
718
  //
524
719
  // The fan-out materializes the package-registry assets the
525
720
  // tool-package resolver picked. They live behind tenant
@@ -528,35 +723,19 @@ export function createSessionService(deps) {
528
723
  // `manifestAssetAttachments`.
529
724
  const fanOut = manifestAssetAttachments;
530
725
  if (assetService !== undefined && fanOut.length > 0) {
531
- // Track the rows this attempt owns so a later fan-out failure can roll
532
- // them back in lockstep with the sidecar undeploy. Allocated rows are
533
- // durable recovery intent, not attempt-owned materialization state, so
534
- // replacement failures must leave them in place for the next worker.
535
- const committed = [];
536
726
  for (const att of fanOut) {
537
727
  try {
538
- const committedRecord = await sendAttachmentPack(runId, agentAddress, att, params.allocationTarget);
539
- if (committedRecord !== null)
540
- committed.push(committedRecord);
728
+ await sendAttachmentPack(runId, agentAddress, att, params.allocationTarget);
541
729
  }
542
730
  catch (err) {
543
- await rollbackCommittedAttachments(committed);
544
- if (!stageOnly && params.allocationTarget === undefined) {
545
- await attemptCleanup(agentAddress, "pack", err);
546
- }
547
- throw new SessionLaunchError("pack", err, !stageOnly && params.allocationTarget !== undefined);
731
+ throw new SessionLaunchError("pack", err, !stageOnly);
548
732
  }
549
733
  }
550
734
  }
551
735
  }
552
736
  finally {
553
737
  if (stageOnly) {
554
- if (params.allocationTarget === undefined) {
555
- sidecarRouter.unbindStepRoute(agentAddress);
556
- }
557
- else {
558
- requireAllocationRouter().unbindAllocatedStepRoute(params.allocationTarget, agentAddress);
559
- }
738
+ requireAllocationRouter().unbindAllocatedStepRoute(params.allocationTarget, agentAddress);
560
739
  }
561
740
  }
562
741
  }
@@ -581,9 +760,7 @@ export function createSessionService(deps) {
581
760
  ...(params.toolPackagePins !== undefined
582
761
  ? { toolPackagePins: params.toolPackagePins }
583
762
  : {}),
584
- ...(params.allocationTarget !== undefined
585
- ? { allocationTarget: params.allocationTarget }
586
- : {}),
763
+ allocationTarget: params.allocationTarget,
587
764
  });
588
765
  }
589
766
  // Resolve the npm registry config a code-sourced install resolves external
@@ -630,11 +807,17 @@ export function createSessionService(deps) {
630
807
  throw new Error("deployWorkflowFromSource requires a db handle to freeze the approval");
631
808
  }
632
809
  const dbHandle = db;
810
+ const allocationTarget = params.allocationTarget;
633
811
  const common = {
634
812
  entry: params.entry,
635
813
  assetId: params.definitionAssetId,
636
- approvals: { mode: "approve-probed" },
637
- router: sidecarRouter,
814
+ approvals: { kind: "approve-probed" },
815
+ router: {
816
+ sendProbe: (args) => requireAllocationRouter().sendProbeToAllocation(allocationTarget, args),
817
+ },
818
+ ...(params.onProbeResult !== undefined
819
+ ? { onProbeResult: params.onProbeResult }
820
+ : {}),
638
821
  db: dbHandle,
639
822
  };
640
823
  const source = params.source;
@@ -703,17 +886,17 @@ export function createSessionService(deps) {
703
886
  // Install + probe + gate + freeze a code-sourced definition, returning the
704
887
  // frozen bundle and the (asset-only) attachment resolver. The gate outcome is
705
888
  // NOT asserted here: `deployWorkflowFromSource` and `installAndApproveWorkflowSource`
706
- // each surface a non-approval as their own domain error. This is the shared
707
- // freeze both the shared deploy and the exclusive prepare run.
889
+ // each surface a non-approval as their own domain error. This is the common
890
+ // freeze used by direct tests and provisioned prepare runs.
708
891
  async function prepareCodeSourcedApproval(params) {
709
892
  const resolveAttachment = bindSourceAttachmentResolver(params.source);
710
893
  const installArgs = await buildInstallArgs(params, resolveAttachment);
711
894
  const approved = await installAndApproveWorkflowDefinition(installArgs);
712
895
  return { approved, resolveAttachment };
713
896
  }
714
- // Freeze a code-sourced approval on shared capacity WITHOUT deploying it. The
715
- // exclusive prepare path persists the returned bundle and deploys it to a
716
- // dedicated allocation later. A non-approval fails closed as an invalid
897
+ // Freeze a code-sourced approval WITHOUT deploying it. The provisioned
898
+ // prepare path persists the returned bundle and deploys it to an allocation
899
+ // later. A non-approval fails closed as an invalid
717
900
  // definition.
718
901
  async function installAndApproveWorkflowSource(params) {
719
902
  const { approved } = await prepareCodeSourcedApproval(params);
@@ -722,79 +905,12 @@ export function createSessionService(deps) {
722
905
  }
723
906
  return approved;
724
907
  }
725
- async function deployWorkflowFromSource(params) {
726
- if (db === undefined) {
727
- throw new Error("deployWorkflowFromSource requires a db handle to record the deployment's anchor run");
728
- }
729
- const source = params.source;
730
- const { approved, resolveAttachment } = await prepareCodeSourcedApproval(params);
731
- if (!approved.approval.ok) {
732
- throw new WorkflowDefinitionInvalidError(approved.projection.id, `code-sourced workflow install did not approve (reason: ${approved.approval.reason})`);
733
- }
734
- // Pin every top-level step's inference source under the frozen approval,
735
- // then hand the frozen bundle to the source-ref deploy.
736
- const sources = buildInertProjectionStepSources({
737
- projection: approved.projection,
738
- config: params.config,
739
- operatorApprovals: approved.approval.approvedGrants,
740
- });
741
- const commonDeploy = {
742
- approved,
743
- sidecarRouter,
744
- agentAddress: params.agentAddress,
745
- config: params.config,
746
- sources,
747
- db,
748
- tenantId: params.tenantId,
749
- anchorRunId: params.anchorRunId,
750
- deploymentDomain: params.deploymentDomain,
751
- };
752
- // Branch on the source discriminant so the deploy args match the
753
- // asset/registry arms of `DeployCodeSourcedWorkflowArgs`: an asset arm
754
- // carries the attachment resolver (asserted non-null here to satisfy the
755
- // union and fail loud on a mis-wired caller), a registry arm carries none.
756
- let result;
757
- if (source.kind === "asset") {
758
- if (resolveAttachment === null) {
759
- throw new Error("deployWorkflowFromSource: asset source deploy is missing its attachment resolver");
760
- }
761
- result = await deployCodeSourcedWorkflow({
762
- ...commonDeploy,
763
- source,
764
- resolveAttachment,
765
- });
766
- }
767
- else {
768
- result = await deployCodeSourcedWorkflow({ ...commonDeploy, source });
769
- }
770
- // Seed the deploying principal's read grant on the deployment's workflow-run
771
- // resource. `deployCodeSourcedWorkflow` wrote the anchor row but deliberately
772
- // leaves this grant to the route, which carries the authenticated deployer
773
- // principal.
774
- const now = new Date();
775
- await db.insert(grantTable).values({
776
- id: generateId("grant"),
777
- tenantId: params.tenantId,
778
- principalId: params.config.principalId,
779
- resource: `workflow-run:${params.anchorRunId}`,
780
- action: "read",
781
- effect: "allow",
782
- origin: "creator",
783
- createdAt: now,
784
- updatedAt: now,
785
- });
786
- return {
787
- anchorRunId: params.anchorRunId,
788
- deploymentAddress: params.agentAddress,
789
- publicKey: result.publicKey,
790
- };
791
- }
792
908
  /**
793
909
  * Update a prepared anchor run's `publicKey` under the allocation-ownership
794
910
  * lock. The anchor row was inserted at prepare time; this stamps the
795
911
  * supervisor key returned by the deploy ack, but only while the allocation
796
- * still names this exact accepted generation for this anchor. A lost lock (the
797
- * allocation moved on, another worker took the generation) fails closed as a
912
+ * still names this exact accepted generation and unexpired reconciliation
913
+ * lease for this anchor. Lost ownership or cancellation fails closed as a
798
914
  * leaked-agent `SessionLaunchError` -- the deploy already reached the sidecar,
799
915
  * so the caller must treat the sidecar agent as possibly live. Used by the
800
916
  * `deployPreparedCodeSourcedWorkflow` prepared path.
@@ -805,35 +921,19 @@ export function createSessionService(deps) {
805
921
  }
806
922
  const dbHandle = db;
807
923
  try {
808
- const updated = await dbHandle.transaction(async (tx) => {
809
- const [allocation] = await tx
810
- .select({
811
- id: sidecarAllocationTable.id,
812
- anchorRunId: sidecarAllocationTable.anchorRunId,
813
- status: sidecarAllocationTable.status,
814
- generation: sidecarAllocationTable.generation,
815
- ensureAcceptedGeneration: sidecarAllocationTable.ensureAcceptedGeneration,
816
- })
817
- .from(sidecarAllocationTable)
818
- .where(eq(sidecarAllocationTable.id, args.allocationTarget.allocationId))
819
- .limit(1)
820
- .for("update");
821
- if (allocation === undefined ||
822
- allocation.anchorRunId !== args.anchorRunId ||
823
- allocation.status !== "allocated" ||
824
- allocation.generation !== args.allocationTarget.generation ||
825
- allocation.ensureAcceptedGeneration !==
826
- args.allocationTarget.generation) {
827
- return null;
828
- }
829
- const [anchor] = await tx
830
- .update(workflowRunTable)
831
- .set({ publicKey: args.publicKey })
832
- .where(and(eq(workflowRunTable.id, args.anchorRunId), eq(workflowRunTable.anchorRunId, args.anchorRunId), eq(workflowRunTable.tenantId, args.tenantId)))
833
- .returning({ id: workflowRunTable.id });
834
- return anchor ?? null;
924
+ const updated = await createSidecarAllocationStore(dbHandle).completeInitialization({
925
+ allocationId: args.allocationTarget.allocationId,
926
+ generation: args.allocationTarget.generation,
927
+ anchorRunId: args.anchorRunId,
928
+ tenantId: args.tenantId,
929
+ leaseId: args.reconciliation.leaseId,
930
+ signal: args.reconciliation.signal,
931
+ publicKey: args.publicKey,
932
+ ...(args.credentialRefs !== undefined
933
+ ? { credentialRefs: args.credentialRefs }
934
+ : {}),
835
935
  });
836
- if (updated === null) {
936
+ if (!updated) {
837
937
  throw new Error(`Prepared anchor run ${args.anchorRunId} lost allocation ownership before initialization completed`);
838
938
  }
839
939
  }
@@ -852,6 +952,8 @@ export function createSessionService(deps) {
852
952
  * resolved source carries a credential secret).
853
953
  */
854
954
  async function deployPreparedCodeSourcedWorkflow(params) {
955
+ const { signal } = params.reconciliation;
956
+ signal.throwIfAborted();
855
957
  if (db === undefined) {
856
958
  throw new Error("deployPreparedCodeSourcedWorkflow requires a db handle to update the prepared anchor run");
857
959
  }
@@ -864,11 +966,11 @@ export function createSessionService(deps) {
864
966
  const source = params.source;
865
967
  const resolveAttachment = bindSourceAttachmentResolver(source);
866
968
  // Re-pin every top-level step's inference source from the re-resolved chain
867
- // under the frozen approval -- the same pin the shared deploy computes.
969
+ // under the frozen approval -- the same pin the source-ref deploy computes.
868
970
  const sources = buildInertProjectionStepSources({
869
971
  projection: params.approved.projection,
870
972
  config: params.config,
871
- operatorApprovals: approval.approvedGrants,
973
+ operatorApprovals: approval.approvedSurface,
872
974
  });
873
975
  // Restore the Hub-authoritative run ref onto the exact allocation generation
874
976
  // before its address is routed.
@@ -877,10 +979,15 @@ export function createSessionService(deps) {
877
979
  allocationRouter,
878
980
  allocationTarget: params.allocationTarget,
879
981
  agentAddress: params.agentAddress,
982
+ signal,
880
983
  });
984
+ signal.throwIfAborted();
985
+ let restoredPublicKey;
881
986
  const commonEmit = {
987
+ onUnsentInitializationCleared(publicKey) {
988
+ restoredPublicKey = publicKey;
989
+ },
882
990
  approved: params.approved,
883
- sidecarRouter,
884
991
  sidecarAllocationRouter: allocationRouter,
885
992
  allocationTarget: params.allocationTarget,
886
993
  agentAddress: params.agentAddress,
@@ -898,48 +1005,68 @@ export function createSessionService(deps) {
898
1005
  // arms: an asset arm carries the rebuilt attachment resolver (asserted
899
1006
  // non-null to satisfy the union), a registry arm carries none.
900
1007
  let result;
901
- if (source.kind === "asset") {
902
- if (resolveAttachment === null) {
903
- throw new Error("deployPreparedCodeSourcedWorkflow: asset source deploy is missing its attachment resolver");
1008
+ const senderAttempt = {
1009
+ ...params.allocationTarget,
1010
+ leaseId: params.reconciliation.leaseId,
1011
+ };
1012
+ signal.throwIfAborted();
1013
+ sidecarRouter.noteSenderDeployStarted(params.agentAddress, senderAttempt);
1014
+ try {
1015
+ // Bracket the allocated pre-ack window: mark the sender's key-record as
1016
+ // mid-flight before the deploy emit so a run that sends mail before its
1017
+ // anchor key is committed parks rather than delivering keyless. The settle
1018
+ // follows durable completion; cancellation leaves the outcome to recovery.
1019
+ if (source.kind === "asset") {
1020
+ if (resolveAttachment === null) {
1021
+ throw new Error("deployPreparedCodeSourcedWorkflow: asset source deploy is missing its attachment resolver");
1022
+ }
1023
+ result = await emitSourceRefDeployFrame({ ...commonEmit, source, resolveAttachment }, params.reconciliation);
904
1024
  }
905
- result = await emitSourceRefDeployFrame({
906
- ...commonEmit,
907
- source,
908
- resolveAttachment,
1025
+ else {
1026
+ result = await emitSourceRefDeployFrame({ ...commonEmit, source }, params.reconciliation);
1027
+ }
1028
+ await updateAnchorPublicKeyUnderAllocationLock({
1029
+ tenantId: params.tenantId,
1030
+ anchorRunId: params.anchorRunId,
1031
+ allocationTarget: params.allocationTarget,
1032
+ reconciliation: params.reconciliation,
1033
+ publicKey: result.publicKey,
1034
+ ...(result.credentialRefs !== undefined
1035
+ ? { credentialRefs: result.credentialRefs }
1036
+ : {}),
909
1037
  });
1038
+ // The anchor's public key is now durable. Wake any mail the run parked
1039
+ // while pre-ack so it delivers with the sender key co-delivered, closing
1040
+ // the window where a run sends before its key is recorded. The write above
1041
+ // happens-before this settle, so a re-drive resolves the recorded key.
1042
+ // `params.agentAddress` is the run's deploy address, byte-identical to the
1043
+ // sender address its mail was sent under (asserted against the anchor at
1044
+ // deploy time), so a settle matches the parked entries.
1045
+ sidecarRouter.noteSenderDeploySettled(senderAttempt, {
1046
+ recorded: result.publicKey,
1047
+ });
1048
+ return {
1049
+ anchorRunId: params.anchorRunId,
1050
+ deploymentAddress: params.agentAddress,
1051
+ publicKey: result.publicKey,
1052
+ };
910
1053
  }
911
- else {
912
- result = await emitSourceRefDeployFrame({ ...commonEmit, source });
913
- }
914
- await updateAnchorPublicKeyUnderAllocationLock({
915
- tenantId: params.tenantId,
916
- anchorRunId: params.anchorRunId,
917
- allocationTarget: params.allocationTarget,
918
- publicKey: result.publicKey,
919
- });
920
- return {
921
- anchorRunId: params.anchorRunId,
922
- deploymentAddress: params.agentAddress,
923
- publicKey: result.publicKey,
924
- };
925
- }
926
- async function rollbackCommittedAttachments(committed) {
927
- if (db === undefined)
928
- return;
929
- if (committed.length === 0)
930
- return;
931
- // Per-row try/catch so a single rollback failure does not stop the
932
- // sweep — every committed row needs to come off the books before
933
- // the caller emits the original sendPack error.
934
- for (const record of committed) {
935
- try {
936
- await db
937
- .delete(sessionAssetTable)
938
- .where(and(eq(sessionAssetTable.runId, record.runId), eq(sessionAssetTable.mountPath, record.mountPath), eq(sessionAssetTable.assetPackSha, record.assetPackSha), eq(sessionAssetTable.sourceCommitSha, record.sourceCommitSha)));
1054
+ catch (error) {
1055
+ if (restoredPublicKey !== undefined) {
1056
+ sidecarRouter.noteSenderDeploySettled(senderAttempt, restoredPublicKey === null
1057
+ ? { failed: error instanceof Error ? error.message : String(error) }
1058
+ : { recorded: restoredPublicKey });
939
1059
  }
940
- catch (err) {
941
- logger.warn `session_asset rollback failed for earlier-committed instance=${record.runId} mountPath=${record.mountPath}: ${err instanceof Error ? err.message : String(err)}`;
1060
+ // A sent deploy or cancelled publication may have committed despite its
1061
+ // lost response, as may an unsent rollback. Without a confirmed rollback,
1062
+ // recovery owns transport failures too. Only an uncancelled preparation
1063
+ // failure can settle definitively here.
1064
+ else if (!signal.aborted && !(error instanceof SessionLaunchError)) {
1065
+ sidecarRouter.noteSenderDeploySettled(senderAttempt, {
1066
+ failed: error instanceof Error ? error.message : String(error),
1067
+ });
942
1068
  }
1069
+ throw error;
943
1070
  }
944
1071
  }
945
1072
  async function sendAttachmentPack(runId, agentAddress, attachment, allocationTarget) {
@@ -957,77 +1084,32 @@ export function createSessionService(deps) {
957
1084
  assetPackSha,
958
1085
  sourceCommitSha,
959
1086
  };
960
- // Reserve the manifest row before the pack send so we never end up in the
961
- // materialized-without-manifest state. Only an allocated launch may reuse
962
- // an identical row: replacement workers keep the stable instance id and
963
- // mount path, while the shared path retains its duplicate-launch guard.
964
- const rollbackRecord = allocationTarget === undefined ? record : null;
965
- if (allocationTarget === undefined) {
966
- await db
967
- .insert(sessionAssetTable)
968
- .values({ ...record, materializedAt: new Date() });
969
- }
970
- else {
971
- const inserted = await db
972
- .insert(sessionAssetTable)
973
- .values({ ...record, materializedAt: new Date() })
974
- .onConflictDoNothing({
975
- target: [sessionAssetTable.runId, sessionAssetTable.mountPath],
976
- })
977
- .returning({ runId: sessionAssetTable.runId });
978
- if (inserted.length === 0) {
979
- const existing = await db.query.sessionAsset.findFirst({
980
- where: and(eq(sessionAssetTable.runId, runId), eq(sessionAssetTable.mountPath, mountPath)),
981
- columns: {
982
- assetPackSha: true,
983
- sourceCommitSha: true,
984
- },
985
- });
986
- if (existing === undefined) {
987
- throw new Error(`session_asset ${runId}/${mountPath} disappeared after its insert conflicted`);
988
- }
989
- if (existing.assetPackSha !== assetPackSha ||
990
- existing.sourceCommitSha !== sourceCommitSha) {
991
- throw new Error(`session_asset ${runId}/${mountPath} conflicts with the allocated workflow's restored asset`);
992
- }
993
- }
994
- }
995
- try {
996
- const options = { mountPath, repoId };
997
- if (allocationTarget === undefined) {
998
- await sidecarRouter.sendPack(agentAddress, pack, ref, sourceCommitSha, options);
999
- }
1000
- else {
1001
- await requireAllocationRouter().sendPackToAllocation(allocationTarget, agentAddress, pack, ref, sourceCommitSha, options);
1087
+ // Reserve durable recovery intent before the pack send. A replacement
1088
+ // generation may reuse the exact row its predecessor recorded.
1089
+ const inserted = await db
1090
+ .insert(sessionAssetTable)
1091
+ .values({ ...record, materializedAt: new Date() })
1092
+ .onConflictDoNothing({
1093
+ target: [sessionAssetTable.runId, sessionAssetTable.mountPath],
1094
+ })
1095
+ .returning({ runId: sessionAssetTable.runId });
1096
+ if (inserted.length === 0) {
1097
+ const existing = await db.query.sessionAsset.findFirst({
1098
+ where: and(eq(sessionAssetTable.runId, runId), eq(sessionAssetTable.mountPath, mountPath)),
1099
+ columns: {
1100
+ assetPackSha: true,
1101
+ sourceCommitSha: true,
1102
+ },
1103
+ });
1104
+ if (existing === undefined) {
1105
+ throw new Error(`session_asset ${runId}/${mountPath} disappeared after its insert conflicted`);
1002
1106
  }
1003
- }
1004
- catch (err) {
1005
- // Shared launches own the row they just created and roll it back when
1006
- // the send fails. Allocated rows are durable recovery intent: even a row
1007
- // first inserted by this attempt can already be reused by another
1008
- // reconciler, so no replacement attempt may delete it.
1009
- // The forensic value of a manifest-without-materialization row is
1010
- // negligible because no agent will read against it. Wrap the
1011
- // rollback in its own try/catch so a rollback failure (DB gone,
1012
- // connection killed mid-launch) is logged rather than masking the
1013
- // primary sendPack error — the caller needs to see the original
1014
- // failure, not the secondary one.
1015
- if (rollbackRecord !== null) {
1016
- try {
1017
- await db
1018
- .delete(sessionAssetTable)
1019
- .where(and(eq(sessionAssetTable.runId, rollbackRecord.runId), eq(sessionAssetTable.mountPath, rollbackRecord.mountPath), eq(sessionAssetTable.assetPackSha, rollbackRecord.assetPackSha), eq(sessionAssetTable.sourceCommitSha, rollbackRecord.sourceCommitSha)));
1020
- }
1021
- catch (rollbackErr) {
1022
- const msg = rollbackErr instanceof Error
1023
- ? rollbackErr.message
1024
- : String(rollbackErr);
1025
- logger.warn `session_asset rollback failed for instance=${runId} mountPath=${mountPath}: ${msg}`;
1026
- }
1107
+ if (existing.assetPackSha !== assetPackSha ||
1108
+ existing.sourceCommitSha !== sourceCommitSha) {
1109
+ throw new Error(`session_asset ${runId}/${mountPath} conflicts with the allocated workflow's restored asset`);
1027
1110
  }
1028
- throw err;
1029
1111
  }
1030
- return rollbackRecord;
1112
+ await requireAllocationRouter().sendPackToAllocation(allocationTarget, agentAddress, pack, ref, sourceCommitSha, { mountPath, repoId });
1031
1113
  }
1032
1114
  /**
1033
1115
  * Build a per-agent `ClosureResolver` from the tenant's visible
@@ -1130,62 +1212,13 @@ export function createSessionService(deps) {
1130
1212
  ref: returnedRef,
1131
1213
  };
1132
1214
  }
1133
- async function attemptCleanup(agentAddress, failedPhase, originalErr) {
1134
- try {
1135
- await sidecarRouter.sendAgentUndeploy(agentAddress, failedPhase);
1136
- }
1137
- catch (cleanupErr) {
1138
- logger.error `Failed to clean up agent ${agentAddress} after ${failedPhase} failure: ${String(cleanupErr)}`;
1139
- // Preserve the original error as cause so the root cause is not
1140
- // lost when the cleanup also fails.
1141
- throw new SessionLaunchError(failedPhase, originalErr, true);
1142
- }
1143
- }
1144
- async function sendUserMessage(params) {
1145
- const { agentAddress, from, messageId, date, content, attachments, inReplyTo, references, sessionId, tenantId, cryptoProvider, } = params;
1146
- const headers = {
1147
- from,
1148
- to: [agentAddress],
1149
- cc: undefined,
1150
- date,
1151
- messageId,
1152
- subject: undefined,
1153
- inReplyTo,
1154
- references,
1155
- mimeVersion: "1.0",
1156
- interchangeType: "conversation.message",
1157
- interchangeCorrelationId: undefined,
1158
- interchangeTenantId: tenantId,
1159
- interchangeAgentId: undefined,
1160
- interchangeSessionId: sessionId,
1161
- interchangeOfferingId: undefined,
1162
- interchangeSchemaVersion: undefined,
1163
- traceparent: undefined,
1164
- tracestate: undefined,
1165
- };
1166
- const signedContent = assembleSignedContent({
1167
- kind: "conversation",
1168
- text: content,
1169
- ...(attachments !== undefined ? { attachments } : {}),
1170
- });
1171
- const signature = await createDetachedSignatureFromProvider(signedContent, cryptoProvider);
1172
- const rawMessage = assembleMessage(headers, signedContent, signature);
1173
- const base64 = base64Encode(rawMessage);
1174
- const delivered = sidecarRouter.routeMail(agentAddress, base64, messageId);
1175
- if (!delivered) {
1176
- throw new Error(`Failed to deliver message to ${agentAddress}: agent is unreachable`);
1177
- }
1178
- return rawMessage;
1179
- }
1180
1215
  async function endSession(agentAddress, reason) {
1181
1216
  await sidecarRouter.sendAgentUndeploy(agentAddress, reason);
1182
1217
  }
1183
1218
  return {
1184
1219
  stageWorkflowStep,
1185
- deployWorkflowFromSource,
1186
1220
  installAndApproveWorkflowSource,
1187
1221
  deployPreparedCodeSourcedWorkflow,
1188
- sendUserMessage,
1189
1222
  endSession,
1190
1223
  };
1191
1224
  }