@intx/hub-sessions 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-repo.d.ts +14 -2
- package/dist/agent-repo.js +17 -4
- package/dist/agent-state-kind.js +14 -63
- package/dist/asset-service.js +14 -10
- package/dist/credential-push.d.ts +48 -4
- package/dist/credential-push.js +138 -6
- package/dist/event-collector-registry.d.ts +2 -1
- package/dist/event-collector-registry.js +38 -9
- package/dist/event-collector.d.ts +11 -1
- package/dist/event-collector.js +36 -3
- package/dist/hub-session-lookups.d.ts +1 -1
- package/dist/hub-session-lookups.js +68 -72
- package/dist/hub-session-orchestrator.d.ts +2 -3
- package/dist/hub-session-orchestrator.js +13 -12
- package/dist/index.d.ts +7 -6
- package/dist/index.js +7 -6
- package/dist/reconciliation-scheduler.d.ts +14 -0
- package/dist/reconciliation-scheduler.js +55 -0
- package/dist/repo-store/index.d.ts +1 -0
- package/dist/repo-store/index.js +1 -0
- package/dist/repo-store/user-principal-gate.d.ts +26 -0
- package/dist/repo-store/user-principal-gate.js +78 -0
- package/dist/session-service.d.ts +66 -121
- package/dist/session-service.js +444 -411
- package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
- package/dist/sidecar-allocation/capability-policy.js +124 -0
- package/dist/sidecar-allocation/contracts.d.ts +29 -6
- package/dist/sidecar-allocation/contracts.js +7 -2
- package/dist/sidecar-allocation/index.d.ts +4 -3
- package/dist/sidecar-allocation/index.js +3 -2
- package/dist/sidecar-allocation/operation.d.ts +10 -0
- package/dist/sidecar-allocation/operation.js +54 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +16 -3
- package/dist/sidecar-allocation/plugin-registry.js +36 -12
- package/dist/sidecar-allocation/reconciler.d.ts +16 -4
- package/dist/sidecar-allocation/reconciler.js +486 -92
- package/dist/skill-kind.js +8 -62
- package/dist/substrate.d.ts +1 -1
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +21 -15
- package/dist/workflow-allocation-service.js +440 -125
- package/dist/workflow-dispatch-service.d.ts +4 -2
- package/dist/workflow-dispatch-service.js +89 -26
- package/dist/workflow-kind.d.ts +12 -0
- package/dist/workflow-kind.js +17 -60
- package/dist/workflow-probe-gate.d.ts +99 -27
- package/dist/workflow-probe-gate.js +196 -21
- package/dist/workflow-run-kind.d.ts +112 -19
- package/dist/workflow-run-kind.js +626 -210
- package/dist/workflow-run-restore.d.ts +1 -0
- package/dist/workflow-run-restore.js +5 -1
- package/dist/workflow-source-pins.d.ts +8 -0
- package/dist/workflow-source-pins.js +14 -0
- package/dist/ws/index.d.ts +1 -1
- package/dist/ws/index.js +1 -1
- package/dist/ws/pending-tracker.d.ts +93 -0
- package/dist/ws/pending-tracker.js +132 -0
- package/dist/ws/sidecar-events.d.ts +43 -29
- package/dist/ws/sidecar-events.js +0 -2
- package/dist/ws/sidecar-handler.d.ts +122 -85
- package/dist/ws/sidecar-handler.js +925 -878
- package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
- package/dist/ws/sidecar-handler.test-helpers.js +95 -0
- package/dist/ws/sidecar-token-authenticator.js +37 -23
- package/package.json +13 -13
- package/dist/sidecar-allocation/placement-policy.d.ts +0 -11
- package/dist/sidecar-allocation/placement-policy.js +0 -21
package/dist/session-service.js
CHANGED
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
import { type } from "arktype";
|
|
2
|
-
import { and, eq } from "drizzle-orm";
|
|
2
|
+
import { and, eq, isNull } from "drizzle-orm";
|
|
3
3
|
import { getLogger } from "@intx/log";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { base64Encode, hexEncode } from "@intx/types";
|
|
8
|
-
import { generateId } from "@intx/hub-common";
|
|
4
|
+
import { buildCredentialDelivery, createSidecarAllocationStore, listAssetsForTenant, resolveInferenceMaterials, } from "@intx/db";
|
|
5
|
+
import { workflowDefinition as workflowDefinitionTable, workflowRun as workflowRunTable, } from "@intx/db/schema";
|
|
6
|
+
import { hexEncode } from "@intx/types";
|
|
9
7
|
import { sessionAsset as sessionAssetTable } from "@intx/db/schema";
|
|
10
8
|
import { AssetRegistrySource, HttpRegistrySource, ManifestInvalidError, createClosureResolver, } from "@intx/tool-packaging";
|
|
11
9
|
import { ToolPackageManifest, } from "@intx/types/tool-packages";
|
|
12
|
-
import {
|
|
13
|
-
import { buildInertProjectionStepSources, deriveRunAddress, enumerateInertOnTriggerBodies, pickStepInferenceSource, WorkflowDefinitionInvalidError, } from "@intx/workflow-deploy";
|
|
10
|
+
import { buildInertProjectionStepSources, collectAgentBearingStepIds, deriveRunAddress, WorkflowDefinitionInvalidError, } from "@intx/workflow-deploy";
|
|
14
11
|
import { DEFAULT_ASSET_REF, } from "./asset-service.js";
|
|
12
|
+
import { isDeployFrameFailure } from "./ws/sidecar-handler.js";
|
|
15
13
|
import { buildSourceAssetMounts, } from "./workflow-closure-resolution.js";
|
|
16
14
|
import { restoreWorkflowRunToAllocation } from "./workflow-run-restore.js";
|
|
17
15
|
import { committedReadsToSourceTree } from "./committed-source-tree.js";
|
|
18
16
|
import { installAndApproveWorkflowDefinition, } from "./workflow-probe-gate.js";
|
|
17
|
+
import { buildReferencedWorkflowSourcePins } from "./workflow-source-pins.js";
|
|
19
18
|
const logger = getLogger(["interchange", "hub", "session-service"]);
|
|
20
19
|
export class SessionLaunchError extends Error {
|
|
21
20
|
/** Which phase failed: "write", "provision", "pack", or "start". */
|
|
@@ -102,7 +101,8 @@ export function bridgeOrchestratorDeployContent(content) {
|
|
|
102
101
|
* closure reaches the wire surface via `sendAgentDeploy` with a `workflow`
|
|
103
102
|
* field structurally matching the `AgentDeployFrame.workflow` schema.
|
|
104
103
|
*/
|
|
105
|
-
export async function sendMultiStepDeployFrame(args) {
|
|
104
|
+
export async function sendMultiStepDeployFrame(args, signal, beforeSend) {
|
|
105
|
+
signal?.throwIfAborted();
|
|
106
106
|
const workflow = {
|
|
107
107
|
// The deploy frame carries no inline definition: the sidecar evaluates the
|
|
108
108
|
// pinned code closure from `sourceRef` and re-verifies it against
|
|
@@ -121,16 +121,7 @@ export async function sendMultiStepDeployFrame(args) {
|
|
|
121
121
|
? { assets: [...args.assets] }
|
|
122
122
|
: {}),
|
|
123
123
|
};
|
|
124
|
-
|
|
125
|
-
// shared deploy sends it on the shared router. The frozen projection/hash/pin
|
|
126
|
-
// ride verbatim in both cases -- only the transport differs.
|
|
127
|
-
if (args.allocationTarget !== undefined) {
|
|
128
|
-
if (args.sidecarAllocationRouter === undefined) {
|
|
129
|
-
throw new Error("Exclusive deployment routing is not configured");
|
|
130
|
-
}
|
|
131
|
-
return args.sidecarAllocationRouter.sendAgentDeployToAllocation(args.allocationTarget, args.agentAddress, args.config, workflow);
|
|
132
|
-
}
|
|
133
|
-
return args.sidecarRouter.sendAgentDeploy(args.agentAddress, args.config, workflow);
|
|
124
|
+
return args.sidecarAllocationRouter.sendAgentDeployToAllocation(args.allocationTarget, args.agentAddress, args.config, workflow, signal, beforeSend);
|
|
134
125
|
}
|
|
135
126
|
function isAssetDeployArgs(args) {
|
|
136
127
|
return args.source.kind === "asset";
|
|
@@ -145,8 +136,16 @@ function isAssetDeployArgs(args) {
|
|
|
145
136
|
* closure, so the child re-verify over the inert projection matches the gate's
|
|
146
137
|
* freeze.
|
|
147
138
|
*
|
|
148
|
-
* Credential MATERIAL
|
|
149
|
-
*
|
|
139
|
+
* Credential MATERIAL rides ONE `CredentialDelivery` delivered to the child on
|
|
140
|
+
* the frame, unioned from three rails and deduped by credentialId: tool bindings
|
|
141
|
+
* (grant-scoped, resolved here via `buildCredentialDelivery`); every top-level
|
|
142
|
+
* inference source; and every inline body step's inference source. Both inference
|
|
143
|
+
* rails are resolved HERE from the DB under the tenant-ownership authority
|
|
144
|
+
* (`resolveInferenceMaterials`), so the deploy is self-contained -- no caller
|
|
145
|
+
* pre-supplies material, and a spawned body child finds its secret in the cell.
|
|
146
|
+
* The merge is a post-authz union of already-cleared material (tool material is
|
|
147
|
+
* grant-scoped, inference material is tenant-ownership-scoped), never a shared
|
|
148
|
+
* authz check.
|
|
150
149
|
* Credential GRANT enforcement is a SEPARATE layer: the `credential:{id}` /
|
|
151
150
|
* `use` grant the runtime gate checks is minted per-run by run-grant
|
|
152
151
|
* materialization into `runs/<runId>/grants.json`, not carried on this frame --
|
|
@@ -156,14 +155,17 @@ function isAssetDeployArgs(args) {
|
|
|
156
155
|
* A gate outcome that did not approve cannot deploy: an unapproved `approval`
|
|
157
156
|
* fails closed here rather than shipping an unfrozen definition.
|
|
158
157
|
*
|
|
159
|
-
* This
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
158
|
+
* This does the READ-ONLY preparation ONLY: it runs the guards, resolves
|
|
159
|
+
* credential material, pins the body sources, and builds the asset mounts, then
|
|
160
|
+
* returns the frozen definition id and the assembled send args. It emits NO
|
|
161
|
+
* frame and writes NO row, so it has no side effect to unwind. The ordinary path
|
|
162
|
+
* (`deployCodeSourcedWorkflow`) sequences prepare -> INSERT anchor -> emit so
|
|
163
|
+
* the anchor is visible before the frame spawns the child; `emitSourceRefDeployFrame`
|
|
164
|
+
* composes prepare -> emit for the prepared provisioned path, whose anchor row
|
|
165
|
+
* already exists from prepare time. It returns the frozen definition id so each
|
|
166
|
+
* caller writes the same content-addressed identity the gate persisted.
|
|
165
167
|
*/
|
|
166
|
-
async function
|
|
168
|
+
async function prepareSourceRefDeploy(args) {
|
|
167
169
|
const { approval, projection, closure } = args.approved;
|
|
168
170
|
if (!approval.ok) {
|
|
169
171
|
throw new Error(`deployCodeSourcedWorkflow: refusing to deploy an unapproved workflow (gate reason: ${approval.reason})`);
|
|
@@ -224,11 +226,12 @@ async function emitSourceRefDeployFrame(args) {
|
|
|
224
226
|
}
|
|
225
227
|
credentials = delivery.delivery;
|
|
226
228
|
}
|
|
227
|
-
// Pin per-step inference sources for the projection's inline
|
|
229
|
+
// Pin per-step inference sources for the projection's inline trigger bodies
|
|
230
|
+
// -- onTrigger sections and childWorkflow children, enumerated transitively.
|
|
228
231
|
// The hub holds only the frozen inert projection, so it enumerates the inline
|
|
229
232
|
// bodies from the wire form and resolves each body step's source through the
|
|
230
233
|
// same resolver + operator-approval gate the top-level steps use
|
|
231
|
-
// (`pickStepInferenceSource` against `approval.
|
|
234
|
+
// (`pickStepInferenceSource` against `approval.approvedSurface`). Each body's
|
|
232
235
|
// wire hash is recomputed from the inert body verbatim, so a body child's
|
|
233
236
|
// re-verify over the re-evaluated closure clears the same barrier a top-level
|
|
234
237
|
// re-verify does. The pinned sources ride OUTSIDE the hash; their trust comes
|
|
@@ -244,43 +247,74 @@ async function emitSourceRefDeployFrame(args) {
|
|
|
244
247
|
// rather than reading it off disk, so no body workflow.json is staged (see the
|
|
245
248
|
// staging loop in workflow-host-wiring.ts and the anti-fallback guard in
|
|
246
249
|
// workflow-host run-child.ts).
|
|
247
|
-
const referencedDefinitions = await
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
250
|
+
const referencedDefinitions = await buildReferencedWorkflowSourcePins({
|
|
251
|
+
projection,
|
|
252
|
+
config: args.config,
|
|
253
|
+
operatorApprovals: approval.approvedSurface,
|
|
254
|
+
});
|
|
255
|
+
// Assemble the ONE credential delivery. Its `materials` cover three rails, each
|
|
256
|
+
// authorized upstream on its own terms, deduped by credentialId into one cell:
|
|
257
|
+
// - tool bindings, grant-scoped through `buildCredentialDelivery` above;
|
|
258
|
+
// - the inference source pinned to each top-level step that can actually
|
|
259
|
+
// invoke inference, tenant-owned;
|
|
260
|
+
// - the same for each inline body step (onTrigger/childWorkflow bodies
|
|
261
|
+
// pinned above), tenant-owned.
|
|
262
|
+
// Which steps those are is read from the hash-covered projection, never from
|
|
263
|
+
// the pinned map: every step carries a pin because the wire shape demands one,
|
|
264
|
+
// but a step that cannot issue a request has no use for a secret. Delivering
|
|
265
|
+
// one anyway decrypts a tenant credential, seals it to the sidecar, and
|
|
266
|
+
// re-delivers it on every reconnect, on behalf of a step that never makes a
|
|
267
|
+
// call.
|
|
268
|
+
// The inference rails are resolved HERE from the DB under the tenant-ownership
|
|
269
|
+
// authority, so this deploy is self-contained: a direct deploy (a test) that
|
|
270
|
+
// seeds the credentials in the DB -- rather than pre-supplying material -- still
|
|
271
|
+
// fills the cell, and a spawned body finds its secret rather than failing closed
|
|
272
|
+
// at resolve time. Precedence on a shared credentialId is tool material first
|
|
273
|
+
// (grant-scoped), then the inference material (tenant-ownership-scoped): the
|
|
274
|
+
// first material for an id wins. Inference sources carry NO binding descriptor
|
|
275
|
+
// -- they reference their credential by id directly.
|
|
276
|
+
const materials = new Map();
|
|
277
|
+
for (const material of credentials?.materials ?? []) {
|
|
278
|
+
materials.set(material.credentialId, material);
|
|
279
|
+
}
|
|
280
|
+
const inferenceCredentialIds = new Set();
|
|
281
|
+
const addAgentBearingCredentials = (definition, pinned, context) => {
|
|
282
|
+
for (const stepId of collectAgentBearingStepIds({ definition, context })) {
|
|
283
|
+
const stepSources = pinned[stepId];
|
|
284
|
+
if (stepSources === undefined) {
|
|
285
|
+
throw new Error(`${context}step ${stepId} can invoke inference but carries no pinned source`);
|
|
286
|
+
}
|
|
287
|
+
for (const source of stepSources) {
|
|
288
|
+
inferenceCredentialIds.add(source.credentialId);
|
|
267
289
|
}
|
|
268
|
-
sources[bodyStepId] = [
|
|
269
|
-
pickStepInferenceSource({
|
|
270
|
-
preferred,
|
|
271
|
-
stepId: bodyStepId,
|
|
272
|
-
workflowId: body.ref,
|
|
273
|
-
config: args.config,
|
|
274
|
-
operatorApprovals: approval.approvedGrants,
|
|
275
|
-
}),
|
|
276
|
-
];
|
|
277
290
|
}
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
291
|
+
};
|
|
292
|
+
addAgentBearingCredentials(projection, args.sources, "deployCodeSourcedWorkflow: ");
|
|
293
|
+
for (const body of referencedDefinitions) {
|
|
294
|
+
addAgentBearingCredentials(body.definition, body.sources, `deployCodeSourcedWorkflow body ${body.definition.id}: `);
|
|
295
|
+
}
|
|
296
|
+
if (inferenceCredentialIds.size > 0) {
|
|
297
|
+
if (args.credentialCipher === undefined) {
|
|
298
|
+
throw new Error("deployCodeSourcedWorkflow: pinned inference sources reference credentials " +
|
|
299
|
+
"but no credentialCipher was supplied to resolve them");
|
|
300
|
+
}
|
|
301
|
+
const inferenceMaterials = await resolveInferenceMaterials(args.db, args.tenantId, inferenceCredentialIds, args.credentialCipher);
|
|
302
|
+
for (const material of inferenceMaterials) {
|
|
303
|
+
if (!materials.has(material.credentialId)) {
|
|
304
|
+
materials.set(material.credentialId, material);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
// A delivery with tool bindings always carries their material, so an empty map
|
|
309
|
+
// means no rail contributed anything -- send no delivery. (Tool bindings never
|
|
310
|
+
// produce a descriptor without a material, so bindings-without-materials cannot
|
|
311
|
+
// occur.)
|
|
312
|
+
const credentialDelivery = materials.size > 0
|
|
313
|
+
? {
|
|
314
|
+
bindings: credentials?.bindings ?? [],
|
|
315
|
+
materials: [...materials.values()],
|
|
316
|
+
}
|
|
317
|
+
: undefined;
|
|
284
318
|
// An asset-sourced pin's `kind:"asset"` closure entries read from source
|
|
285
319
|
// assets the sidecar cannot fetch itself; deliver them inline on the frame so
|
|
286
320
|
// the sidecar checks them out into its durable per-deployment source store. A
|
|
@@ -288,58 +322,238 @@ async function emitSourceRefDeployFrame(args) {
|
|
|
288
322
|
const assets = isAssetDeployArgs(args)
|
|
289
323
|
? await buildSourceAssetMounts(closure, args.resolveAttachment)
|
|
290
324
|
: [];
|
|
291
|
-
const
|
|
325
|
+
const sendArgs = {
|
|
292
326
|
lineage: "source-ref",
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
? { sidecarAllocationRouter: args.sidecarAllocationRouter }
|
|
296
|
-
: {}),
|
|
297
|
-
...(args.allocationTarget !== undefined
|
|
298
|
-
? { allocationTarget: args.allocationTarget }
|
|
299
|
-
: {}),
|
|
327
|
+
sidecarAllocationRouter: args.sidecarAllocationRouter,
|
|
328
|
+
allocationTarget: args.allocationTarget,
|
|
300
329
|
agentAddress: args.agentAddress,
|
|
301
330
|
config: args.config,
|
|
302
331
|
sources: args.sources,
|
|
303
332
|
approvedWireHash: approval.approvedWireHash,
|
|
304
333
|
sourceRef: { source: args.source, closure },
|
|
305
|
-
...(
|
|
334
|
+
...(credentialDelivery !== undefined
|
|
335
|
+
? { credentials: credentialDelivery }
|
|
336
|
+
: {}),
|
|
306
337
|
...(referencedDefinitions.length > 0 ? { referencedDefinitions } : {}),
|
|
307
338
|
...(assets.length > 0 ? { assets } : {}),
|
|
308
|
-
}
|
|
309
|
-
return {
|
|
339
|
+
};
|
|
340
|
+
return { definitionId: approval.definitionId, sendArgs };
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Prepare then emit the source-ref deploy frame, for the prepared provisioned
|
|
344
|
+
* path whose anchor `workflow_run` row already exists (inserted at prepare
|
|
345
|
+
* time). It emits the frame but does NOT touch the anchor row: the caller
|
|
346
|
+
* (`deployPreparedCodeSourcedWorkflow`) stamps the acked key under the
|
|
347
|
+
* allocation-ownership lock. A tagged `DeployFrameFailure` is converted to the
|
|
348
|
+
* `SessionLaunchError` disposition the allocation reconciler consumes, while
|
|
349
|
+
* untagged preparation errors remain safe same-generation retries. The ordinary
|
|
350
|
+
* path does NOT use this wrapper -- it must interleave the anchor INSERT between
|
|
351
|
+
* prepare and emit, so it drives `prepareSourceRefDeploy` and
|
|
352
|
+
* `sendMultiStepDeployFrame` directly.
|
|
353
|
+
*/
|
|
354
|
+
// The non-secret projection of a delivery, persisted on the anchor run so the
|
|
355
|
+
// reconnect resync can re-resolve current materials. Secrets never land here.
|
|
356
|
+
function credentialRefsFromDelivery(delivery) {
|
|
357
|
+
return {
|
|
358
|
+
credentialIds: delivery.materials.map((material) => material.credentialId),
|
|
359
|
+
bindings: delivery.bindings,
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
async function emitSourceRefDeployFrame(args, reconciliation) {
|
|
363
|
+
const { signal, leaseId } = reconciliation;
|
|
364
|
+
signal.throwIfAborted();
|
|
365
|
+
const { definitionId, sendArgs } = await prepareSourceRefDeploy(args);
|
|
366
|
+
signal.throwIfAborted();
|
|
367
|
+
const allocationStore = createSidecarAllocationStore(args.db);
|
|
368
|
+
const initialization = {
|
|
369
|
+
allocationId: sendArgs.allocationTarget.allocationId,
|
|
370
|
+
generation: sendArgs.allocationTarget.generation,
|
|
371
|
+
anchorRunId: args.anchorRunId,
|
|
372
|
+
tenantId: args.tenantId,
|
|
373
|
+
leaseId,
|
|
374
|
+
signal,
|
|
375
|
+
};
|
|
376
|
+
let previousPublicKey;
|
|
377
|
+
try {
|
|
378
|
+
const result = await sendMultiStepDeployFrame(sendArgs, signal, async () => {
|
|
379
|
+
const reserved = await allocationStore.beginInitialization(initialization);
|
|
380
|
+
if (reserved === null) {
|
|
381
|
+
throw new Error("Allocation no longer permits this initialization attempt");
|
|
382
|
+
}
|
|
383
|
+
previousPublicKey = reserved.previousPublicKey;
|
|
384
|
+
});
|
|
385
|
+
return {
|
|
386
|
+
publicKey: result.publicKey,
|
|
387
|
+
definitionId,
|
|
388
|
+
...(sendArgs.credentials !== undefined
|
|
389
|
+
? { credentialRefs: credentialRefsFromDelivery(sendArgs.credentials) }
|
|
390
|
+
: {}),
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
catch (cause) {
|
|
394
|
+
if (!isDeployFrameFailure(cause))
|
|
395
|
+
throw cause;
|
|
396
|
+
// Only a confirmed reservation gives us the key to restore. An ambiguous
|
|
397
|
+
// reservation response leaves its durable marker for conservative cleanup.
|
|
398
|
+
// A confirmed unsent attempt can roll back even after lease cancellation.
|
|
399
|
+
if (!cause.frameSent && previousPublicKey !== undefined) {
|
|
400
|
+
try {
|
|
401
|
+
const cleared = await allocationStore.clearUnsentInitialization({
|
|
402
|
+
...initialization,
|
|
403
|
+
previousPublicKey,
|
|
404
|
+
});
|
|
405
|
+
if (cleared)
|
|
406
|
+
args.onUnsentInitializationCleared(previousPublicKey);
|
|
407
|
+
}
|
|
408
|
+
catch (error) {
|
|
409
|
+
logger.warn `Could not clear unsent initialization for ${initialization.allocationId}: ${error instanceof Error ? error.message : String(error)}`;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
throw new SessionLaunchError("start", cause, cause.frameSent);
|
|
413
|
+
}
|
|
310
414
|
}
|
|
311
415
|
/**
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
* routing address and public key. Run-grant materialization keys
|
|
316
|
-
* (address + live status), so WITHOUT it no per-run grants (tool,
|
|
317
|
-
* credential) ever materialize for a source-ref deployment. Born
|
|
318
|
-
* (live but pre-trigger): the first trigger's
|
|
319
|
-
* "running" via `anchorWithPrincipal`'s guarded
|
|
320
|
-
* "running" would skip. Its `anchorRunId` equals its
|
|
321
|
-
* references itself. The deployer read grant is deferred
|
|
322
|
-
* route, which carries the authenticated deployer principal
|
|
323
|
-
*
|
|
416
|
+
* A direct allocation-bound composition entrypoint for tests and low-level
|
|
417
|
+
* callers: prepare, INSERT the deployment's anchor `workflow_run` row, THEN emit
|
|
418
|
+
* the source-ref frame. The anchor row is the deployment's first-class record
|
|
419
|
+
* that owns its routing address and public key. Run-grant materialization keys
|
|
420
|
+
* off this row (address + live status), so WITHOUT it no per-run grants (tool,
|
|
421
|
+
* capability, OR credential) ever materialize for a source-ref deployment. Born
|
|
422
|
+
* "deployed" (live but pre-trigger) with a null public key: the first trigger's
|
|
423
|
+
* materialization flips it to "running" via `anchorWithPrincipal`'s guarded
|
|
424
|
+
* update, which a row born "running" would skip. Its `anchorRunId` equals its
|
|
425
|
+
* own id, so the anchor references itself. The deployer read grant is deferred
|
|
426
|
+
* to the production route, which carries the authenticated deployer principal.
|
|
427
|
+
*
|
|
428
|
+
* ORDERING IS LOAD-BEARING. The anchor row must be committed and visible to the
|
|
429
|
+
* pack-receipt connection BEFORE the frame reaches the wire: the frame spawns
|
|
430
|
+
* the child, whose first events pack races the ack back, and
|
|
431
|
+
* `receiveWorkflowRunPack` fails closed on a missing live anchor. Emitting first
|
|
432
|
+
* (the previous order) rejected that first pack and never bootstrapped the log.
|
|
433
|
+
* This works because `args.db` is the autocommit handle (`DB["db"]`, which the
|
|
434
|
+
* type forbids from being a transaction) and the INSERT is NOT wrapped in a
|
|
435
|
+
* transaction with the emit -- so the row is durably visible the instant the
|
|
436
|
+
* INSERT statement returns. Do NOT relax `db` to a transaction executor or wrap
|
|
437
|
+
* anchor+emit in one transaction to make them atomic: that reopens the race.
|
|
324
438
|
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
439
|
+
* On emit failure the anchor row is rolled back or fenced by the `frameSent`
|
|
440
|
+
* evidence from the transport. `leakedAgent: false` (safe to fully roll back) is
|
|
441
|
+
* the STRONG claim and is made only on positive proof the frame never reached
|
|
442
|
+
* the wire (`isDeployFrameFailure && frameSent === false`); every other failure
|
|
443
|
+
* -- a sent-but-unacked frame OR any untagged error -- is treated as
|
|
444
|
+
* possibly-live: the anchor is fenced `deployed` -> `failed` and the error is
|
|
445
|
+
* `leakedAgent: true`.
|
|
446
|
+
*
|
|
447
|
+
* The prepared provisioned path does NOT use this composition: its anchor row
|
|
448
|
+
* already exists from prepare time, so it drives `emitSourceRefDeployFrame` and
|
|
449
|
+
* an UPDATE-under-allocation-lock instead.
|
|
328
450
|
*/
|
|
329
451
|
export async function deployCodeSourcedWorkflow(args) {
|
|
330
|
-
const {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
452
|
+
const { definitionId, sendArgs } = await prepareSourceRefDeploy(args);
|
|
453
|
+
// INSERT the anchor before the frame. A collision or DB error here spawned
|
|
454
|
+
// nothing (no frame went out), so it is a clean, non-leaking failure.
|
|
455
|
+
try {
|
|
456
|
+
await args.db.insert(workflowRunTable).values({
|
|
457
|
+
id: args.anchorRunId,
|
|
458
|
+
tenantId: args.tenantId,
|
|
459
|
+
anchorRunId: args.anchorRunId,
|
|
460
|
+
definitionId,
|
|
461
|
+
address: args.agentAddress,
|
|
462
|
+
publicKey: null,
|
|
463
|
+
status: "deployed",
|
|
464
|
+
createdAt: new Date(),
|
|
465
|
+
// Persist the non-secret shape of the delivery so the reconnect resync
|
|
466
|
+
// can re-resolve current materials for these ids. Secrets never land here.
|
|
467
|
+
...(sendArgs.credentials !== undefined
|
|
468
|
+
? { credentialRefs: credentialRefsFromDelivery(sendArgs.credentials) }
|
|
469
|
+
: {}),
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
catch (cause) {
|
|
473
|
+
throw new SessionLaunchError("start", cause, false);
|
|
474
|
+
}
|
|
475
|
+
let publicKey;
|
|
476
|
+
try {
|
|
477
|
+
const result = await sendMultiStepDeployFrame(sendArgs);
|
|
478
|
+
publicKey = result.publicKey;
|
|
479
|
+
}
|
|
480
|
+
catch (cause) {
|
|
481
|
+
if (isDeployFrameFailure(cause) && cause.frameSent === false) {
|
|
482
|
+
// Positive proof the frame never reached the wire: nothing spawned, so
|
|
483
|
+
// fully roll the anchor back. The guard (`deployed`, null key) is a
|
|
484
|
+
// tripwire on the `frameSent: false` contract -- a 0-row delete means the
|
|
485
|
+
// row advanced or vanished, so the contract lied and a child may be live;
|
|
486
|
+
// surface that loudly and refuse to claim it is safe to roll back.
|
|
487
|
+
const deleted = await args.db
|
|
488
|
+
.delete(workflowRunTable)
|
|
489
|
+
.where(and(eq(workflowRunTable.id, args.anchorRunId), eq(workflowRunTable.anchorRunId, args.anchorRunId), eq(workflowRunTable.tenantId, args.tenantId), eq(workflowRunTable.status, "deployed"), isNull(workflowRunTable.publicKey)))
|
|
490
|
+
.returning({ id: workflowRunTable.id });
|
|
491
|
+
if (deleted.length === 0) {
|
|
492
|
+
logger.error `anchor-before-frame rollback found no deployed/null-key row for ${args.anchorRunId} after a frameSent:false failure; the never-sent contract was violated and a child may be live`;
|
|
493
|
+
throw new SessionLaunchError("start", cause, true);
|
|
494
|
+
}
|
|
495
|
+
throw new SessionLaunchError("start", cause, false);
|
|
496
|
+
}
|
|
497
|
+
// A sent-but-unacked frame, OR any untagged/unexpected error: no positive
|
|
498
|
+
// proof of a clean send, so treat the agent as possibly-live. Fence the
|
|
499
|
+
// anchor `deployed` -> `failed` (guarded so a self-flip to "running" by a
|
|
500
|
+
// trigger that already landed is left alone). Do NOT delete: a live child
|
|
501
|
+
// needs the anchor to bootstrap.
|
|
502
|
+
const flipped = await args.db
|
|
503
|
+
.update(workflowRunTable)
|
|
504
|
+
.set({ status: "failed" })
|
|
505
|
+
.where(and(eq(workflowRunTable.id, args.anchorRunId), eq(workflowRunTable.anchorRunId, args.anchorRunId), eq(workflowRunTable.tenantId, args.tenantId), eq(workflowRunTable.status, "deployed"), isNull(workflowRunTable.publicKey)))
|
|
506
|
+
.returning({ id: workflowRunTable.id });
|
|
507
|
+
if (flipped.length === 0) {
|
|
508
|
+
// The anchor already advanced past deployed -- a trigger flipped it to
|
|
509
|
+
// "running", so the deploy actually succeeded and the run is progressing
|
|
510
|
+
// despite the ack failure. Leave it; the leaked-agent disposition still
|
|
511
|
+
// holds because the frame was (or may have been) sent.
|
|
512
|
+
logger.warn `anchor-before-frame: anchor ${args.anchorRunId} already advanced past deployed on an unacked/failed emit; the agent is live and the run is progressing despite the ack failure`;
|
|
513
|
+
}
|
|
514
|
+
else {
|
|
515
|
+
logger.warn `anchor-before-frame: fenced anchor ${args.anchorRunId} deployed->failed on an unacked/failed emit; the agent may be leaked but the run is dead`;
|
|
516
|
+
}
|
|
517
|
+
throw new SessionLaunchError("start", cause, true);
|
|
518
|
+
}
|
|
519
|
+
// Emit succeeded: stamp the acked key. No status guard -- the key is a fact
|
|
520
|
+
// regardless of whether the pack-ack race already flipped the row to
|
|
521
|
+
// "running", and skipping the stamp there would strand a live run with a null
|
|
522
|
+
// key. A 0-row update is an anomaly (nothing should remove a deployed anchor
|
|
523
|
+
// on the success path), but the deploy succeeded, so log it rather than
|
|
524
|
+
// failing a live run.
|
|
525
|
+
const stamped = await args.db
|
|
526
|
+
.update(workflowRunTable)
|
|
527
|
+
.set({ publicKey })
|
|
528
|
+
.where(and(eq(workflowRunTable.id, args.anchorRunId), eq(workflowRunTable.anchorRunId, args.anchorRunId), eq(workflowRunTable.tenantId, args.tenantId)))
|
|
529
|
+
.returning({ id: workflowRunTable.id });
|
|
530
|
+
if (stamped.length === 0) {
|
|
531
|
+
logger.error `anchor-before-frame: anchor ${args.anchorRunId} vanished before its public key could be stamped on a successful deploy`;
|
|
532
|
+
}
|
|
341
533
|
return { publicKey };
|
|
342
534
|
}
|
|
535
|
+
/** Resolve deferred sender mail after claiming the previous initializer's lease. */
|
|
536
|
+
export async function recoverSenderDeploy(args) {
|
|
537
|
+
const { allocation, reconciliation } = args;
|
|
538
|
+
reconciliation.signal.throwIfAborted();
|
|
539
|
+
// A proven-unsent clear may still restore the previous key after this claim.
|
|
540
|
+
// Do not fail its mail using the claim's stale marker. Cleanup rechecks under
|
|
541
|
+
// the allocation lock; an advanced fence settles failure, while a rolled-back
|
|
542
|
+
// attempt is resolved by its caller or the next claim's completed key.
|
|
543
|
+
if (allocation.initializationLeaseId !== undefined)
|
|
544
|
+
return;
|
|
545
|
+
// Claiming the lease prevents the previous attempt from publishing. Its
|
|
546
|
+
// marker and key now distinguish a committed initialization from failure,
|
|
547
|
+
// even before the worker reconnects or the old response arrives.
|
|
548
|
+
const anchor = await args.db.query.workflowRun.findFirst({
|
|
549
|
+
where: eq(workflowRunTable.id, allocation.anchorRunId),
|
|
550
|
+
columns: { publicKey: true },
|
|
551
|
+
});
|
|
552
|
+
reconciliation.signal.throwIfAborted();
|
|
553
|
+
args.sidecarRouter.noteSenderDeploySettled({ allocationId: allocation.id, generation: allocation.generation }, anchor !== undefined && anchor.publicKey !== null
|
|
554
|
+
? { recorded: anchor.publicKey }
|
|
555
|
+
: { failed: "Previous deployment initialization did not complete" });
|
|
556
|
+
}
|
|
343
557
|
export function createSessionService(deps) {
|
|
344
558
|
const { sidecarRouter, sidecarAllocationRouter, agentRepoStore, assetService, db, toolPackageRegistries, } = deps;
|
|
345
559
|
if (assetService !== undefined && db === undefined) {
|
|
@@ -350,7 +564,7 @@ export function createSessionService(deps) {
|
|
|
350
564
|
}
|
|
351
565
|
function requireAllocationRouter() {
|
|
352
566
|
if (sidecarAllocationRouter === undefined) {
|
|
353
|
-
throw new Error("
|
|
567
|
+
throw new Error("Provisioned deployment routing is not configured");
|
|
354
568
|
}
|
|
355
569
|
return sidecarAllocationRouter;
|
|
356
570
|
}
|
|
@@ -455,12 +669,7 @@ export function createSessionService(deps) {
|
|
|
455
669
|
// route is held only for the pack window and dropped in the `finally`.
|
|
456
670
|
if (stageOnly) {
|
|
457
671
|
try {
|
|
458
|
-
|
|
459
|
-
sidecarRouter.bindStepRoute(agentAddress);
|
|
460
|
-
}
|
|
461
|
-
else {
|
|
462
|
-
await requireAllocationRouter().bindAllocatedStepRoute(params.allocationTarget, agentAddress);
|
|
463
|
-
}
|
|
672
|
+
await requireAllocationRouter().bindAllocatedStepRoute(params.allocationTarget, agentAddress);
|
|
464
673
|
}
|
|
465
674
|
catch (err) {
|
|
466
675
|
throw new SessionLaunchError("provision", err, false);
|
|
@@ -474,12 +683,7 @@ export function createSessionService(deps) {
|
|
|
474
683
|
// exist before the pack applies.
|
|
475
684
|
try {
|
|
476
685
|
if (stageOnly) {
|
|
477
|
-
|
|
478
|
-
await sidecarRouter.sendProvisionStep(agentAddress, config);
|
|
479
|
-
}
|
|
480
|
-
else {
|
|
481
|
-
await requireAllocationRouter().sendProvisionStepToAllocation(params.allocationTarget, agentAddress, config);
|
|
482
|
-
}
|
|
686
|
+
await requireAllocationRouter().sendProvisionStepToAllocation(params.allocationTarget, agentAddress, config);
|
|
483
687
|
}
|
|
484
688
|
else {
|
|
485
689
|
// Every caller supplies `stageOnly`. A deploy without it has no
|
|
@@ -500,26 +704,17 @@ export function createSessionService(deps) {
|
|
|
500
704
|
// the orphaned repo. This is an acceptable minor leak on the exceptional
|
|
501
705
|
// staging-failure path, not a live-path cost.
|
|
502
706
|
try {
|
|
503
|
-
|
|
504
|
-
await sidecarRouter.sendPack(agentAddress, pack, ref, commitSha);
|
|
505
|
-
}
|
|
506
|
-
else {
|
|
507
|
-
await requireAllocationRouter().sendPackToAllocation(params.allocationTarget, agentAddress, pack, ref, commitSha);
|
|
508
|
-
}
|
|
707
|
+
await requireAllocationRouter().sendPackToAllocation(params.allocationTarget, agentAddress, pack, ref, commitSha);
|
|
509
708
|
}
|
|
510
709
|
catch (err) {
|
|
511
|
-
|
|
512
|
-
await attemptCleanup(agentAddress, "pack", err);
|
|
513
|
-
}
|
|
514
|
-
throw new SessionLaunchError("pack", err, !stageOnly && params.allocationTarget !== undefined);
|
|
710
|
+
throw new SessionLaunchError("pack", err, !stageOnly);
|
|
515
711
|
}
|
|
516
712
|
// Phase 2b: Asset-pack fan-out. For each attached asset, build a
|
|
517
713
|
// pack, reserve the manifest row, then send the pack. The manifest
|
|
518
714
|
// reservation MUST happen before the pack send: if the sidecar acks
|
|
519
715
|
// but the row is missing, the session has materialization without
|
|
520
|
-
// a recorded manifest. An allocated replacement may reuse the exact
|
|
521
|
-
//
|
|
522
|
-
// row. If reservation fails, the pack send must not happen.
|
|
716
|
+
// a recorded manifest. An allocated replacement may reuse the exact row
|
|
717
|
+
// its predecessor recorded. If reservation fails, no pack is sent.
|
|
523
718
|
//
|
|
524
719
|
// The fan-out materializes the package-registry assets the
|
|
525
720
|
// tool-package resolver picked. They live behind tenant
|
|
@@ -528,35 +723,19 @@ export function createSessionService(deps) {
|
|
|
528
723
|
// `manifestAssetAttachments`.
|
|
529
724
|
const fanOut = manifestAssetAttachments;
|
|
530
725
|
if (assetService !== undefined && fanOut.length > 0) {
|
|
531
|
-
// Track the rows this attempt owns so a later fan-out failure can roll
|
|
532
|
-
// them back in lockstep with the sidecar undeploy. Allocated rows are
|
|
533
|
-
// durable recovery intent, not attempt-owned materialization state, so
|
|
534
|
-
// replacement failures must leave them in place for the next worker.
|
|
535
|
-
const committed = [];
|
|
536
726
|
for (const att of fanOut) {
|
|
537
727
|
try {
|
|
538
|
-
|
|
539
|
-
if (committedRecord !== null)
|
|
540
|
-
committed.push(committedRecord);
|
|
728
|
+
await sendAttachmentPack(runId, agentAddress, att, params.allocationTarget);
|
|
541
729
|
}
|
|
542
730
|
catch (err) {
|
|
543
|
-
|
|
544
|
-
if (!stageOnly && params.allocationTarget === undefined) {
|
|
545
|
-
await attemptCleanup(agentAddress, "pack", err);
|
|
546
|
-
}
|
|
547
|
-
throw new SessionLaunchError("pack", err, !stageOnly && params.allocationTarget !== undefined);
|
|
731
|
+
throw new SessionLaunchError("pack", err, !stageOnly);
|
|
548
732
|
}
|
|
549
733
|
}
|
|
550
734
|
}
|
|
551
735
|
}
|
|
552
736
|
finally {
|
|
553
737
|
if (stageOnly) {
|
|
554
|
-
|
|
555
|
-
sidecarRouter.unbindStepRoute(agentAddress);
|
|
556
|
-
}
|
|
557
|
-
else {
|
|
558
|
-
requireAllocationRouter().unbindAllocatedStepRoute(params.allocationTarget, agentAddress);
|
|
559
|
-
}
|
|
738
|
+
requireAllocationRouter().unbindAllocatedStepRoute(params.allocationTarget, agentAddress);
|
|
560
739
|
}
|
|
561
740
|
}
|
|
562
741
|
}
|
|
@@ -581,9 +760,7 @@ export function createSessionService(deps) {
|
|
|
581
760
|
...(params.toolPackagePins !== undefined
|
|
582
761
|
? { toolPackagePins: params.toolPackagePins }
|
|
583
762
|
: {}),
|
|
584
|
-
|
|
585
|
-
? { allocationTarget: params.allocationTarget }
|
|
586
|
-
: {}),
|
|
763
|
+
allocationTarget: params.allocationTarget,
|
|
587
764
|
});
|
|
588
765
|
}
|
|
589
766
|
// Resolve the npm registry config a code-sourced install resolves external
|
|
@@ -630,11 +807,17 @@ export function createSessionService(deps) {
|
|
|
630
807
|
throw new Error("deployWorkflowFromSource requires a db handle to freeze the approval");
|
|
631
808
|
}
|
|
632
809
|
const dbHandle = db;
|
|
810
|
+
const allocationTarget = params.allocationTarget;
|
|
633
811
|
const common = {
|
|
634
812
|
entry: params.entry,
|
|
635
813
|
assetId: params.definitionAssetId,
|
|
636
|
-
approvals: {
|
|
637
|
-
router:
|
|
814
|
+
approvals: { kind: "approve-probed" },
|
|
815
|
+
router: {
|
|
816
|
+
sendProbe: (args) => requireAllocationRouter().sendProbeToAllocation(allocationTarget, args),
|
|
817
|
+
},
|
|
818
|
+
...(params.onProbeResult !== undefined
|
|
819
|
+
? { onProbeResult: params.onProbeResult }
|
|
820
|
+
: {}),
|
|
638
821
|
db: dbHandle,
|
|
639
822
|
};
|
|
640
823
|
const source = params.source;
|
|
@@ -703,17 +886,17 @@ export function createSessionService(deps) {
|
|
|
703
886
|
// Install + probe + gate + freeze a code-sourced definition, returning the
|
|
704
887
|
// frozen bundle and the (asset-only) attachment resolver. The gate outcome is
|
|
705
888
|
// NOT asserted here: `deployWorkflowFromSource` and `installAndApproveWorkflowSource`
|
|
706
|
-
// each surface a non-approval as their own domain error. This is the
|
|
707
|
-
// freeze
|
|
889
|
+
// each surface a non-approval as their own domain error. This is the common
|
|
890
|
+
// freeze used by direct tests and provisioned prepare runs.
|
|
708
891
|
async function prepareCodeSourcedApproval(params) {
|
|
709
892
|
const resolveAttachment = bindSourceAttachmentResolver(params.source);
|
|
710
893
|
const installArgs = await buildInstallArgs(params, resolveAttachment);
|
|
711
894
|
const approved = await installAndApproveWorkflowDefinition(installArgs);
|
|
712
895
|
return { approved, resolveAttachment };
|
|
713
896
|
}
|
|
714
|
-
// Freeze a code-sourced approval
|
|
715
|
-
//
|
|
716
|
-
//
|
|
897
|
+
// Freeze a code-sourced approval WITHOUT deploying it. The provisioned
|
|
898
|
+
// prepare path persists the returned bundle and deploys it to an allocation
|
|
899
|
+
// later. A non-approval fails closed as an invalid
|
|
717
900
|
// definition.
|
|
718
901
|
async function installAndApproveWorkflowSource(params) {
|
|
719
902
|
const { approved } = await prepareCodeSourcedApproval(params);
|
|
@@ -722,79 +905,12 @@ export function createSessionService(deps) {
|
|
|
722
905
|
}
|
|
723
906
|
return approved;
|
|
724
907
|
}
|
|
725
|
-
async function deployWorkflowFromSource(params) {
|
|
726
|
-
if (db === undefined) {
|
|
727
|
-
throw new Error("deployWorkflowFromSource requires a db handle to record the deployment's anchor run");
|
|
728
|
-
}
|
|
729
|
-
const source = params.source;
|
|
730
|
-
const { approved, resolveAttachment } = await prepareCodeSourcedApproval(params);
|
|
731
|
-
if (!approved.approval.ok) {
|
|
732
|
-
throw new WorkflowDefinitionInvalidError(approved.projection.id, `code-sourced workflow install did not approve (reason: ${approved.approval.reason})`);
|
|
733
|
-
}
|
|
734
|
-
// Pin every top-level step's inference source under the frozen approval,
|
|
735
|
-
// then hand the frozen bundle to the source-ref deploy.
|
|
736
|
-
const sources = buildInertProjectionStepSources({
|
|
737
|
-
projection: approved.projection,
|
|
738
|
-
config: params.config,
|
|
739
|
-
operatorApprovals: approved.approval.approvedGrants,
|
|
740
|
-
});
|
|
741
|
-
const commonDeploy = {
|
|
742
|
-
approved,
|
|
743
|
-
sidecarRouter,
|
|
744
|
-
agentAddress: params.agentAddress,
|
|
745
|
-
config: params.config,
|
|
746
|
-
sources,
|
|
747
|
-
db,
|
|
748
|
-
tenantId: params.tenantId,
|
|
749
|
-
anchorRunId: params.anchorRunId,
|
|
750
|
-
deploymentDomain: params.deploymentDomain,
|
|
751
|
-
};
|
|
752
|
-
// Branch on the source discriminant so the deploy args match the
|
|
753
|
-
// asset/registry arms of `DeployCodeSourcedWorkflowArgs`: an asset arm
|
|
754
|
-
// carries the attachment resolver (asserted non-null here to satisfy the
|
|
755
|
-
// union and fail loud on a mis-wired caller), a registry arm carries none.
|
|
756
|
-
let result;
|
|
757
|
-
if (source.kind === "asset") {
|
|
758
|
-
if (resolveAttachment === null) {
|
|
759
|
-
throw new Error("deployWorkflowFromSource: asset source deploy is missing its attachment resolver");
|
|
760
|
-
}
|
|
761
|
-
result = await deployCodeSourcedWorkflow({
|
|
762
|
-
...commonDeploy,
|
|
763
|
-
source,
|
|
764
|
-
resolveAttachment,
|
|
765
|
-
});
|
|
766
|
-
}
|
|
767
|
-
else {
|
|
768
|
-
result = await deployCodeSourcedWorkflow({ ...commonDeploy, source });
|
|
769
|
-
}
|
|
770
|
-
// Seed the deploying principal's read grant on the deployment's workflow-run
|
|
771
|
-
// resource. `deployCodeSourcedWorkflow` wrote the anchor row but deliberately
|
|
772
|
-
// leaves this grant to the route, which carries the authenticated deployer
|
|
773
|
-
// principal.
|
|
774
|
-
const now = new Date();
|
|
775
|
-
await db.insert(grantTable).values({
|
|
776
|
-
id: generateId("grant"),
|
|
777
|
-
tenantId: params.tenantId,
|
|
778
|
-
principalId: params.config.principalId,
|
|
779
|
-
resource: `workflow-run:${params.anchorRunId}`,
|
|
780
|
-
action: "read",
|
|
781
|
-
effect: "allow",
|
|
782
|
-
origin: "creator",
|
|
783
|
-
createdAt: now,
|
|
784
|
-
updatedAt: now,
|
|
785
|
-
});
|
|
786
|
-
return {
|
|
787
|
-
anchorRunId: params.anchorRunId,
|
|
788
|
-
deploymentAddress: params.agentAddress,
|
|
789
|
-
publicKey: result.publicKey,
|
|
790
|
-
};
|
|
791
|
-
}
|
|
792
908
|
/**
|
|
793
909
|
* Update a prepared anchor run's `publicKey` under the allocation-ownership
|
|
794
910
|
* lock. The anchor row was inserted at prepare time; this stamps the
|
|
795
911
|
* supervisor key returned by the deploy ack, but only while the allocation
|
|
796
|
-
* still names this exact accepted generation
|
|
797
|
-
*
|
|
912
|
+
* still names this exact accepted generation and unexpired reconciliation
|
|
913
|
+
* lease for this anchor. Lost ownership or cancellation fails closed as a
|
|
798
914
|
* leaked-agent `SessionLaunchError` -- the deploy already reached the sidecar,
|
|
799
915
|
* so the caller must treat the sidecar agent as possibly live. Used by the
|
|
800
916
|
* `deployPreparedCodeSourcedWorkflow` prepared path.
|
|
@@ -805,35 +921,19 @@ export function createSessionService(deps) {
|
|
|
805
921
|
}
|
|
806
922
|
const dbHandle = db;
|
|
807
923
|
try {
|
|
808
|
-
const updated = await dbHandle.
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
.
|
|
818
|
-
|
|
819
|
-
.limit(1)
|
|
820
|
-
.for("update");
|
|
821
|
-
if (allocation === undefined ||
|
|
822
|
-
allocation.anchorRunId !== args.anchorRunId ||
|
|
823
|
-
allocation.status !== "allocated" ||
|
|
824
|
-
allocation.generation !== args.allocationTarget.generation ||
|
|
825
|
-
allocation.ensureAcceptedGeneration !==
|
|
826
|
-
args.allocationTarget.generation) {
|
|
827
|
-
return null;
|
|
828
|
-
}
|
|
829
|
-
const [anchor] = await tx
|
|
830
|
-
.update(workflowRunTable)
|
|
831
|
-
.set({ publicKey: args.publicKey })
|
|
832
|
-
.where(and(eq(workflowRunTable.id, args.anchorRunId), eq(workflowRunTable.anchorRunId, args.anchorRunId), eq(workflowRunTable.tenantId, args.tenantId)))
|
|
833
|
-
.returning({ id: workflowRunTable.id });
|
|
834
|
-
return anchor ?? null;
|
|
924
|
+
const updated = await createSidecarAllocationStore(dbHandle).completeInitialization({
|
|
925
|
+
allocationId: args.allocationTarget.allocationId,
|
|
926
|
+
generation: args.allocationTarget.generation,
|
|
927
|
+
anchorRunId: args.anchorRunId,
|
|
928
|
+
tenantId: args.tenantId,
|
|
929
|
+
leaseId: args.reconciliation.leaseId,
|
|
930
|
+
signal: args.reconciliation.signal,
|
|
931
|
+
publicKey: args.publicKey,
|
|
932
|
+
...(args.credentialRefs !== undefined
|
|
933
|
+
? { credentialRefs: args.credentialRefs }
|
|
934
|
+
: {}),
|
|
835
935
|
});
|
|
836
|
-
if (updated
|
|
936
|
+
if (!updated) {
|
|
837
937
|
throw new Error(`Prepared anchor run ${args.anchorRunId} lost allocation ownership before initialization completed`);
|
|
838
938
|
}
|
|
839
939
|
}
|
|
@@ -852,6 +952,8 @@ export function createSessionService(deps) {
|
|
|
852
952
|
* resolved source carries a credential secret).
|
|
853
953
|
*/
|
|
854
954
|
async function deployPreparedCodeSourcedWorkflow(params) {
|
|
955
|
+
const { signal } = params.reconciliation;
|
|
956
|
+
signal.throwIfAborted();
|
|
855
957
|
if (db === undefined) {
|
|
856
958
|
throw new Error("deployPreparedCodeSourcedWorkflow requires a db handle to update the prepared anchor run");
|
|
857
959
|
}
|
|
@@ -864,11 +966,11 @@ export function createSessionService(deps) {
|
|
|
864
966
|
const source = params.source;
|
|
865
967
|
const resolveAttachment = bindSourceAttachmentResolver(source);
|
|
866
968
|
// Re-pin every top-level step's inference source from the re-resolved chain
|
|
867
|
-
// under the frozen approval -- the same pin the
|
|
969
|
+
// under the frozen approval -- the same pin the source-ref deploy computes.
|
|
868
970
|
const sources = buildInertProjectionStepSources({
|
|
869
971
|
projection: params.approved.projection,
|
|
870
972
|
config: params.config,
|
|
871
|
-
operatorApprovals: approval.
|
|
973
|
+
operatorApprovals: approval.approvedSurface,
|
|
872
974
|
});
|
|
873
975
|
// Restore the Hub-authoritative run ref onto the exact allocation generation
|
|
874
976
|
// before its address is routed.
|
|
@@ -877,10 +979,15 @@ export function createSessionService(deps) {
|
|
|
877
979
|
allocationRouter,
|
|
878
980
|
allocationTarget: params.allocationTarget,
|
|
879
981
|
agentAddress: params.agentAddress,
|
|
982
|
+
signal,
|
|
880
983
|
});
|
|
984
|
+
signal.throwIfAborted();
|
|
985
|
+
let restoredPublicKey;
|
|
881
986
|
const commonEmit = {
|
|
987
|
+
onUnsentInitializationCleared(publicKey) {
|
|
988
|
+
restoredPublicKey = publicKey;
|
|
989
|
+
},
|
|
882
990
|
approved: params.approved,
|
|
883
|
-
sidecarRouter,
|
|
884
991
|
sidecarAllocationRouter: allocationRouter,
|
|
885
992
|
allocationTarget: params.allocationTarget,
|
|
886
993
|
agentAddress: params.agentAddress,
|
|
@@ -898,48 +1005,68 @@ export function createSessionService(deps) {
|
|
|
898
1005
|
// arms: an asset arm carries the rebuilt attachment resolver (asserted
|
|
899
1006
|
// non-null to satisfy the union), a registry arm carries none.
|
|
900
1007
|
let result;
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
1008
|
+
const senderAttempt = {
|
|
1009
|
+
...params.allocationTarget,
|
|
1010
|
+
leaseId: params.reconciliation.leaseId,
|
|
1011
|
+
};
|
|
1012
|
+
signal.throwIfAborted();
|
|
1013
|
+
sidecarRouter.noteSenderDeployStarted(params.agentAddress, senderAttempt);
|
|
1014
|
+
try {
|
|
1015
|
+
// Bracket the allocated pre-ack window: mark the sender's key-record as
|
|
1016
|
+
// mid-flight before the deploy emit so a run that sends mail before its
|
|
1017
|
+
// anchor key is committed parks rather than delivering keyless. The settle
|
|
1018
|
+
// follows durable completion; cancellation leaves the outcome to recovery.
|
|
1019
|
+
if (source.kind === "asset") {
|
|
1020
|
+
if (resolveAttachment === null) {
|
|
1021
|
+
throw new Error("deployPreparedCodeSourcedWorkflow: asset source deploy is missing its attachment resolver");
|
|
1022
|
+
}
|
|
1023
|
+
result = await emitSourceRefDeployFrame({ ...commonEmit, source, resolveAttachment }, params.reconciliation);
|
|
904
1024
|
}
|
|
905
|
-
|
|
906
|
-
...commonEmit,
|
|
907
|
-
|
|
908
|
-
|
|
1025
|
+
else {
|
|
1026
|
+
result = await emitSourceRefDeployFrame({ ...commonEmit, source }, params.reconciliation);
|
|
1027
|
+
}
|
|
1028
|
+
await updateAnchorPublicKeyUnderAllocationLock({
|
|
1029
|
+
tenantId: params.tenantId,
|
|
1030
|
+
anchorRunId: params.anchorRunId,
|
|
1031
|
+
allocationTarget: params.allocationTarget,
|
|
1032
|
+
reconciliation: params.reconciliation,
|
|
1033
|
+
publicKey: result.publicKey,
|
|
1034
|
+
...(result.credentialRefs !== undefined
|
|
1035
|
+
? { credentialRefs: result.credentialRefs }
|
|
1036
|
+
: {}),
|
|
909
1037
|
});
|
|
1038
|
+
// The anchor's public key is now durable. Wake any mail the run parked
|
|
1039
|
+
// while pre-ack so it delivers with the sender key co-delivered, closing
|
|
1040
|
+
// the window where a run sends before its key is recorded. The write above
|
|
1041
|
+
// happens-before this settle, so a re-drive resolves the recorded key.
|
|
1042
|
+
// `params.agentAddress` is the run's deploy address, byte-identical to the
|
|
1043
|
+
// sender address its mail was sent under (asserted against the anchor at
|
|
1044
|
+
// deploy time), so a settle matches the parked entries.
|
|
1045
|
+
sidecarRouter.noteSenderDeploySettled(senderAttempt, {
|
|
1046
|
+
recorded: result.publicKey,
|
|
1047
|
+
});
|
|
1048
|
+
return {
|
|
1049
|
+
anchorRunId: params.anchorRunId,
|
|
1050
|
+
deploymentAddress: params.agentAddress,
|
|
1051
|
+
publicKey: result.publicKey,
|
|
1052
|
+
};
|
|
910
1053
|
}
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
anchorRunId: params.anchorRunId,
|
|
917
|
-
allocationTarget: params.allocationTarget,
|
|
918
|
-
publicKey: result.publicKey,
|
|
919
|
-
});
|
|
920
|
-
return {
|
|
921
|
-
anchorRunId: params.anchorRunId,
|
|
922
|
-
deploymentAddress: params.agentAddress,
|
|
923
|
-
publicKey: result.publicKey,
|
|
924
|
-
};
|
|
925
|
-
}
|
|
926
|
-
async function rollbackCommittedAttachments(committed) {
|
|
927
|
-
if (db === undefined)
|
|
928
|
-
return;
|
|
929
|
-
if (committed.length === 0)
|
|
930
|
-
return;
|
|
931
|
-
// Per-row try/catch so a single rollback failure does not stop the
|
|
932
|
-
// sweep — every committed row needs to come off the books before
|
|
933
|
-
// the caller emits the original sendPack error.
|
|
934
|
-
for (const record of committed) {
|
|
935
|
-
try {
|
|
936
|
-
await db
|
|
937
|
-
.delete(sessionAssetTable)
|
|
938
|
-
.where(and(eq(sessionAssetTable.runId, record.runId), eq(sessionAssetTable.mountPath, record.mountPath), eq(sessionAssetTable.assetPackSha, record.assetPackSha), eq(sessionAssetTable.sourceCommitSha, record.sourceCommitSha)));
|
|
1054
|
+
catch (error) {
|
|
1055
|
+
if (restoredPublicKey !== undefined) {
|
|
1056
|
+
sidecarRouter.noteSenderDeploySettled(senderAttempt, restoredPublicKey === null
|
|
1057
|
+
? { failed: error instanceof Error ? error.message : String(error) }
|
|
1058
|
+
: { recorded: restoredPublicKey });
|
|
939
1059
|
}
|
|
940
|
-
|
|
941
|
-
|
|
1060
|
+
// A sent deploy or cancelled publication may have committed despite its
|
|
1061
|
+
// lost response, as may an unsent rollback. Without a confirmed rollback,
|
|
1062
|
+
// recovery owns transport failures too. Only an uncancelled preparation
|
|
1063
|
+
// failure can settle definitively here.
|
|
1064
|
+
else if (!signal.aborted && !(error instanceof SessionLaunchError)) {
|
|
1065
|
+
sidecarRouter.noteSenderDeploySettled(senderAttempt, {
|
|
1066
|
+
failed: error instanceof Error ? error.message : String(error),
|
|
1067
|
+
});
|
|
942
1068
|
}
|
|
1069
|
+
throw error;
|
|
943
1070
|
}
|
|
944
1071
|
}
|
|
945
1072
|
async function sendAttachmentPack(runId, agentAddress, attachment, allocationTarget) {
|
|
@@ -957,77 +1084,32 @@ export function createSessionService(deps) {
|
|
|
957
1084
|
assetPackSha,
|
|
958
1085
|
sourceCommitSha,
|
|
959
1086
|
};
|
|
960
|
-
// Reserve
|
|
961
|
-
//
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
})
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
const existing = await db.query.sessionAsset.findFirst({
|
|
980
|
-
where: and(eq(sessionAssetTable.runId, runId), eq(sessionAssetTable.mountPath, mountPath)),
|
|
981
|
-
columns: {
|
|
982
|
-
assetPackSha: true,
|
|
983
|
-
sourceCommitSha: true,
|
|
984
|
-
},
|
|
985
|
-
});
|
|
986
|
-
if (existing === undefined) {
|
|
987
|
-
throw new Error(`session_asset ${runId}/${mountPath} disappeared after its insert conflicted`);
|
|
988
|
-
}
|
|
989
|
-
if (existing.assetPackSha !== assetPackSha ||
|
|
990
|
-
existing.sourceCommitSha !== sourceCommitSha) {
|
|
991
|
-
throw new Error(`session_asset ${runId}/${mountPath} conflicts with the allocated workflow's restored asset`);
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
|
-
}
|
|
995
|
-
try {
|
|
996
|
-
const options = { mountPath, repoId };
|
|
997
|
-
if (allocationTarget === undefined) {
|
|
998
|
-
await sidecarRouter.sendPack(agentAddress, pack, ref, sourceCommitSha, options);
|
|
999
|
-
}
|
|
1000
|
-
else {
|
|
1001
|
-
await requireAllocationRouter().sendPackToAllocation(allocationTarget, agentAddress, pack, ref, sourceCommitSha, options);
|
|
1087
|
+
// Reserve durable recovery intent before the pack send. A replacement
|
|
1088
|
+
// generation may reuse the exact row its predecessor recorded.
|
|
1089
|
+
const inserted = await db
|
|
1090
|
+
.insert(sessionAssetTable)
|
|
1091
|
+
.values({ ...record, materializedAt: new Date() })
|
|
1092
|
+
.onConflictDoNothing({
|
|
1093
|
+
target: [sessionAssetTable.runId, sessionAssetTable.mountPath],
|
|
1094
|
+
})
|
|
1095
|
+
.returning({ runId: sessionAssetTable.runId });
|
|
1096
|
+
if (inserted.length === 0) {
|
|
1097
|
+
const existing = await db.query.sessionAsset.findFirst({
|
|
1098
|
+
where: and(eq(sessionAssetTable.runId, runId), eq(sessionAssetTable.mountPath, mountPath)),
|
|
1099
|
+
columns: {
|
|
1100
|
+
assetPackSha: true,
|
|
1101
|
+
sourceCommitSha: true,
|
|
1102
|
+
},
|
|
1103
|
+
});
|
|
1104
|
+
if (existing === undefined) {
|
|
1105
|
+
throw new Error(`session_asset ${runId}/${mountPath} disappeared after its insert conflicted`);
|
|
1002
1106
|
}
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
// the send fails. Allocated rows are durable recovery intent: even a row
|
|
1007
|
-
// first inserted by this attempt can already be reused by another
|
|
1008
|
-
// reconciler, so no replacement attempt may delete it.
|
|
1009
|
-
// The forensic value of a manifest-without-materialization row is
|
|
1010
|
-
// negligible because no agent will read against it. Wrap the
|
|
1011
|
-
// rollback in its own try/catch so a rollback failure (DB gone,
|
|
1012
|
-
// connection killed mid-launch) is logged rather than masking the
|
|
1013
|
-
// primary sendPack error — the caller needs to see the original
|
|
1014
|
-
// failure, not the secondary one.
|
|
1015
|
-
if (rollbackRecord !== null) {
|
|
1016
|
-
try {
|
|
1017
|
-
await db
|
|
1018
|
-
.delete(sessionAssetTable)
|
|
1019
|
-
.where(and(eq(sessionAssetTable.runId, rollbackRecord.runId), eq(sessionAssetTable.mountPath, rollbackRecord.mountPath), eq(sessionAssetTable.assetPackSha, rollbackRecord.assetPackSha), eq(sessionAssetTable.sourceCommitSha, rollbackRecord.sourceCommitSha)));
|
|
1020
|
-
}
|
|
1021
|
-
catch (rollbackErr) {
|
|
1022
|
-
const msg = rollbackErr instanceof Error
|
|
1023
|
-
? rollbackErr.message
|
|
1024
|
-
: String(rollbackErr);
|
|
1025
|
-
logger.warn `session_asset rollback failed for instance=${runId} mountPath=${mountPath}: ${msg}`;
|
|
1026
|
-
}
|
|
1107
|
+
if (existing.assetPackSha !== assetPackSha ||
|
|
1108
|
+
existing.sourceCommitSha !== sourceCommitSha) {
|
|
1109
|
+
throw new Error(`session_asset ${runId}/${mountPath} conflicts with the allocated workflow's restored asset`);
|
|
1027
1110
|
}
|
|
1028
|
-
throw err;
|
|
1029
1111
|
}
|
|
1030
|
-
|
|
1112
|
+
await requireAllocationRouter().sendPackToAllocation(allocationTarget, agentAddress, pack, ref, sourceCommitSha, { mountPath, repoId });
|
|
1031
1113
|
}
|
|
1032
1114
|
/**
|
|
1033
1115
|
* Build a per-agent `ClosureResolver` from the tenant's visible
|
|
@@ -1130,62 +1212,13 @@ export function createSessionService(deps) {
|
|
|
1130
1212
|
ref: returnedRef,
|
|
1131
1213
|
};
|
|
1132
1214
|
}
|
|
1133
|
-
async function attemptCleanup(agentAddress, failedPhase, originalErr) {
|
|
1134
|
-
try {
|
|
1135
|
-
await sidecarRouter.sendAgentUndeploy(agentAddress, failedPhase);
|
|
1136
|
-
}
|
|
1137
|
-
catch (cleanupErr) {
|
|
1138
|
-
logger.error `Failed to clean up agent ${agentAddress} after ${failedPhase} failure: ${String(cleanupErr)}`;
|
|
1139
|
-
// Preserve the original error as cause so the root cause is not
|
|
1140
|
-
// lost when the cleanup also fails.
|
|
1141
|
-
throw new SessionLaunchError(failedPhase, originalErr, true);
|
|
1142
|
-
}
|
|
1143
|
-
}
|
|
1144
|
-
async function sendUserMessage(params) {
|
|
1145
|
-
const { agentAddress, from, messageId, date, content, attachments, inReplyTo, references, sessionId, tenantId, cryptoProvider, } = params;
|
|
1146
|
-
const headers = {
|
|
1147
|
-
from,
|
|
1148
|
-
to: [agentAddress],
|
|
1149
|
-
cc: undefined,
|
|
1150
|
-
date,
|
|
1151
|
-
messageId,
|
|
1152
|
-
subject: undefined,
|
|
1153
|
-
inReplyTo,
|
|
1154
|
-
references,
|
|
1155
|
-
mimeVersion: "1.0",
|
|
1156
|
-
interchangeType: "conversation.message",
|
|
1157
|
-
interchangeCorrelationId: undefined,
|
|
1158
|
-
interchangeTenantId: tenantId,
|
|
1159
|
-
interchangeAgentId: undefined,
|
|
1160
|
-
interchangeSessionId: sessionId,
|
|
1161
|
-
interchangeOfferingId: undefined,
|
|
1162
|
-
interchangeSchemaVersion: undefined,
|
|
1163
|
-
traceparent: undefined,
|
|
1164
|
-
tracestate: undefined,
|
|
1165
|
-
};
|
|
1166
|
-
const signedContent = assembleSignedContent({
|
|
1167
|
-
kind: "conversation",
|
|
1168
|
-
text: content,
|
|
1169
|
-
...(attachments !== undefined ? { attachments } : {}),
|
|
1170
|
-
});
|
|
1171
|
-
const signature = await createDetachedSignatureFromProvider(signedContent, cryptoProvider);
|
|
1172
|
-
const rawMessage = assembleMessage(headers, signedContent, signature);
|
|
1173
|
-
const base64 = base64Encode(rawMessage);
|
|
1174
|
-
const delivered = sidecarRouter.routeMail(agentAddress, base64, messageId);
|
|
1175
|
-
if (!delivered) {
|
|
1176
|
-
throw new Error(`Failed to deliver message to ${agentAddress}: agent is unreachable`);
|
|
1177
|
-
}
|
|
1178
|
-
return rawMessage;
|
|
1179
|
-
}
|
|
1180
1215
|
async function endSession(agentAddress, reason) {
|
|
1181
1216
|
await sidecarRouter.sendAgentUndeploy(agentAddress, reason);
|
|
1182
1217
|
}
|
|
1183
1218
|
return {
|
|
1184
1219
|
stageWorkflowStep,
|
|
1185
|
-
deployWorkflowFromSource,
|
|
1186
1220
|
installAndApproveWorkflowSource,
|
|
1187
1221
|
deployPreparedCodeSourcedWorkflow,
|
|
1188
|
-
sendUserMessage,
|
|
1189
1222
|
endSession,
|
|
1190
1223
|
};
|
|
1191
1224
|
}
|