@nylorun/runtime 0.10.0-beta → 0.11.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. package/CHANGELOG.md +46 -0
  2. package/README.md +53 -51
  3. package/dist/adapters/execution/restate.d.ts +130 -0
  4. package/dist/adapters/execution/restate.js +442 -0
  5. package/dist/adapters/sandbox/virtual.js +1 -1
  6. package/dist/adapters/streams/s2.d.ts +18 -0
  7. package/dist/adapters/streams/s2.js +361 -0
  8. package/dist/core/executors.d.ts +4 -1
  9. package/dist/core/executors.js +20 -1
  10. package/dist/core/flow-host.d.ts +93 -132
  11. package/dist/core/flow-host.js +282 -547
  12. package/dist/core/sandbox-routes.d.ts +8 -3
  13. package/dist/core/sandbox-routes.js +10 -10
  14. package/dist/execution/memory.d.ts +54 -0
  15. package/dist/execution/memory.js +217 -0
  16. package/dist/execution/types.d.ts +125 -0
  17. package/dist/execution/types.js +52 -0
  18. package/dist/host/config-for.d.ts +2 -1
  19. package/dist/host/config-for.js +3 -21
  20. package/dist/host/config.d.ts +0 -8
  21. package/dist/host/create-host.d.ts +36 -1
  22. package/dist/host/create-host.js +48 -24
  23. package/dist/host/execution.d.ts +61 -0
  24. package/dist/host/execution.js +69 -0
  25. package/dist/host/http.d.ts +4 -0
  26. package/dist/host/http.js +6 -0
  27. package/dist/host/index.d.ts +3 -1
  28. package/dist/host/index.js +2 -0
  29. package/dist/host/main.js +106 -47
  30. package/dist/host/stack-config.d.ts +72 -0
  31. package/dist/host/stack-config.js +193 -0
  32. package/dist/index.d.ts +1 -1
  33. package/dist/infra/database.d.ts +16 -0
  34. package/dist/infra/database.js +31 -0
  35. package/dist/infra/execution.d.ts +39 -0
  36. package/dist/infra/execution.js +68 -0
  37. package/dist/infra/index.d.ts +42 -0
  38. package/dist/infra/index.js +56 -0
  39. package/dist/infra/readiness.d.ts +33 -0
  40. package/dist/infra/readiness.js +53 -0
  41. package/dist/infra/streams.d.ts +17 -0
  42. package/dist/infra/streams.js +33 -0
  43. package/dist/infra/workers.d.ts +39 -0
  44. package/dist/infra/workers.js +31 -0
  45. package/dist/model/pi-model.d.ts +2 -2
  46. package/dist/model/pi-model.js +5 -5
  47. package/dist/sandbox/manager.d.ts +26 -8
  48. package/dist/sandbox/manager.js +95 -47
  49. package/dist/sandbox/select.d.ts +1 -1
  50. package/dist/sandbox/select.js +4 -5
  51. package/dist/sandbox/types.d.ts +2 -2
  52. package/dist/store/canonical.d.ts +5 -0
  53. package/dist/store/canonical.js +17 -0
  54. package/dist/store/cursor.d.ts +9 -0
  55. package/dist/store/cursor.js +18 -0
  56. package/dist/store/memory.d.ts +57 -0
  57. package/dist/store/memory.js +643 -0
  58. package/dist/store/ownership.d.ts +19 -0
  59. package/dist/store/ownership.js +33 -0
  60. package/dist/store/postgres/connect.d.ts +25 -0
  61. package/dist/store/postgres/connect.js +25 -0
  62. package/dist/store/postgres/locking.d.ts +16 -0
  63. package/dist/store/postgres/locking.js +22 -0
  64. package/dist/store/postgres/migrations/001_initial.d.ts +27 -0
  65. package/dist/store/postgres/migrations/001_initial.js +196 -0
  66. package/dist/store/postgres/migrations/index.d.ts +49 -0
  67. package/dist/store/postgres/migrations/index.js +75 -0
  68. package/dist/store/postgres/names.d.ts +11 -0
  69. package/dist/store/postgres/names.js +42 -0
  70. package/dist/store/postgres/store.d.ts +11 -0
  71. package/dist/store/postgres/store.js +790 -0
  72. package/dist/store/postgres/tenants.d.ts +90 -0
  73. package/dist/store/postgres/tenants.js +207 -0
  74. package/dist/store/types.d.ts +440 -0
  75. package/dist/store/types.js +10 -0
  76. package/dist/streams/basin.d.ts +23 -0
  77. package/dist/streams/basin.js +51 -0
  78. package/dist/streams/memory.d.ts +25 -0
  79. package/dist/streams/memory.js +124 -0
  80. package/dist/streams/relay.d.ts +37 -0
  81. package/dist/streams/relay.js +254 -0
  82. package/dist/streams/types.d.ts +115 -0
  83. package/dist/streams/types.js +86 -0
  84. package/dist/tenant/actions.d.ts +153 -0
  85. package/dist/tenant/actions.js +191 -0
  86. package/dist/tenant/advance.d.ts +15 -0
  87. package/dist/tenant/advance.js +439 -0
  88. package/dist/tenant/auth.d.ts +11 -0
  89. package/dist/tenant/auth.js +34 -0
  90. package/dist/tenant/commands.d.ts +3 -0
  91. package/dist/tenant/commands.js +384 -0
  92. package/dist/tenant/context.d.ts +140 -0
  93. package/dist/tenant/context.js +40 -0
  94. package/dist/tenant/effects.d.ts +20 -0
  95. package/dist/tenant/effects.js +539 -0
  96. package/dist/tenant/envelope.d.ts +0 -3
  97. package/dist/tenant/envelope.js +2 -34
  98. package/dist/tenant/ephemeral.d.ts +15 -4
  99. package/dist/tenant/ephemeral.js +106 -38
  100. package/dist/tenant/http.d.ts +32 -0
  101. package/dist/tenant/http.js +51 -0
  102. package/dist/tenant/live.d.ts +92 -0
  103. package/dist/tenant/live.js +290 -0
  104. package/dist/tenant/model-setting.d.ts +16 -0
  105. package/dist/tenant/model-setting.js +12 -0
  106. package/dist/tenant/module.d.ts +29 -8
  107. package/dist/tenant/module.js +162 -158
  108. package/dist/tenant/paths.d.ts +12 -8
  109. package/dist/tenant/paths.js +38 -12
  110. package/dist/tenant/principals.d.ts +9 -13
  111. package/dist/tenant/principals.js +31 -38
  112. package/dist/tenant/quarantine-error.d.ts +1 -1
  113. package/dist/tenant/quarantine-error.js +2 -8
  114. package/dist/tenant/quarantine.d.ts +0 -8
  115. package/dist/tenant/quarantine.js +8 -30
  116. package/dist/tenant/reset.d.ts +7 -6
  117. package/dist/tenant/reset.js +7 -51
  118. package/dist/tenant/routes-tenant.d.ts +12 -0
  119. package/dist/tenant/routes-tenant.js +126 -0
  120. package/dist/tenant/routes.d.ts +9 -0
  121. package/dist/tenant/routes.js +142 -0
  122. package/dist/tenant/runtime.d.ts +59 -58
  123. package/dist/tenant/runtime.js +243 -2201
  124. package/dist/tenant/scheduler.d.ts +25 -0
  125. package/dist/tenant/scheduler.js +84 -0
  126. package/dist/tenant/session.d.ts +60 -0
  127. package/dist/tenant/session.js +94 -0
  128. package/dist/tenant/sessions.d.ts +34 -0
  129. package/dist/tenant/sessions.js +149 -0
  130. package/dist/tenant/status.d.ts +13 -4
  131. package/dist/tenant/status.js +76 -41
  132. package/dist/tenant/store-memory.d.ts +9 -12
  133. package/dist/tenant/store-memory.js +79 -68
  134. package/dist/tenant/store-pg.d.ts +15 -0
  135. package/dist/tenant/store-pg.js +140 -0
  136. package/dist/tenant/streams.d.ts +126 -0
  137. package/dist/tenant/streams.js +419 -0
  138. package/dist/tenant/sweep.d.ts +15 -0
  139. package/dist/tenant/sweep.js +122 -0
  140. package/dist/tenant/types.d.ts +95 -15
  141. package/dist/tenant/types.js +23 -1
  142. package/dist/tenant/worker.d.ts +84 -0
  143. package/dist/tenant/worker.js +189 -0
  144. package/dist/vault/service.d.ts +37 -31
  145. package/dist/vault/service.js +439 -359
  146. package/dist/version.d.ts +1 -1
  147. package/dist/version.js +1 -1
  148. package/package.json +13 -13
  149. package/dist/adapters/sandbox/microsandbox.d.ts +0 -2
  150. package/dist/adapters/sandbox/microsandbox.js +0 -238
  151. package/dist/core/store.d.ts +0 -32
  152. package/dist/core/store.js +0 -143
  153. package/dist/launcher/commands.d.ts +0 -24
  154. package/dist/launcher/commands.js +0 -260
  155. package/dist/launcher/ensure-config.d.ts +0 -16
  156. package/dist/launcher/ensure-config.js +0 -79
  157. package/dist/launcher/errors.d.ts +0 -12
  158. package/dist/launcher/errors.js +0 -22
  159. package/dist/launcher/host-config.d.ts +0 -17
  160. package/dist/launcher/host-config.js +0 -64
  161. package/dist/launcher/host-state.d.ts +0 -17
  162. package/dist/launcher/host-state.js +0 -61
  163. package/dist/launcher/lifecycle.d.ts +0 -66
  164. package/dist/launcher/lifecycle.js +0 -407
  165. package/dist/launcher/locks.d.ts +0 -26
  166. package/dist/launcher/locks.js +0 -117
  167. package/dist/launcher/logs.d.ts +0 -18
  168. package/dist/launcher/logs.js +0 -117
  169. package/dist/launcher/main.d.ts +0 -2
  170. package/dist/launcher/main.js +0 -66
  171. package/dist/launcher/output.d.ts +0 -11
  172. package/dist/launcher/output.js +0 -68
  173. package/dist/launcher/paths.d.ts +0 -20
  174. package/dist/launcher/paths.js +0 -36
  175. package/dist/launcher/probe.d.ts +0 -23
  176. package/dist/launcher/probe.js +0 -72
  177. package/dist/launcher/protocol.d.ts +0 -62
  178. package/dist/launcher/protocol.js +0 -1
  179. package/dist/launcher/schema-guard.d.ts +0 -6
  180. package/dist/launcher/schema-guard.js +0 -36
  181. package/dist/launcher/spawn.d.ts +0 -22
  182. package/dist/launcher/spawn.js +0 -85
  183. package/dist/launcher/status.d.ts +0 -6
  184. package/dist/launcher/status.js +0 -65
  185. package/dist/tenant/migration.d.ts +0 -16
  186. package/dist/tenant/migration.js +0 -116
  187. package/dist/tenant/schema.d.ts +0 -23
  188. package/dist/tenant/schema.js +0 -151
  189. package/dist/tenant/store-fs.d.ts +0 -16
  190. package/dist/tenant/store-fs.js +0 -205
@@ -1,228 +1,60 @@
1
- import {} from "node:http";
2
- import { randomUUID, timingSafeEqual } from "node:crypto";
3
- import { mkdirSync, openSync, closeSync, unlinkSync, writeFileSync, readFileSync, existsSync, } from "node:fs";
4
- import { dirname } from "node:path";
5
- import { AgentManifestSchema, CreateCredentialRequestSchema, CreateVaultRequestSchema, PutAgentRequestSchema, PutHostModelRequestSchema, SelectHostModelRequestSchema, PutSessionRequestSchema, RegisterExecutorsRequestSchema, RotateCredentialRequestSchema, SessionCommandSchema, ActionClaimRequestSchema, ActionHeartbeatRequestSchema, ResetTenantRequestSchema, SeedTenantConfigRequestSchema, DefinitionDocumentSchema, } from "@nylorun/core/contracts";
6
- import { createDurableCheckpoint, createFlowCheckpoint, runDurable, runFlowDurable, agentTurnValue, } from "@nylorun/harness/run";
7
- import { aggregateWaits, cancelSiblingWork, countActiveFlowWork, deriveAgentEffectSessionId, emitLoopActionEvents, fenceWorkflowActions, foreignInteractionConflict, isFlowEffect, isFlowToolEffect, isWorkflowManifest, planCancelCascade, reconcilePendingAgentEffects, reofferOrphanedFnVerifyClaims, wakeForQueuedEffects, wakeLinkedWorkflow, } from "../core/flow-host.js";
8
- import { mayDispatchMore, resolveFlowLimits, } from "../core/limits.js";
9
- import { allowedManifestHashes, rebaseTurnState, resolveMessageManifest, } from "../core/turn-manifest.js";
10
- import { hashManifest } from "@nylorun/core/compatibility";
11
- import { delegateManifest, schemaFromJSON, } from "@nylorun/core/define";
12
- import { Store, canonical } from "../core/store.js";
13
- import { ExecutorRegistry, assertExecutorCredential, hashToken, } from "../core/executors.js";
14
- import { hostModelCatalog } from "../model/catalog.js";
15
- import { piModel } from "../model/pi-model.js";
1
+ import { mkdirSync } from "node:fs";
2
+ import { resolveFlowLimits } from "../core/limits.js";
3
+ import { loadExecutorRegistry } from "../core/executors.js";
16
4
  import { scriptedModel, gatewayModel, toolFixtureModel, } from "../core/provider.js";
17
- import { scrub } from "../redact.js";
18
5
  import { createKekFile, readVaultKek } from "../vault/kek.js";
19
- import { QuarantineError } from "./quarantine-error.js";
20
- import { applicationTokenHashes, findPrincipalByTokenHash, } from "./principals.js";
21
- import { resetTenant } from "./reset.js";
22
- import { buildTenantStatus, seedTenantConfig } from "./status.js";
23
- import { VaultError } from "../vault/error.js";
24
6
  import { VaultService } from "../vault/service.js";
25
- import { McpPool, serversOf } from "../mcp/pool.js";
26
- import { SandboxManager, sandboxCapabilityOf } from "../sandbox/manager.js";
7
+ import { McpPool } from "../mcp/pool.js";
8
+ import { SandboxManager } from "../sandbox/manager.js";
27
9
  import { defaultSandboxBackends } from "../sandbox/select.js";
28
- import { owningSandboxSessionId, sandboxSpecOf } from "../sandbox/share.js";
29
- import { SandboxRouteError, handleActionSandboxTool, handleSessionSandboxTool, validateSandboxAttach, } from "../core/sandbox-routes.js";
30
- /** D5 opaque failure for unknown/rejected credentials on Tenant routes. */
31
- const OPAQUE_NOT_FOUND = {
32
- status: "rejected",
33
- code: "not_found",
34
- message: "Not found",
35
- };
36
- class HttpError extends Error {
37
- status;
38
- constructor(status, message) {
39
- super(message);
40
- this.status = status;
41
- }
42
- }
43
- const fail = (status, message) => {
44
- throw new HttpError(status, message);
45
- };
46
- /** AbortSignal tied to the HTTP request being closed by the client. */
47
- function requestAborted(request) {
48
- const controller = new AbortController();
49
- request.on("close", () => {
50
- if (!request.complete)
51
- controller.abort();
52
- });
53
- return controller.signal;
54
- }
55
- class OpaqueAuthError extends Error {
56
- status = 404;
57
- body = OPAQUE_NOT_FOUND;
58
- constructor() {
59
- super(OPAQUE_NOT_FOUND.message);
60
- this.name = "OpaqueAuthError";
61
- }
62
- }
63
- const failOpaque = () => {
64
- throw new OpaqueAuthError();
65
- };
66
- function sessionToolsOf(snapshot) {
67
- if (!snapshot?.mcpTools.length)
68
- return undefined;
69
- return snapshot.mcpTools.map((tool) => ({
70
- ...(tool.agentId === undefined ? {} : { agentId: tool.agentId }),
71
- capabilityId: tool.capabilityId,
72
- name: tool.name,
73
- ...(tool.description === undefined
74
- ? {}
75
- : { description: tool.description }),
76
- inputSchema: tool.inputSchema,
77
- ...(tool.outputSchema === undefined
78
- ? {}
79
- : { outputSchema: tool.outputSchema }),
80
- }));
81
- }
82
- /** Turn-manifest store backed by the session's `variants` map (no schema change). */
83
- function variantStore(session) {
84
- return {
85
- get(hash) {
86
- return session.variants?.[hash];
87
- },
88
- put(hash, manifest) {
89
- session.variants = { ...(session.variants ?? {}), [hash]: manifest };
90
- },
91
- };
92
- }
93
- /** Manifest this turn's checkpoint is pinned to (session pin or a stored variant). */
94
- function turnManifestOf(session) {
95
- const hash = session.checkpoint?.manifestHash;
96
- if (!hash || hash === session.manifestHash)
97
- return session.manifest;
98
- return session.variants?.[hash] ?? session.manifest;
99
- }
100
- /** Wrap a linked agent turn so the Loop learns the turn's manifest (SD-I4 pin stays on session). */
101
- function linkedAgentOutput(session, output) {
102
- if (isWorkflowManifest(session.manifest))
103
- return output ?? null;
104
- return agentTurnValue(output ?? null, turnManifestOf(session));
105
- }
106
- function rebaseSessionState(session, turnHash) {
107
- const allowed = allowedManifestHashes({
108
- pinnedHash: session.manifestHash,
109
- variantHashes: Object.keys(session.variants ?? {}),
110
- });
111
- const next = rebaseTurnState({
112
- state: session.state,
113
- turnManifestHash: turnHash,
114
- isAllowedHash: (hash) => allowed.has(hash),
10
+ import { MemoryExecution } from "../execution/memory.js";
11
+ import { QuarantineError } from "./quarantine-error.js";
12
+ import { MemoryStreams } from "../streams/memory.js";
13
+ import { connectedExecutorCount, createLiveHub, endAllStreams, } from "./live.js";
14
+ import { closeStreams, drainOutbox, streamsStatus, wireStreams, } from "./streams.js";
15
+ import { abortAll, abortLocal, createWorkState, drain, waitForIdle, } from "./scheduler.js";
16
+ import { advance } from "./advance.js";
17
+ import { sweep } from "./sweep.js";
18
+ import { TenantWorkers, WORKER_ID, } from "./worker.js";
19
+ import { authorize } from "./effects.js";
20
+ import { handle } from "./routes.js";
21
+ /** Default ownership lease of an advance; the heartbeat renews it every third. */
22
+ const DEFAULT_OWNER_LEASE_MS = 30_000;
23
+ function validateConfig(config) {
24
+ for (const key of ["leaseMs", "ownerLeaseMs", "sweepIntervalMs"]) {
25
+ const value = config[key];
26
+ if (value !== undefined && (!Number.isFinite(value) || value <= 0))
27
+ throw new Error(`${key} must be finite and positive`);
28
+ }
29
+ const flowLimits = resolveFlowLimits({
30
+ flow: config.flow,
31
+ env: config.flowEnv,
115
32
  });
116
- if (next !== session.state)
117
- session.state = next;
118
- }
119
- function mcpToolOf(session, request) {
120
- return session.mcpSnapshot?.mcpTools.find((tool) => tool.agentId === request.agent?.id &&
121
- tool.capabilityId === request.capabilityId &&
122
- tool.name === request.toolName);
33
+ if ((config.model.kind === "fixture" || config.model.kind === "scripted") &&
34
+ config.mode === "shared")
35
+ throw new Error("fixture/scripted models require ephemeral or test mode");
36
+ return flowLimits;
123
37
  }
124
- /** The manifest of the agent an effect belongs to: the root, or an agent it uses as a tool. */
125
- function manifestFor(manifest, agent) {
126
- return agent ? delegateManifest(manifest, agent.id) : manifest;
127
- }
128
- function pinnedTool(manifest, capabilityId, toolName) {
129
- const capability = manifest.capabilities.find((item) => item.id === capabilityId);
130
- return capability?.tools?.find((tool) => tool.name === toolName);
131
- }
132
- function acceptedToolResult(action, command) {
133
- if (action.kind !== "tool" || !action.outputSchema)
134
- return command;
135
- const value = command.outcome.value;
136
- if (isFailedToolValue(value))
137
- return command;
138
- // Executors wrap successful tool output as `{ kind: "completed", output }`.
139
- // Validate the tool payload, not the outcome envelope.
140
- const candidate = completedToolOutput(value);
141
- let matches = false;
142
- try {
143
- matches = schemaFromJSON(action.outputSchema).validate(candidate).ok;
144
- }
145
- catch {
146
- matches = false;
147
- }
148
- if (matches)
149
- return command;
150
- return {
151
- ...command,
152
- outcome: {
153
- ...command.outcome,
154
- value: {
155
- kind: "failed",
156
- code: "tool.invalid-output",
157
- message: "Tool result does not match the output schema stored on the action",
158
- },
159
- },
160
- };
161
- }
162
- function completedToolOutput(value) {
163
- if (value &&
164
- typeof value === "object" &&
165
- !Array.isArray(value) &&
166
- value.kind === "completed" &&
167
- "output" in value) {
168
- return value.output;
169
- }
170
- return value;
171
- }
172
- function isFailedToolValue(value) {
173
- return (!!value &&
174
- typeof value === "object" &&
175
- !Array.isArray(value) &&
176
- value.kind === "failed");
177
- }
178
- const semantic = (value) => {
179
- const { requestId: _, ...body } = value;
180
- return canonical(body);
181
- };
182
- const sessionIdentity = (value) => {
183
- const { requestId: _requestId, vaultIds: _vaultIds, credentialSelections: _selections, ...body } = value ?? {};
184
- return canonical(body);
185
- };
186
- const equals = (a, b) => {
187
- const aa = Buffer.from(a), bb = Buffer.from(b);
188
- return aa.length === bb.length && timingSafeEqual(aa, bb);
189
- };
190
38
  export class TenantRuntime {
191
- config;
192
- store;
193
- vault;
194
- kek;
195
- kekPath;
196
- running = new Map();
197
- pending = new Set();
198
- observers = new Map();
199
- // Keyed by token hash so a rotation can end exactly the streams that the replaced token owns.
200
- executorStreams = new Map();
201
- registry = new ExecutorRegistry();
202
- mcp;
203
- sandbox;
204
- closing = false;
205
- closed = false;
206
- lockPath;
207
- timer;
208
- modelProvider;
209
- useVaultModel;
210
- createKekIfMissing;
211
- flowLimits;
39
+ ctx;
212
40
  envelope;
213
- constructor(config, hooks, envelope) {
214
- this.config = config;
41
+ worker;
42
+ detach;
43
+ closeGraceMs;
44
+ constructor(ctx, envelope,
45
+ /** This Tenant's Worker handlers, as registered with the execution. */
46
+ worker, detach,
47
+ /** How long close waits for running advances: the execution's advance grace period. */
48
+ closeGraceMs) {
49
+ this.ctx = ctx;
215
50
  this.envelope = envelope;
216
- if (config.leaseMs !== undefined &&
217
- (!Number.isFinite(config.leaseMs) || config.leaseMs <= 0))
218
- throw new Error("leaseMs must be finite and positive");
219
- this.flowLimits = resolveFlowLimits({
220
- flow: config.flow,
221
- env: config.flowEnv,
222
- });
223
- if ((config.model.kind === "fixture" || config.model.kind === "scripted") &&
224
- config.mode === "shared")
225
- throw new Error("fixture/scripted models require ephemeral or test mode");
51
+ this.worker = worker;
52
+ this.detach = detach;
53
+ this.closeGraceMs = closeGraceMs;
54
+ }
55
+ static async open(config, hooks) {
56
+ const envelope = hooks.envelope;
57
+ const flowLimits = validateConfig(config);
226
58
  const paths = config.paths;
227
59
  mkdirSync(paths.root, { recursive: true });
228
60
  mkdirSync(paths.home, { recursive: true });
@@ -230,2027 +62,237 @@ export class TenantRuntime {
230
62
  mkdirSync(paths.sandboxes, { recursive: true });
231
63
  mkdirSync(paths.pluginData, { recursive: true });
232
64
  mkdirSync(paths.logs, { recursive: true });
233
- mkdirSync(dirname(paths.database), { recursive: true });
234
- this.lockPath = paths.lock;
65
+ const store = hooks.store;
66
+ let wired;
67
+ let detach;
235
68
  try {
236
- const oldPid = Number(readFileSync(this.lockPath, "utf8"));
237
- if (!Number.isSafeInteger(oldPid) || oldPid < 1)
238
- throw new Error("Invalid runtime lock; inspect before removing");
239
- try {
240
- process.kill(oldPid, 0);
241
- throw new QuarantineError("locked", "Another Runtime owns this Tenant database", "nylorun tenant status", { lockPath: this.lockPath, lockPid: oldPid });
242
- }
243
- catch (error) {
244
- if (error instanceof QuarantineError)
245
- throw error;
246
- if (error.code !== "ESRCH")
247
- throw error;
248
- unlinkSync(this.lockPath);
249
- }
250
- }
251
- catch (error) {
252
- if (error instanceof QuarantineError)
253
- throw error;
254
- if (error.code !== "ENOENT")
255
- throw error;
256
- }
257
- const fd = openSync(this.lockPath, "wx");
258
- writeFileSync(fd, String(process.pid));
259
- closeSync(fd);
260
- this.kekPath = paths.kek;
261
- this.createKekIfMissing = hooks.createKekIfMissing !== false;
262
- let store;
263
- try {
264
- store = new Store(paths.database, config.tenantId);
265
- this.kek = readVaultKek({
69
+ let kek = readVaultKek({
266
70
  vaultKek: hooks.vaultKek,
267
- vaultKekPath: this.kekPath,
71
+ vaultKekPath: paths.kek,
268
72
  });
269
- if (store.credentialCount() > 0 && !this.kek) {
270
- throw new QuarantineError("kek-missing", "Vault key-encryption key is missing for ciphertext in this Tenant", "restore the vault-kek file beside tenant.sqlite");
271
- }
272
- this.store = store;
273
- }
274
- catch (e) {
275
- store?.db.close();
276
- if (this.lockPath && existsSync(this.lockPath))
277
- unlinkSync(this.lockPath);
278
- throw e;
279
- }
280
- this.registry.seed(this.store.allExecutors().map((row) => ({
281
- agentId: row.agentId,
282
- implementationVersion: row.implementationVersion,
283
- ...(row.manifestHash === undefined
284
- ? {}
285
- : { manifestHash: row.manifestHash }),
286
- tokenHash: row.tokenHash,
287
- persisted: true,
288
- updatedAt: row.updatedAt,
289
- ...(row.principalId === undefined
290
- ? {}
291
- : { principalId: row.principalId }),
292
- })));
293
- this.vault = new VaultService(this.store.db, (fn) => this.store.tx(fn), () => this.ensureKek(), config.vaultFetch ?? globalThis.fetch);
294
- this.mcp = new McpPool({
295
- pluginData: paths.pluginData,
296
- childEnv: config.childEnv,
297
- authorize: (sessionId, request) => this.authorize(sessionId, request),
298
- });
299
- const ephemeral = config.mode === "ephemeral";
300
- this.sandbox = new SandboxManager({
301
- scope: config.tenantId,
302
- store: this.store,
303
- backends: config.sandbox.backends ??
304
- defaultSandboxBackends({ root: paths.sandboxes }),
305
- preference: config.sandbox.backend,
306
- ephemeral,
307
- emit: (sessionId, turnId, type, payload) => {
308
- if (this.closed)
309
- return;
310
- const event = this.store.tx(() => this.store.event(sessionId, turnId, type, payload));
311
- this.publish(event);
312
- },
313
- });
314
- if (!ephemeral && this.sandbox.hasRecords())
315
- void this.sandbox
316
- .reconcile((id) => !this.closed && !!this.store.get("sessions", id))
317
- .catch(() => undefined);
318
- this.useVaultModel = config.model.kind === "vault";
319
- if (hooks.modelProvider)
320
- this.modelProvider = hooks.modelProvider;
321
- else if (config.model.kind === "scripted")
322
- this.modelProvider = scriptedModel(config.model.output);
323
- else if (config.model.kind === "fixture")
324
- this.modelProvider = toolFixtureModel();
325
- else if (config.model.kind === "gateway") {
326
- const gateway = config.model;
327
- this.modelProvider = async (effect, signal) => {
328
- const secret = this.vault.readHostModel();
329
- const token = typeof secret?.credential.key === "string"
330
- ? secret.credential.key
331
- : "";
332
- return gatewayModel({
333
- url: gateway.url,
334
- model: gateway.model,
335
- token,
336
- })(effect, signal);
73
+ if ((await store.tx((t) => t.countCredentials())) > 0 && !kek) {
74
+ throw new QuarantineError("kek-missing", "Vault key-encryption key is missing for ciphertext in this Tenant", "restore the vault-kek file in the Tenant directory");
75
+ }
76
+ const createKekIfMissing = hooks.createKekIfMissing !== false;
77
+ const ensureKek = () => {
78
+ if (kek)
79
+ return kek;
80
+ if (!createKekIfMissing)
81
+ throw new QuarantineError("kek-missing", "Vault key-encryption key is required", "restore the vault-kek file in the Tenant directory");
82
+ kek = createKekFile(paths.kek);
83
+ return kek;
337
84
  };
338
- }
339
- else
340
- this.modelProvider = scriptedModel();
341
- this.store.tx(() => {
342
- for (const effect of this.store.all("effects"))
343
- if (effect.status === "invoking") {
344
- effect.status = "uncertain";
345
- this.store.put("effects", effect.request.effectId, effect);
346
- const sess = this.store.get("sessions", effect.request.sessionId);
347
- if (sess &&
348
- sess.status !== "cancelled" &&
349
- sess.activeTurnId === effect.request.turnId) {
350
- sess.status = "uncertain";
351
- this.store.put("sessions", sess.id, sess);
352
- this.store.event(sess.id, sess.activeTurnId, "effect.uncertain", {
353
- effectId: effect.request.effectId,
354
- });
355
- }
356
- }
357
- });
358
- this.retireLegacyHooks();
359
- this.expireClaims();
360
- // Orphaned fn/verify claims from a prior process: re-offer immediately.
361
- this.store.tx(() => {
362
- reofferOrphanedFnVerifyClaims(this.store);
363
- });
364
- this.notify();
365
- this.timer = setInterval(() => this.expireClaims(), Math.min(config.leaseMs ?? 30000, 5000));
366
- this.timer.unref();
367
- for (const sess of this.store.all("sessions"))
368
- if (sess.status === "running" || sess.status === "runnable")
369
- this.schedule(sess.id);
370
- this.store.tx(() => {
371
- reconcilePendingAgentEffects({
372
- store: this.store,
373
- schedule: (sid) => this.schedule(sid),
374
- publish: (e) => this.publish(e),
85
+ const opened = store;
86
+ const registry = await loadExecutorRegistry(opened);
87
+ // A seeded Tenant setting wins over the Host default (A18).
88
+ const seededBackend = await opened.tx((t) => t.getSetting("sandbox.backend"));
89
+ const vault = new VaultService({
90
+ store: opened,
91
+ kek: ensureKek,
92
+ fetch: config.vaultFetch ?? globalThis.fetch,
375
93
  });
376
- });
377
- }
378
- static async open(config, hooks = {}) {
379
- const envelope = readEnvelope(config);
380
- return new TenantRuntime(config, hooks, envelope);
381
- }
382
- session(id) {
383
- return (this.store.get("sessions", id) ?? fail(404, "Session not found"));
384
- }
385
- publish(event) {
386
- for (const response of this.observers.get(event.sessionId) ?? [])
387
- this.send(response, `id: ${event.cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
388
- }
389
- send(response, data) {
390
- if (!response.write(data))
391
- response.destroy();
392
- }
393
- notify() {
394
- for (const streams of this.executorStreams.values())
395
- for (const response of streams)
396
- this.send(response, 'event: work_available\ndata: {"type":"work_available"}\n\n');
397
- }
398
- /**
399
- * Manifest v4 replaced beforeModelCall/afterModelCall with scoped hooks. Turns started under
400
- * an older manifest cannot continue, and their hook actions no longer parse, so end them.
401
- */
402
- retireLegacyHooks() {
403
- this.store.tx(() => {
404
- for (const action of this.store.all("actions"))
405
- if ((action.kind === "beforeModelCall" ||
406
- action.kind === "afterModelCall") &&
407
- (action.status === "pending" || action.status === "claimed")) {
408
- action.status = "cancelled";
409
- this.store.put("actions", action.actionId, action);
410
- }
411
- for (const s of this.store.all("sessions"))
412
- if (s.activeTurnId &&
413
- !isWorkflowManifest(s.manifest) &&
414
- s.manifest?.manifestSchemaVersion !== 4) {
415
- this.store.event(s.id, s.activeTurnId, "turn.failed", {
416
- error: {
417
- code: "execution.incompatible",
418
- message: "This turn started under manifest schema 3; see MIGRATION.md for before/after hooks",
419
- },
420
- });
421
- s.status = "failed";
422
- s.activeTurnId = null;
423
- this.store.put("sessions", s.id, s);
424
- }
425
- });
426
- }
427
- expireClaims() {
428
- const events = [];
429
- let redeliver = false;
430
- this.store.tx(() => {
431
- for (const action of this.store.all("actions"))
432
- if (action.status === "claimed" &&
433
- Date.parse(action.leaseExpiresAt) <= Date.now()) {
434
- if (action.kind === "hook" ||
435
- action.kind === "fn" ||
436
- action.kind === "verify") {
437
- // Hooks, fn and verify are pure / repeat-safe; a lost claim is offered again.
438
- // The next claim bumps the generation, which fences any late result.
439
- action.status = "pending";
440
- action.claimId = null;
441
- action.leaseExpiresAt = null;
442
- this.store.put("actions", action.actionId, action);
443
- redeliver = true;
444
- continue;
445
- }
446
- action.status = "uncertain";
447
- this.store.put("actions", action.actionId, action);
448
- const effect = this.store.get("effects", action.actionId);
449
- effect.status = "uncertain";
450
- this.store.put("effects", action.actionId, effect);
451
- const s = this.session(action.sessionId);
452
- if (s.status !== "cancelled" && s.activeTurnId === action.turnId) {
453
- s.status = "uncertain";
454
- this.store.put("sessions", s.id, s);
455
- events.push(this.store.event(s.id, s.activeTurnId, "action.uncertain", {
456
- actionId: action.actionId,
457
- ...(action.agent ? { agent: action.agent } : {}),
458
- }));
459
- }
460
- }
461
- });
462
- events.forEach((e) => this.publish(e));
463
- if (redeliver)
464
- this.notify();
465
- }
466
- schedule(id) {
467
- if (this.closing)
468
- return;
469
- this.pending.add(id);
470
- setImmediate(() => {
471
- if (!this.running.has(id) && this.pending.delete(id) && !this.closing)
472
- void this.execute(id);
473
- });
474
- }
475
- async execute(id) {
476
- const s = this.session(id);
477
- if (!s.checkpoint || !["running", "runnable"].includes(s.status))
478
- return;
479
- s.status = "running";
480
- this.store.tx(() => this.store.put("sessions", id, s));
481
- const controller = new AbortController();
482
- this.running.set(id, controller);
483
- try {
484
- // prepareMcp mutates the session's mcpSnapshot; read current after it.
485
- if (!isWorkflowManifest(this.session(id).manifest))
486
- await this.prepareMcp(id, controller.signal);
487
- const current = this.session(id);
488
- const host = {
489
- resolveEffect: (e) => this.resolveEffect(e, controller.signal),
490
- };
491
- if (!isWorkflowManifest(current.manifest) && current.checkpoint) {
492
- rebaseSessionState(current, current.checkpoint.manifestHash);
493
- const cp = current.checkpoint;
494
- current.checkpoint = { ...cp, state: current.state };
495
- this.store.tx(() => this.store.put("sessions", id, current));
496
- }
497
- const result = isWorkflowManifest(current.manifest)
498
- ? await runFlowDurable({
499
- manifest: current.manifest,
500
- checkpoint: current.checkpoint,
501
- signal: controller.signal,
502
- host,
503
- limits: this.flowLimits,
504
- })
505
- : await runDurable({
506
- manifest: turnManifestOf(current),
507
- checkpoint: current.checkpoint,
508
- signal: controller.signal,
509
- sessionTools: sessionToolsOf(current.mcpSnapshot),
510
- host,
511
- });
512
- let event;
513
- const siblingCancelIds = [];
514
- this.store.tx(() => {
515
- const current = this.session(id);
516
- if (current.status === "cancelled" ||
517
- current.activeTurnId !== s.activeTurnId)
518
- return;
519
- // A result can arrive during concurrent action persistence. Preserve the runnable marker.
520
- const resumeRequested = current.status === "runnable";
521
- if (result.status === "waiting" || result.status === "uncertain") {
522
- const linkedWaits = isWorkflowManifest(current.manifest)
523
- ? aggregateWaits({
524
- store: this.store,
525
- workflowSessionId: id,
526
- })
527
- : [];
528
- if (linkedWaits.length > 0 && !resumeRequested) {
529
- current.status = "paused";
530
- current.waits = linkedWaits;
531
- }
532
- else {
533
- current.status = resumeRequested
534
- ? "runnable"
535
- : current.status === "uncertain"
536
- ? "uncertain"
537
- : result.status;
538
- current.waits =
539
- linkedWaits.length > 0
540
- ? linkedWaits
541
- : { effectIds: result.effectIds };
542
- }
543
- }
544
- else if ("result" in result) {
545
- const flow = isWorkflowManifest(current.manifest);
546
- if (!flow) {
547
- current.state =
548
- result.status === "failed"
549
- ? current.turnStartState
550
- : result.result.state;
94
+ // `ctx` is assigned below; these callbacks only run once the Tenant is open.
95
+ let ctx;
96
+ const mcp = new McpPool({
97
+ pluginData: paths.pluginData,
98
+ childEnv: config.childEnv,
99
+ authorize: (sessionId, request) => authorize(ctx, sessionId, request),
100
+ });
101
+ const ephemeral = config.mode === "ephemeral";
102
+ const sandbox = new SandboxManager({
103
+ scope: config.tenantId,
104
+ store: opened,
105
+ backends: config.sandbox.backends ??
106
+ defaultSandboxBackends({ root: paths.sandboxes }),
107
+ preference: seededBackend ?? config.sandbox.backend,
108
+ ephemeral,
109
+ emit: async (sessionId, turnId, type, payload) => {
110
+ if (ctx.closed)
111
+ return;
112
+ try {
113
+ await opened.tx((t) => t.event(sessionId, turnId, type, payload));
551
114
  }
552
- current.checkpoint = result.checkpoint;
553
- this.store.put("checkpoints", JSON.stringify([id, s.activeTurnId, result.checkpoint.segment]), { checkpoint: result.checkpoint, status: result.status });
554
- current.status = result.status;
555
- current.waits =
556
- result.status === "paused"
557
- ? result.result.pending
558
- : undefined;
559
- if (result.status === "completed")
560
- current.lastOutput = result.result.output;
561
- // Fail-fast: cancel pending Parallel/Map siblings before clearing the turn.
562
- if (result.status === "failed" &&
563
- flow &&
564
- "cancelEffectIds" in result &&
565
- Array.isArray(result.cancelEffectIds) &&
566
- result.cancelEffectIds.length > 0 &&
567
- s.activeTurnId) {
568
- const cancelResult = cancelSiblingWork({
569
- store: this.store,
570
- workflowSessionId: id,
571
- turnId: s.activeTurnId,
572
- cancelEffectIds: result.cancelEffectIds,
573
- });
574
- siblingCancelIds.push(...cancelResult.agentSessionIds);
115
+ catch {
116
+ /* the session is gone (reset) or the store closed */
575
117
  }
576
- if (result.status !== "paused")
577
- current.activeTurnId = null;
578
- event = this.store.event(id, s.activeTurnId, `turn.${result.status}`, result.status === "completed"
579
- ? { output: result.result.output }
580
- : result.status === "paused"
581
- ? {
582
- interactions: (result.result.pending ?? []).map((call) => ({
583
- invocationId: call.invocationId,
584
- interaction: call.interaction,
585
- wait: call.wait,
586
- status: call.status,
587
- })),
588
- }
589
- : result.status === "failed"
590
- ? {
591
- error: {
592
- code: result.result.error?.code,
593
- message: result.result.error?.message,
594
- ...(result.result.error?.path
595
- ? { path: result.result.error.path }
596
- : {}),
597
- },
598
- }
599
- : {});
600
- }
601
- this.store.put("sessions", id, current);
602
- });
603
- for (const agentId of siblingCancelIds) {
604
- try {
605
- this.command(agentId, {
606
- type: "cancel",
607
- requestId: `flow-sibling-${id}-${agentId}`,
608
- idempotencyKey: `flow-sibling-cancel:${id}:${agentId}:${s.activeTurnId}`,
609
- reason: "sibling branch failed",
610
- }, {
611
- kind: "application",
612
- principalId: "flow-host",
613
- });
614
- }
615
- catch {
616
- /* agent may already be terminal */
617
- }
618
- }
619
- if (event) {
620
- this.publish(event);
621
- if (event.type === "turn.completed" || event.type === "turn.failed") {
622
- this.store.tx(() => {
623
- wakeLinkedWorkflow({
624
- agentSessionId: id,
625
- store: this.store,
626
- output: event.type === "turn.completed"
627
- ? linkedAgentOutput(this.session(id), event.payload.output)
628
- : undefined,
629
- failed: event.type === "turn.failed",
630
- error: event.type === "turn.failed"
631
- ? String(event.payload.message ?? "")
632
- : undefined,
633
- schedule: (sid) => this.schedule(sid),
634
- publish: (e) => this.publish(e),
635
- });
636
- });
637
- }
638
- if (event.type === "turn.cancelled") {
639
- this.store.tx(() => {
640
- wakeLinkedWorkflow({
641
- agentSessionId: id,
642
- store: this.store,
643
- cancelled: true,
644
- error: "Agent turn was cancelled",
645
- schedule: (sid) => this.schedule(sid),
646
- publish: (e) => this.publish(e),
647
- });
648
- });
649
- }
650
- }
651
- }
652
- catch (error) {
653
- let event;
654
- this.store.tx(() => {
655
- const current = this.session(id);
656
- if (current.status === "cancelled" ||
657
- current.activeTurnId !== s.activeTurnId)
658
- return;
659
- current.status = "failed";
660
- current.state = current.turnStartState;
661
- if (current.checkpoint)
662
- this.store.put("checkpoints", JSON.stringify([id, s.activeTurnId, current.checkpoint.segment]), { checkpoint: current.checkpoint, status: "failed" });
663
- current.error = error instanceof Error ? error.message : String(error);
664
- this.store.put("sessions", id, current);
665
- event = this.store.event(id, current.activeTurnId, "turn.failed", {
666
- message: current.error,
667
- });
668
- current.activeTurnId = null;
669
- this.store.put("sessions", id, current);
118
+ },
670
119
  });
671
- if (event) {
672
- this.publish(event);
673
- this.store.tx(() => {
674
- wakeLinkedWorkflow({
675
- agentSessionId: id,
676
- store: this.store,
677
- failed: true,
678
- error: String(event.payload.message ?? ""),
679
- schedule: (sid) => this.schedule(sid),
680
- publish: (e) => this.publish(e),
681
- });
682
- });
683
- }
684
- }
685
- finally {
686
- this.running.delete(id);
687
- if (!this.closing &&
688
- (this.pending.has(id) || this.session(id).status === "runnable"))
689
- this.schedule(id);
690
- }
691
- }
692
- invokeModel(request, signal) {
693
- if (!this.useVaultModel)
694
- return this.modelProvider(request, signal);
695
- const adapter = piModel({
696
- root: this.config.paths.home,
697
- readHostModel: () => this.vault.readHostModel(),
698
- writeHostCredential: (credential) => this.vault.updateHostCredential(credential),
699
- });
700
- return adapter(request.input, {
701
- request: request.context.request,
702
- invocationId: String(request.context.invocationId),
703
- signal,
704
- reportPreparedCall() { },
705
- });
706
- }
707
- async resolveEffect(request, signal) {
708
- let invoke;
709
- let notify = false;
710
- let event;
711
- const resolution = this.store.tx(() => {
712
- const s = this.session(request.sessionId);
713
- if (s.status === "cancelled" ||
714
- s.activeTurnId !== request.turnId ||
715
- signal.aborted)
716
- throw new Error("Turn cancelled");
717
- const existing = this.store.get("effects", request.effectId);
718
- if (existing) {
719
- if (canonical(existing.request) !== canonical(request))
720
- throw new Error("Effect identity request drift");
721
- if (existing.status === "completed")
722
- return { status: "completed", outcome: existing.outcome };
723
- if (request.kind === "agent" && existing.status === "pending") {
724
- const agentSessionId = existing.agentSessionId;
725
- if (agentSessionId) {
726
- const agent = this.store.get("sessions", agentSessionId);
727
- if (agent?.status === "completed") {
728
- const outcome = {
729
- value: linkedAgentOutput(agent, agent.lastOutput ?? null),
730
- };
731
- existing.status = "completed";
732
- existing.outcome = outcome;
733
- this.store.put("effects", request.effectId, existing);
734
- return { status: "completed", outcome };
735
- }
736
- if (agent?.status === "failed") {
737
- const outcome = {
738
- value: {
739
- kind: "failed",
740
- code: "agent.failed",
741
- message: agent.error ?? "Agent turn failed",
742
- },
743
- };
744
- existing.status = "completed";
745
- existing.outcome = outcome;
746
- this.store.put("effects", request.effectId, existing);
747
- return { status: "completed", outcome };
748
- }
749
- }
750
- }
751
- return {
752
- status: existing.status === "uncertain" || existing.status === "invoking"
753
- ? "uncertain"
754
- : "pending",
120
+ const useVaultModel = config.model.kind === "vault";
121
+ let modelProvider;
122
+ if (hooks.modelProvider)
123
+ modelProvider = hooks.modelProvider;
124
+ else if (config.model.kind === "scripted")
125
+ modelProvider = scriptedModel(config.model.output);
126
+ else if (config.model.kind === "fixture")
127
+ modelProvider = toolFixtureModel();
128
+ else if (config.model.kind === "gateway") {
129
+ const gateway = config.model;
130
+ modelProvider = async (effect, signal) => {
131
+ const secret = await vault.readHostModel();
132
+ const token = typeof secret?.credential.key === "string"
133
+ ? secret.credential.key
134
+ : "";
135
+ return gatewayModel({
136
+ url: gateway.url,
137
+ model: gateway.model,
138
+ token,
139
+ })(effect, signal);
755
140
  };
756
141
  }
757
- if (request.kind === "agent" ||
758
- request.kind === "fn" ||
759
- request.kind === "verify" ||
760
- isFlowToolEffect(request)) {
761
- // Handled outside the agent-manifest path below.
762
- return { status: "pending", __flow: true };
763
- }
764
- if (request.kind === "delegation") {
765
- // Lifecycle points of an agent used as a tool: journaled once, so replays never re-emit.
766
- const outcome = { value: null };
767
- this.store.put("effects", request.effectId, {
768
- request,
769
- status: "completed",
770
- outcome,
142
+ else
143
+ modelProvider = scriptedModel();
144
+ const live = createLiveHub();
145
+ const local = hooks.execution
146
+ ? undefined
147
+ : new MemoryExecution({
148
+ sweepIntervalMs: config.sweepIntervalMs ?? Math.min(config.leaseMs ?? 30_000, 5000),
149
+ onError: (error) => config.logger.error("tenant execution failed", {
150
+ message: error instanceof Error ? error.message : String(error),
151
+ }),
771
152
  });
772
- const settled = request.effectId.endsWith(":settled");
773
- event = this.store.event(s.id, s.activeTurnId, settled ? "delegation.completed" : "delegation.started", { agent: request.agent, ...request.input });
774
- return { status: "completed", outcome };
775
- }
776
- const agentManifest = manifestFor(s.manifest, request.agent);
777
- if (!agentManifest)
778
- throw new Error(`Agent '${request.agent?.id ?? ""}' is not used as a tool`);
779
- const mcpTool = request.kind === "tool" ? mcpToolOf(s, request) : undefined;
780
- const sandboxTool = request.kind === "tool" && !mcpTool
781
- ? sandboxCapabilityOf(agentManifest, request.capabilityId, request.toolName)
782
- : undefined;
783
- this.store.put("effects", request.effectId, {
784
- request,
785
- status: request.kind === "model" || mcpTool || sandboxTool
786
- ? "invoking"
787
- : "pending",
788
- });
789
- if (request.kind === "model" || mcpTool || sandboxTool) {
790
- invoke = mcpTool ? "mcp" : sandboxTool ? "sandbox" : "model";
791
- return undefined;
792
- }
793
- const tool = request.kind === "tool"
794
- ? pinnedTool(agentManifest, request.capabilityId, request.toolName)
795
- : undefined;
796
- const base = {
797
- actionId: request.effectId,
798
- sessionId: request.sessionId,
799
- turnId: request.turnId,
800
- agentId: request.agentId,
801
- manifestHash: request.manifestHash,
802
- implementationVersion: s.implementationVersion,
803
- input: request.input,
804
- context: request.context,
805
- status: "pending",
806
- generation: 0,
807
- claimId: null,
808
- leaseExpiresAt: null,
809
- ...(request.agent ? { agent: request.agent } : {}),
153
+ const { execution, workers } = hooks.execution ?? {
154
+ execution: local,
155
+ workers: new TenantWorkers(),
810
156
  };
811
- const action = request.kind === "hook"
812
- ? {
813
- ...base,
814
- kind: "hook",
815
- hook: {
816
- at: request.hook.at,
817
- scope: request.hook.scope,
818
- capabilityIds: [...request.hook.capabilityIds],
819
- },
820
- }
821
- : {
822
- ...base,
823
- kind: "tool",
824
- capabilityId: request.capabilityId,
825
- toolName: request.toolName,
826
- ...(tool?.inputSchema ? { inputSchema: tool.inputSchema } : {}),
827
- ...(tool?.outputSchema
828
- ? { outputSchema: tool.outputSchema }
829
- : {}),
830
- };
831
- this.store.put("actions", action.actionId, action);
832
- event = this.store.event(s.id, s.activeTurnId, "action.pending", {
833
- actionId: action.actionId,
834
- kind: action.kind,
835
- ...actionTarget(action),
836
- input: action.input,
837
- });
838
- notify = true;
839
- return { status: "pending" };
840
- });
841
- if (event)
842
- this.publish(event);
843
- if (notify)
844
- this.notify();
845
- if (resolution && resolution.__flow && isFlowEffect(request)) {
846
- return this.resolveNewFlowEffect(request, signal);
847
- }
848
- if (!invoke)
849
- return resolution;
850
- try {
851
- const value = invoke === "mcp"
852
- ? await this.callMcpTool(request)
853
- : invoke === "sandbox"
854
- ? await this.callSandboxTool(request, signal)
855
- : await this.invokeModel(request, signal);
856
- return this.store.tx(() => {
857
- const s = this.session(request.sessionId);
858
- if (s.status === "cancelled" ||
859
- s.activeTurnId !== request.turnId ||
860
- signal.aborted)
861
- throw new Error("Turn cancelled");
862
- const effect = this.store.get("effects", request.effectId);
863
- effect.status = "completed";
864
- effect.outcome = { value };
865
- this.store.put("effects", request.effectId, effect);
866
- return { status: "completed", outcome: effect.outcome };
867
- });
868
- }
869
- catch (error) {
870
- let event;
871
- this.store.tx(() => {
872
- const effect = this.store.get("effects", request.effectId);
873
- effect.status = "uncertain";
874
- effect.error = error instanceof Error ? error.message : String(error);
875
- this.store.put("effects", request.effectId, effect);
876
- const s = this.session(request.sessionId);
877
- if (s.status !== "cancelled" && s.activeTurnId === request.turnId)
878
- event = this.store.event(s.id, request.turnId, "effect.uncertain", {
879
- effectId: request.effectId,
880
- message: effect.error,
881
- });
882
- });
883
- if (event)
884
- this.publish(event);
885
- return { status: "uncertain" };
886
- }
887
- }
888
- /** Journal and dispatch a new flow effect (agent / tool node / fn / verify). */
889
- async resolveNewFlowEffect(request, signal) {
890
- if (signal.aborted)
891
- throw new Error("Turn cancelled");
892
- const existing = this.store.get("effects", request.effectId);
893
- if (existing) {
894
- if (existing.status === "completed")
895
- return { status: "completed", outcome: existing.outcome };
896
- if (existing.status === "queued") {
897
- // Fall through to dispatch when a concurrency slot is free.
898
- }
899
- else if (request.kind === "agent") {
900
- const agentSessionId = existing.agentSessionId;
901
- if (agentSessionId) {
902
- const agent = this.store.get("sessions", agentSessionId);
903
- if (agent?.status === "completed") {
904
- const outcome = {
905
- value: linkedAgentOutput(agent, agent.lastOutput ?? null),
906
- };
907
- this.store.tx(() => {
908
- existing.status = "completed";
909
- existing.outcome = outcome;
910
- this.store.put("effects", request.effectId, existing);
911
- });
912
- return { status: "completed", outcome };
913
- }
914
- if (agent?.status === "cancelled") {
915
- const outcome = {
916
- value: {
917
- kind: "failed",
918
- code: "agent.cancelled",
919
- message: agent.error ?? "Agent turn was cancelled",
920
- },
921
- };
922
- this.store.tx(() => {
923
- existing.status = "completed";
924
- existing.outcome = outcome;
925
- this.store.put("effects", request.effectId, existing);
926
- });
927
- return { status: "completed", outcome };
928
- }
929
- if (agent?.status === "failed") {
930
- const outcome = {
931
- value: {
932
- kind: "failed",
933
- code: "agent.failed",
934
- message: agent.error ?? "Agent turn failed",
935
- },
936
- };
937
- this.store.tx(() => {
938
- existing.status = "completed";
939
- existing.outcome = outcome;
940
- this.store.put("effects", request.effectId, existing);
941
- });
942
- return { status: "completed", outcome };
943
- }
944
- }
945
- return { status: "pending" };
946
- }
947
- else {
948
- return { status: "pending" };
949
- }
950
- }
951
- const active = countActiveFlowWork(this.store, request.sessionId, request.turnId);
952
- if (!mayDispatchMore(active, this.flowLimits)) {
953
- if (!existing || existing.status !== "queued") {
954
- this.store.tx(() => {
955
- this.store.put("effects", request.effectId, {
956
- request,
957
- status: "queued",
958
- });
959
- });
960
- }
961
- return { status: "pending" };
962
- }
963
- if (request.kind === "fn" ||
964
- request.kind === "verify" ||
965
- isFlowToolEffect(request)) {
966
- let event;
967
- let started;
968
- this.store.tx(() => {
969
- const s = this.session(request.sessionId);
970
- this.store.put("effects", request.effectId, {
971
- request,
972
- status: "pending",
973
- });
974
- const action = request.kind === "fn" || request.kind === "verify"
157
+ const sweepHooks = new Set();
158
+ ctx = {
159
+ config,
160
+ envelope,
161
+ store: opened,
162
+ vault,
163
+ registry,
164
+ mcp,
165
+ sandbox,
166
+ flowLimits,
167
+ modelProvider,
168
+ useVaultModel,
169
+ closing: false,
170
+ closed: false,
171
+ work: createWorkState(),
172
+ live,
173
+ workerId: hooks.workerId ?? WORKER_ID,
174
+ ownerLeaseMs: config.ownerLeaseMs ?? DEFAULT_OWNER_LEASE_MS,
175
+ wake: async (sessionId, wake) => {
176
+ if (ctx.closing || ctx.closed)
177
+ return;
178
+ await execution.wake(config.tenantId, sessionId, wake);
179
+ },
180
+ abortLocal: (sessionId) => abortLocal(ctx, sessionId),
181
+ ...(hooks.execution?.stuckInvocations
975
182
  ? {
976
- actionId: request.effectId,
977
- sessionId: request.sessionId,
978
- turnId: request.turnId,
979
- agentId: request.agentId,
980
- manifestHash: request.manifestHash,
981
- implementationVersion: s.implementationVersion,
982
- input: request.input,
983
- context: request.context,
984
- status: "pending",
985
- generation: 0,
986
- claimId: null,
987
- leaseExpiresAt: null,
988
- kind: request.kind,
989
- path: request.path,
990
- key: request.key,
183
+ stuckInvocations: () => hooks.execution.stuckInvocations(config.tenantId),
991
184
  }
992
- : {
993
- actionId: request.effectId,
994
- sessionId: request.sessionId,
995
- turnId: request.turnId,
996
- agentId: request.agentId,
997
- manifestHash: request.manifestHash,
998
- implementationVersion: s.implementationVersion,
999
- input: request.input,
1000
- context: request.context,
1001
- status: "pending",
1002
- generation: 0,
1003
- claimId: null,
1004
- leaseExpiresAt: null,
1005
- kind: "tool",
1006
- path: request.path,
1007
- key: request.key,
1008
- };
1009
- this.store.put("actions", action.actionId, action);
1010
- if (isFlowToolEffect(request)) {
1011
- started = this.store.event(s.id, s.activeTurnId, "node.started", {
1012
- path: request.path,
1013
- kind: "tool",
1014
- key: request.key,
1015
- ...(request.iterations !== undefined
1016
- ? { iterations: request.iterations }
1017
- : {}),
1018
- });
1019
- }
1020
- event = this.store.event(s.id, s.activeTurnId, "action.pending", {
1021
- actionId: action.actionId,
1022
- kind: action.kind,
1023
- path: action.path,
1024
- key: action.key,
1025
- input: action.input,
1026
- });
1027
- });
1028
- if (started)
1029
- this.publish(started);
1030
- if (event)
1031
- this.publish(event);
1032
- this.notify();
1033
- return { status: "pending" };
1034
- }
1035
- // agent effect: create linked session + message via the public contract path
1036
- const body = request.input;
1037
- const path = body.path ?? request.path;
1038
- const workflow = this.session(request.sessionId);
1039
- const agentSessionId = deriveAgentEffectSessionId(workflow.id, path, request);
1040
- const iterations = request.iterations ?? "-";
1041
- const n = Number(request.context.n ?? iterations.split(".")[0] ?? 1);
1042
- this.store.tx(() => {
1043
- this.store.put("effects", request.effectId, {
1044
- request,
1045
- status: "pending",
1046
- agentSessionId,
1047
- });
1048
- this.store.put("links", agentSessionId, {
1049
- workflowSessionId: workflow.id,
1050
- path,
1051
- effectId: request.effectId,
1052
- turnId: request.turnId,
1053
- });
1054
- });
1055
- if (!this.store.get("sessions", agentSessionId)) {
1056
- const definition = this.store.get("definitions", body.agentId) ??
1057
- fail(404, "Definition not found");
1058
- const sandboxOwnerId = sandboxSpecOf(workflow.manifest) || workflow.sandboxOwnerId
1059
- ? owningSandboxSessionId(workflow, (sid) => this.store.get("sessions", sid))
1060
- : undefined;
1061
- const created = {
1062
- id: agentSessionId,
1063
- agentId: body.agentId,
1064
- ownerUserId: workflow.ownerUserId,
1065
- manifest: definition.manifest,
1066
- manifestHash: definition.manifestHash,
1067
- implementationVersion: definition.implementationVersion,
1068
- status: "idle",
1069
- activeTurnId: null,
1070
- creation: {
1071
- requestId: `flow-put-${request.effectId}`,
1072
- agentId: body.agentId,
1073
- ownerUserId: workflow.ownerUserId,
1074
- ...(sandboxOwnerId ? { sandbox: { session: sandboxOwnerId } } : {}),
185
+ : {}),
186
+ sweepHooks,
187
+ onSweep: (hook) => {
188
+ sweepHooks.add(hook);
189
+ return () => sweepHooks.delete(hook);
1075
190
  },
1076
- vaultIds: workflow.vaultIds,
1077
- credentialSelections: workflow.credentialSelections,
1078
- pluginRoots: definition.pluginRoots ?? {},
1079
- ...(sandboxOwnerId ? { sandboxOwnerId } : {}),
1080
- };
1081
- this.store.tx(() => this.store.put("sessions", agentSessionId, created));
1082
- }
1083
- const idempotencyKey = `${request.turnId}:${path}:${iterations}`;
1084
- const messageInput = body.input;
1085
- const messageCommand = typeof messageInput === "string"
1086
- ? {
1087
- type: "message",
1088
- content: messageInput,
1089
- requestId: `flow-${request.effectId}`,
1090
- idempotencyKey,
1091
- ...(body.manifest ? { manifest: body.manifest } : {}),
1092
- }
1093
- : {
1094
- type: "message",
1095
- data: messageInput,
1096
- requestId: `flow-${request.effectId}`,
1097
- idempotencyKey,
1098
- ...(body.manifest ? { manifest: body.manifest } : {}),
1099
- };
1100
- const accepted = this.command(agentSessionId, messageCommand, {
1101
- kind: "application",
1102
- principalId: "flow-host",
1103
- });
1104
- const events = [];
1105
- this.store.tx(() => {
1106
- const s = this.session(request.sessionId);
1107
- const agentSession = this.session(agentSessionId);
1108
- events.push(this.store.event(s.id, s.activeTurnId, "loop.iteration", {
1109
- path: String(request.context.loopPath ?? path.split("/")[0]),
1110
- n,
1111
- sessionId: agentSessionId,
1112
- turnId: accepted.turnId ?? undefined,
1113
- manifestHash: agentSession.checkpoint?.manifestHash,
1114
- }));
1115
- events.push(this.store.event(s.id, s.activeTurnId, "node.agent", {
1116
- path,
1117
- iterations,
1118
- sessionId: agentSessionId,
1119
- turnId: accepted.turnId,
1120
- }));
1121
- });
1122
- for (const e of events)
1123
- this.publish(e);
1124
- // May already be settled if the agent was fast / replayed.
1125
- const agent = this.session(agentSessionId);
1126
- if (agent.status === "completed") {
1127
- const outcome = {
1128
- value: linkedAgentOutput(agent, agent.lastOutput ?? null),
1129
191
  };
1130
- this.store.tx(() => {
1131
- this.store.put("effects", request.effectId, {
1132
- request,
1133
- status: "completed",
1134
- outcome,
1135
- agentSessionId,
1136
- });
192
+ wired = await wireStreams(ctx, {
193
+ store: opened,
194
+ streams: hooks.streams ?? new MemoryStreams(),
195
+ ownsStreams: !hooks.streams,
196
+ tenantId: config.tenantId,
1137
197
  });
1138
- return { status: "completed", outcome };
1139
- }
1140
- return { status: "pending" };
1141
- }
1142
- scope(request) {
1143
- const header = request.headers.authorization;
1144
- const token = header?.startsWith("Bearer ") ? header.slice(7) : "";
1145
- if (!token || !header?.startsWith("Bearer ")) {
1146
- this.config.logger.warn("credential rejected", {
1147
- reason: "missing_bearer",
1148
- });
1149
- return failOpaque();
1150
- }
1151
- const tokenHash = hashToken(token);
1152
- const principal = findPrincipalByTokenHash(this.store.db, tokenHash);
1153
- if (principal)
1154
- return { kind: "application", principalId: principal.id };
1155
- const executor = this.registry.find(tokenHash);
1156
- if (!executor) {
1157
- this.config.logger.warn("credential rejected", {
1158
- reason: "unknown_token",
198
+ // Outbox rows a lost relay step left behind are appended by the sweep.
199
+ sweepHooks.add(async () => {
200
+ await drainOutbox(ctx);
1159
201
  });
1160
- return failOpaque();
1161
- }
1162
- return { kind: "executor", executor };
1163
- }
1164
- scoped(scope, action) {
1165
- if (scope.kind !== "executor" || scope.executor.agentId !== action.agentId)
1166
- fail(403, "Executor scope does not authorize this action");
1167
- }
1168
- requireApplication(scope) {
1169
- if (scope.kind === "application")
1170
- return scope.principalId;
1171
- return fail(403, "Application credential required");
1172
- }
1173
- sandboxRouteDeps() {
1174
- return {
1175
- sandbox: this.sandbox,
1176
- session: (id) => this.session(id),
1177
- lookup: (id) => this.store.get("sessions", id),
1178
- getAction: (id) => this.store.get("actions", id),
1179
- };
1180
- }
1181
- async body(request) {
1182
- let data = "";
1183
- for await (const chunk of request) {
1184
- data += chunk;
1185
- if (Buffer.byteLength(data) > 1024 * 1024)
1186
- fail(413, "Request too large");
1187
- }
1188
- try {
1189
- return JSON.parse(data);
1190
- }
1191
- catch {
1192
- return fail(400, "Invalid JSON");
1193
- }
1194
- }
1195
- command(id, input, scope) {
1196
- let event;
1197
- const extraEvents = [];
1198
- const cascadeCancelIds = [];
1199
- let schedule = false;
1200
- const response = this.store.tx(() => {
1201
- const s = this.session(id);
1202
- let command = input;
1203
- if (command.type === "action_result") {
1204
- const a = this.store.get("actions", command.actionId) ??
1205
- fail(404, "Action not found");
1206
- if (a.sessionId !== id)
1207
- fail(403, "Action belongs to another session");
1208
- this.scoped(scope, a);
1209
- command = acceptedToolResult(a, command);
1210
- }
1211
- else if (scope.kind !== "application")
1212
- fail(403, "Application credential required");
1213
- const key = JSON.stringify([id, command.idempotencyKey]);
1214
- const existing = this.store.get("commands", key);
1215
- if (existing) {
1216
- if (semantic(existing.command) !== semantic(command))
1217
- fail(409, "Idempotency key already binds another command");
1218
- return existing.response;
1219
- }
1220
- if (command.type === "action_result") {
1221
- const action = this.store.get("actions", command.actionId);
1222
- const prior = this.store.get("effects", command.actionId);
1223
- if (action.status === "completed") {
1224
- if (action.claimId !== command.claimId ||
1225
- action.generation !== command.generation ||
1226
- canonical(prior.outcome) !== canonical(command.outcome))
1227
- fail(409, "Conflicting action result");
1228
- this.store.put("commands", key, { command, response: prior.receipt });
1229
- return prior.receipt;
1230
- }
1231
- if (s.status === "cancelled" ||
1232
- s.activeTurnId !== action.turnId ||
1233
- action.status !== "claimed" ||
1234
- action.claimId !== command.claimId ||
1235
- action.generation !== command.generation ||
1236
- Date.parse(action.leaseExpiresAt) <= Date.now())
1237
- fail(409, "Stale, expired, or cancelled claim");
1238
- action.status = "completed";
1239
- this.store.put("actions", action.actionId, action);
1240
- prior.status = "completed";
1241
- prior.outcome = command.outcome;
1242
- s.status = "runnable";
1243
- schedule = true;
1244
- event = this.store.event(id, s.activeTurnId, "action.completed", {
1245
- actionId: action.actionId,
1246
- ...actionTarget(action),
1247
- kind: action.kind,
1248
- result: command.outcome.value,
1249
- });
1250
- if (action.kind === "verify") {
1251
- const verdict = command.outcome.value;
1252
- if (verdict?.kind !== "failed") {
1253
- extraEvents.push(this.store.event(id, s.activeTurnId, "loop.verified", {
1254
- path: String(action.context?.loopPath ?? action.path ?? ""),
1255
- n: Number(action.context?.n ?? 1),
1256
- pass: Boolean(verdict?.pass),
1257
- ...(verdict?.feedback !== undefined
1258
- ? { feedback: verdict.feedback }
1259
- : {}),
1260
- ...(verdict?.data !== undefined ? { data: verdict.data } : {}),
1261
- }));
1262
- }
1263
- }
1264
- else if (action.kind === "fn" && action.context?.role === "decide") {
1265
- const decision = command.outcome.value;
1266
- extraEvents.push(this.store.event(id, s.activeTurnId, "loop.decided", {
1267
- path: String(action.context?.loopPath ?? action.path ?? ""),
1268
- n: Number(action.context?.n ?? 1),
1269
- next: "output" in decision && !("input" in decision)
1270
- ? "output"
1271
- : "input",
1272
- patched: Boolean(decision.agent),
1273
- }));
1274
- }
1275
- const receipt = {
1276
- status: "accepted",
1277
- turnId: s.activeTurnId,
1278
- cursor: event.cursor,
1279
- requestId: command.requestId,
1280
- };
1281
- prior.receipt = receipt;
1282
- this.store.put("effects", action.actionId, prior);
1283
- if (isWorkflowManifest(s.manifest) && s.activeTurnId) {
1284
- wakeForQueuedEffects({
1285
- store: this.store,
1286
- workflowSessionId: id,
1287
- turnId: s.activeTurnId,
1288
- limits: this.flowLimits,
1289
- schedule: (sid) => {
1290
- schedule = true;
1291
- void sid;
1292
- },
1293
- });
1294
- }
1295
- }
1296
- else if (command.type === "cancel") {
1297
- const cancelledTurnId = s.activeTurnId;
1298
- const workflowCancel = isWorkflowManifest(s.manifest);
1299
- const cascade = workflowCancel
1300
- ? planCancelCascade({
1301
- store: this.store,
1302
- workflowSessionId: id,
1303
- turnId: cancelledTurnId,
1304
- })
1305
- : null;
1306
- s.status = "cancelled";
1307
- if (workflowCancel) {
1308
- fenceWorkflowActions({
1309
- store: this.store,
1310
- workflowSessionId: id,
1311
- turnId: cancelledTurnId,
1312
- });
1313
- }
1314
- else {
1315
- for (const a of this.store.all("actions"))
1316
- if (a.sessionId === id &&
1317
- a.turnId === cancelledTurnId &&
1318
- ["pending", "claimed"].includes(a.status)) {
1319
- // Claimed work may already have an external effect. Preserve it for reconciliation.
1320
- a.status = a.status === "claimed" ? "uncertain" : "cancelled";
1321
- this.store.put("actions", a.actionId, a);
1322
- const effect = this.store.get("effects", a.actionId);
1323
- effect.status = a.status;
1324
- this.store.put("effects", a.actionId, effect);
1325
- }
1326
- }
1327
- for (const effect of this.store.all("effects"))
1328
- if (effect.request.sessionId === id &&
1329
- effect.request.turnId === cancelledTurnId &&
1330
- effect.status === "invoking") {
1331
- effect.status = "uncertain";
1332
- effect.error =
1333
- "Turn cancelled before model outcome was durably recorded";
1334
- this.store.put("effects", effect.request.effectId, effect);
1335
- }
1336
- event = this.store.event(id, cancelledTurnId, "turn.cancelled", {
1337
- reason: command.reason,
1338
- });
1339
- s.activeTurnId = null;
1340
- // The next turn starts from the state preceding the cancelled turn, never its paused plan.
1341
- if (cancelledTurnId !== null)
1342
- s.state = s.turnStartState;
1343
- s.checkpoint = undefined;
1344
- s.waits = undefined;
1345
- s.error = undefined;
1346
- // Cascade: cancel linked agent sessions deepest-first (after fencing this session).
1347
- if (cascade) {
1348
- for (const agentId of cascade.agentSessionIds) {
1349
- const agent = this.store.get("sessions", agentId);
1350
- if (!agent ||
1351
- !agent.activeTurnId ||
1352
- ["idle", "completed", "failed", "cancelled"].includes(agent.status))
1353
- continue;
1354
- cascadeCancelIds.push(agentId);
1355
- }
1356
- }
1357
- }
1358
- else {
1359
- if (command.type === "message") {
1360
- if (!["idle", "completed", "failed", "cancelled"].includes(s.status))
1361
- fail(409, "Session has active or unresolved work");
1362
- s.turnStartState = s.state;
1363
- s.activeTurnId = randomUUID();
1364
- if (isWorkflowManifest(s.manifest)) {
1365
- const input = "content" in command
1366
- ? command.content
1367
- : command.data;
1368
- s.checkpoint = createFlowCheckpoint({
1369
- manifest: s.manifest,
1370
- sessionId: id,
1371
- turnId: s.activeTurnId,
1372
- input,
1373
- });
1374
- }
1375
- else {
1376
- // Optional message.manifest: validate as variant, pin by hash, apply this turn only.
1377
- const resolved = resolveMessageManifest({
1378
- pinned: s.manifest,
1379
- pinnedHash: s.manifestHash,
1380
- messageManifest: command.manifest,
1381
- store: variantStore(s),
1382
- });
1383
- if (!resolved.ok) {
1384
- fail(400, resolved.message);
1385
- }
1386
- else {
1387
- rebaseSessionState(s, resolved.hash);
1388
- s.checkpoint = createDurableCheckpoint({
1389
- manifest: resolved.manifest,
1390
- sessionId: id,
1391
- turnId: s.activeTurnId,
1392
- input: "content" in command
1393
- ? command.content
1394
- : typeof command.data === "string"
1395
- ? command.data
1396
- : JSON.stringify(command.data),
1397
- state: s.state,
1398
- info: s.info,
1399
- });
1400
- }
1401
- }
1402
- }
1403
- else {
1404
- if (s.status !== "paused" || !s.state || !s.activeTurnId)
1405
- fail(409, "Session is not awaiting a response");
1406
- const pending = s.state.plan?.calls ?? [];
1407
- if (!pending.some((c) => c.status === "interaction" &&
1408
- c.interaction?.id === command.interactionId &&
1409
- (command.type === "approve") ===
1410
- (c.interaction?.kind === "approval")))
1411
- fail(409, "Unknown interaction");
1412
- const input = command.type === "approve"
1413
- ? {
1414
- kind: "approve",
1415
- interactionId: command.interactionId,
1416
- approved: command.approved,
1417
- }
1418
- : {
1419
- kind: "respond",
1420
- interactionId: command.interactionId,
1421
- value: command.value,
1422
- };
1423
- if (isWorkflowManifest(s.manifest)) {
1424
- // Approvals owned by linked agent sessions must be answered there (WF-R52).
1425
- const conflict = foreignInteractionConflict({
1426
- store: this.store,
1427
- workflowSessionId: id,
1428
- interactionId: command.interactionId,
1429
- });
1430
- if (conflict)
1431
- fail(409, conflict.message);
1432
- // Workflow-owned interactions (tool-node / verify) resume on this session once
1433
- // the flow engine supports pause segments; tracer root Loop has none yet.
1434
- fail(409, "Workflow sessions do not accept approve/respond on the root without a pending interaction");
1435
- }
1436
- s.checkpoint = createDurableCheckpoint({
1437
- manifest: turnManifestOf(s),
1438
- sessionId: id,
1439
- turnId: s.activeTurnId,
1440
- input: input,
1441
- state: s.state,
1442
- info: s.info,
1443
- segment: (s.checkpoint?.segment ?? 0) + 1,
1444
- });
1445
- }
1446
- this.store.put("checkpoints", JSON.stringify([id, s.activeTurnId, s.checkpoint.segment]), { checkpoint: s.checkpoint, status: "runnable" });
1447
- s.status = "runnable";
1448
- s.waits = undefined;
1449
- schedule = true;
1450
- event = this.store.event(id, s.activeTurnId, `command.${command.type}`, command);
1451
- }
1452
- this.store.put("sessions", id, s);
1453
- const response = {
1454
- status: "accepted",
1455
- turnId: s.activeTurnId,
1456
- cursor: event?.cursor ?? this.store.history(id).cursor,
1457
- requestId: command.requestId,
202
+ // Register the handlers, then arm the sweep: its first pass runs at once and re-wakes
203
+ // sessions a previous process left runnable or running (takeover handles the rest).
204
+ let afterOpen = true;
205
+ const worker = {
206
+ advance: (sessionId, signal) => advance(ctx, sessionId, signal),
207
+ sweep: async () => {
208
+ const first = afterOpen;
209
+ afterOpen = false;
210
+ await sweep(ctx, { afterOpen: first });
211
+ },
1458
212
  };
1459
- this.store.put("commands", key, { command, response });
1460
- return response;
1461
- });
1462
- if (event)
1463
- this.publish(event);
1464
- for (const extra of extraEvents)
1465
- this.publish(extra);
1466
- if (input.type === "cancel")
1467
- this.running.get(id)?.abort();
1468
- // Cascade cancel linked agents deepest-first under existing fencing (WF-R53).
1469
- for (const agentId of cascadeCancelIds) {
1470
- try {
1471
- this.command(agentId, {
1472
- type: "cancel",
1473
- requestId: `flow-cascade-${id}-${agentId}`,
1474
- idempotencyKey: `flow-cascade-cancel:${id}:${agentId}`,
1475
- reason: "workflow cancelled",
1476
- }, scope);
1477
- }
1478
- catch {
1479
- /* agent may already be terminal */
1480
- }
213
+ const unregister = workers.register(config.tenantId, worker);
214
+ detach = async () => {
215
+ unregister();
216
+ await local?.stop();
217
+ };
218
+ if (local)
219
+ await local.start(workers.handlers);
220
+ await execution.armSweep(config.tenantId);
221
+ return new TenantRuntime(ctx, envelope, worker, detach, workers.graceMs);
1481
222
  }
1482
- // Direct cancel of a linked agent fails that node with agent.cancelled (WF-R54).
1483
- if (input.type === "cancel") {
1484
- const link = this.store.get("links", id);
1485
- if (link) {
1486
- this.store.tx(() => {
1487
- wakeLinkedWorkflow({
1488
- agentSessionId: id,
1489
- store: this.store,
1490
- cancelled: true,
1491
- error: "Agent turn was cancelled",
1492
- schedule: (sid) => this.schedule(sid),
1493
- publish: (e) => this.publish(e),
1494
- });
1495
- });
1496
- }
223
+ catch (error) {
224
+ await detach?.().catch(() => undefined);
225
+ await wired?.close().catch(() => undefined);
226
+ await store.close().catch(() => undefined);
227
+ throw error;
1497
228
  }
1498
- if (schedule)
1499
- this.schedule(id);
1500
- return response;
1501
229
  }
1502
- sse(request, response, set, whenEmpty) {
1503
- response.writeHead(200, {
1504
- "content-type": "text/event-stream",
1505
- "cache-control": "no-cache",
1506
- connection: "keep-alive",
1507
- "x-accel-buffering": "no",
1508
- });
1509
- response.flushHeaders();
1510
- set.add(response);
1511
- const timer = setInterval(() => this.send(response, ": keepalive\n\n"), 15000);
1512
- timer.unref();
1513
- request.on("close", () => {
1514
- clearInterval(timer);
1515
- set.delete(response);
1516
- if (!set.size)
1517
- whenEmpty?.();
1518
- });
230
+ handle(request, response, url) {
231
+ return handle(this.ctx, request, response, url);
1519
232
  }
1520
- async handle(request, response, _url) {
1521
- const json = (value, status = 200) => {
1522
- const payload = JSON.stringify(value);
1523
- response.writeHead(status, {
1524
- "content-type": "application/json",
1525
- "content-length": Buffer.byteLength(payload),
1526
- });
1527
- response.end(payload);
1528
- };
1529
- try {
1530
- const url = _url ?? new URL(request.url ?? "/", "http://runtime");
1531
- const path = url.pathname
1532
- .split("/")
1533
- .filter(Boolean)
1534
- .map(decodeURIComponent);
1535
- const method = request.method;
1536
- const scope = this.scope(request);
1537
- if (path[0] !== "v1")
1538
- fail(404, "Route not found");
1539
- if (path[1] === "executors" &&
1540
- path[2] === "connect" &&
1541
- method === "GET") {
1542
- if (scope.kind !== "executor")
1543
- return fail(403, "Executor credential required");
1544
- const tokenHash = scope.executor.tokenHash;
1545
- let streams = this.executorStreams.get(tokenHash);
1546
- if (!streams)
1547
- this.executorStreams.set(tokenHash, (streams = new Set()));
1548
- this.sse(request, response, streams, () => this.executorStreams.delete(tokenHash));
1549
- this.send(response, 'event: work_available\ndata: {"type":"work_available"}\n\n');
1550
- return;
1551
- }
1552
- if (path[1] === "actions") {
1553
- if (scope.kind !== "executor")
1554
- return fail(403, "Executor credential required");
1555
- this.expireClaims();
1556
- if (path.length === 2 && method === "GET")
1557
- return json({
1558
- actions: this.store
1559
- .all("actions")
1560
- .filter((a) => a.status === "pending" && a.agentId === scope.executor.agentId),
1561
- });
1562
- const actionId = path[2];
1563
- if (!actionId)
1564
- fail(404, "Action not found");
1565
- if (method === "POST" &&
1566
- path[3] === "sandbox" &&
1567
- path[4] &&
1568
- path.length === 5) {
1569
- const action = this.store.get("actions", actionId) ??
1570
- fail(404, "Action not found");
1571
- this.scoped(scope, action);
1572
- try {
1573
- const outcome = await handleActionSandboxTool(this.sandboxRouteDeps(), actionId, path[4], await this.body(request), requestAborted(request));
1574
- return json(outcome);
1575
- }
1576
- catch (error) {
1577
- if (error instanceof SandboxRouteError)
1578
- fail(error.status, error.message);
1579
- throw error;
1580
- }
1581
- }
1582
- const body = await this.body(request);
1583
- let event;
1584
- const result = this.store.tx(() => {
1585
- const action = this.store.get("actions", actionId) ??
1586
- fail(404, "Action not found");
1587
- this.scoped(scope, action);
1588
- if (method === "POST" && path[3] === "claim") {
1589
- ActionClaimRequestSchema.parse(body);
1590
- if (action.status !== "pending" ||
1591
- this.session(action.sessionId).status === "cancelled" ||
1592
- this.session(action.sessionId).activeTurnId !== action.turnId)
1593
- fail(409, "Action unavailable");
1594
- action.status = "claimed";
1595
- action.generation++;
1596
- action.claimId = randomUUID();
1597
- action.leaseExpiresAt = new Date(Date.now() + (this.config.leaseMs ?? 30000)).toISOString();
1598
- this.store.put("actions", actionId, action);
1599
- event = this.store.event(action.sessionId, action.turnId, "action.claimed", {
1600
- actionId,
1601
- generation: action.generation,
1602
- ...(action.agent ? { agent: action.agent } : {}),
1603
- });
1604
- return {
1605
- action,
1606
- claimId: action.claimId,
1607
- generation: action.generation,
1608
- leaseExpiresAt: action.leaseExpiresAt,
1609
- };
1610
- }
1611
- if (method === "POST" && path[3] === "heartbeat") {
1612
- const beat = ActionHeartbeatRequestSchema.parse(body);
1613
- if (this.session(action.sessionId).activeTurnId !== action.turnId ||
1614
- action.status !== "claimed" ||
1615
- action.claimId !== beat.claimId ||
1616
- action.generation !== beat.generation ||
1617
- Date.parse(action.leaseExpiresAt) <= Date.now())
1618
- fail(409, "Stale or expired claim");
1619
- action.leaseExpiresAt = new Date(Date.now() + (this.config.leaseMs ?? 30000)).toISOString();
1620
- this.store.put("actions", actionId, action);
1621
- return { leaseExpiresAt: action.leaseExpiresAt };
1622
- }
1623
- return fail(404, "Route not found");
1624
- });
1625
- if (event)
1626
- this.publish(event);
1627
- return json(result);
1628
- }
1629
- if (path[1] === "sessions" &&
1630
- path[2] &&
1631
- path[3] === "commands" &&
1632
- method === "POST")
1633
- return json(this.command(path[2], SessionCommandSchema.parse(await this.body(request)), scope));
1634
- if (path[1] === "vaults")
1635
- return json(await this.dispatchVault(scope, method, path, url, request));
1636
- if (path[1] === "tenant")
1637
- return json(await this.dispatchTenant(scope, method, path, request));
1638
- const principalId = this.requireApplication(scope);
1639
- if (path[1] === "executors" && path.length === 2 && method === "GET")
1640
- return json({
1641
- executors: this.registry.list().map((e) => ({
1642
- agentId: e.agentId,
1643
- implementationVersion: e.implementationVersion,
1644
- ...(e.manifestHash === undefined
1645
- ? {}
1646
- : { manifestHash: e.manifestHash }),
1647
- connected: (this.executorStreams.get(e.tokenHash)?.size ?? 0) > 0,
1648
- updatedAt: e.updatedAt,
1649
- })),
1650
- });
1651
- // The length guard matters: the connect branch above only matches GET, so without it a
1652
- // PUT to /v1/executors/connect would register an agent literally named "connect".
1653
- if (path[1] === "executors" && path.length === 2 && method === "PUT") {
1654
- const body = RegisterExecutorsRequestSchema.parse(await this.body(request));
1655
- // Credential rules are shared with startup validation, which throws plainly; on the
1656
- // wire a rejected registration is a bad request, not a server fault.
1657
- const applicationHashes = applicationTokenHashes(this.store.db);
1658
- try {
1659
- for (const executor of body.executors)
1660
- assertExecutorCredential(executor, applicationHashes);
1661
- }
1662
- catch (error) {
1663
- fail(400, error instanceof Error ? error.message : String(error));
1664
- }
1665
- if (new Set(body.executors.map((e) => e.agentId)).size !==
1666
- body.executors.length ||
1667
- new Set(body.executors.map((e) => e.token)).size !==
1668
- body.executors.length)
1669
- fail(400, "Executor registrations must be unique per agent and token");
1670
- const updatedAt = new Date().toISOString();
1671
- const records = body.executors.map((executor) => ({
1672
- agentId: executor.agentId,
1673
- implementationVersion: executor.implementationVersion,
1674
- ...(executor.manifestHash === undefined
1675
- ? {}
1676
- : { manifestHash: executor.manifestHash }),
1677
- tokenHash: hashToken(executor.token),
1678
- persisted: true,
1679
- updatedAt,
1680
- principalId,
1681
- }));
1682
- for (const record of records) {
1683
- const collision = this.registry.find(record.tokenHash);
1684
- if (collision && collision.agentId !== record.agentId)
1685
- fail(409, "Executor tokens must be unique");
1686
- // A token hash may not appear in both principals and executors (D3).
1687
- if (applicationHashes.includes(record.tokenHash))
1688
- fail(400, "Executor tokens require independent credentials and an agent id");
1689
- }
1690
- // Persist the whole batch first; the in-memory registry must never run ahead of SQLite.
1691
- this.store.tx(() => {
1692
- for (const record of records)
1693
- this.store.putExecutor({
1694
- agentId: record.agentId,
1695
- tokenHash: record.tokenHash,
1696
- implementationVersion: record.implementationVersion,
1697
- ...(record.manifestHash === undefined
1698
- ? {}
1699
- : { manifestHash: record.manifestHash }),
1700
- principalId: record.principalId,
1701
- updatedAt,
1702
- });
1703
- });
1704
- const results = records.map((record) => {
1705
- const previous = this.registry.get(record.agentId);
1706
- const { rotated, previousHash } = this.registry.upsert(record);
1707
- if (rotated && previousHash) {
1708
- for (const r of this.executorStreams.get(previousHash) ?? [])
1709
- r.end();
1710
- this.executorStreams.delete(previousHash);
1711
- }
1712
- const replacedByDifferent = previous?.principalId !== undefined &&
1713
- previous.principalId !== principalId;
1714
- if (replacedByDifferent)
1715
- this.config.logger.warn("executor registration replaced by different application credential", {
1716
- agentId: record.agentId,
1717
- previousPrincipalId: previous.principalId,
1718
- });
1719
- return {
1720
- agentId: record.agentId,
1721
- implementationVersion: record.implementationVersion,
1722
- rotated,
1723
- ...(replacedByDifferent
1724
- ? { replacedBy: "different-credential" }
1725
- : {}),
1726
- };
1727
- });
1728
- return json({ executors: results });
1729
- }
1730
- if (path[1] === "executors" &&
1731
- path.length === 3 &&
1732
- path[2] &&
1733
- method === "DELETE") {
1734
- const agentId = path[2];
1735
- const existing = this.registry.get(agentId) ?? fail(404, "Executor not found");
1736
- this.store.tx(() => this.store.deleteExecutor(agentId));
1737
- this.registry.remove(agentId);
1738
- for (const r of this.executorStreams.get(existing.tokenHash) ?? [])
1739
- r.end();
1740
- this.executorStreams.delete(existing.tokenHash);
1741
- return json({ agentId, deleted: true });
1742
- }
1743
- if (path[1] === "agents" && path.length === 2 && method === "GET")
1744
- return json({
1745
- agents: this.store.all("definitions").map((d) => ({
1746
- agentId: d.manifest.id,
1747
- manifest: d.manifest,
1748
- manifestHash: d.manifestHash,
1749
- implementationVersion: d.implementationVersion,
1750
- })),
1751
- });
1752
- if (path[1] === "sessions" && path.length === 2 && method === "GET")
1753
- return json({
1754
- sessions: this.store
1755
- .all("sessions")
1756
- .filter((s) => !url.searchParams.has("agentId") ||
1757
- s.agentId === url.searchParams.get("agentId"))
1758
- .map((s) => ({
1759
- id: s.id,
1760
- agentId: s.agentId,
1761
- ownerUserId: s.ownerUserId,
1762
- status: s.status,
1763
- activeTurnId: s.activeTurnId,
1764
- })),
1765
- });
1766
- if (path[1] === "agents" &&
1767
- path[2] &&
1768
- path.length === 3 &&
1769
- method === "PUT") {
1770
- const body = PutAgentRequestSchema.parse(await this.body(request));
1771
- if (body.manifest.id !== path[2])
1772
- fail(400, "Agent id mismatch");
1773
- DefinitionDocumentSchema.parse(body.manifest);
1774
- const definition = {
1775
- ...body,
1776
- manifestHash: hashManifest(body.manifest),
1777
- };
1778
- this.store.tx(() => {
1779
- this.store.put("definitions", path[2], definition);
1780
- });
1781
- return json({
1782
- agentId: path[2],
1783
- manifestHash: definition.manifestHash,
1784
- implementationVersion: body.implementationVersion,
1785
- });
1786
- }
1787
- if (path[1] === "sessions" && path[2]) {
1788
- const id = path[2];
1789
- if (method === "PUT" && path.length === 3) {
1790
- const body = PutSessionRequestSchema.parse(await this.body(request));
1791
- const vaultIds = body.vaultIds ?? [];
1792
- const credentialSelections = body.credentialSelections ?? [];
1793
- const prior = this.store.get("sessions", id);
1794
- const definition = prior === undefined || body.sandbox
1795
- ? this.store.get("definitions", body.agentId) ??
1796
- fail(404, "Definition not found")
1797
- : undefined;
1798
- const sandboxOwnerId = body.sandbox
1799
- ? validateSandboxAttach(body, (definition ?? prior).manifest, (sid) => this.store.get("sessions", sid))
1800
- : undefined;
1801
- const result = this.store.tx(() => {
1802
- this.vault.assertAttachment(body.ownerUserId, vaultIds, credentialSelections);
1803
- const existing = this.store.get("sessions", id);
1804
- if (existing) {
1805
- if (sessionIdentity(existing.creation) !== sessionIdentity(body))
1806
- fail(409, "Session already exists with different creation parameters");
1807
- existing.vaultIds = vaultIds;
1808
- existing.credentialSelections = credentialSelections;
1809
- existing.creation = body;
1810
- if (sandboxOwnerId !== undefined)
1811
- existing.sandboxOwnerId = sandboxOwnerId;
1812
- this.store.put("sessions", id, existing);
1813
- this.vault.recordAttachment(id, vaultIds);
1814
- return existing;
1815
- }
1816
- const created = {
1817
- id,
1818
- agentId: body.agentId,
1819
- ownerUserId: body.ownerUserId,
1820
- manifest: definition.manifest,
1821
- manifestHash: definition.manifestHash,
1822
- implementationVersion: definition.implementationVersion,
1823
- info: body.info,
1824
- status: "idle",
1825
- activeTurnId: null,
1826
- creation: body,
1827
- vaultIds,
1828
- credentialSelections,
1829
- pluginRoots: definition.pluginRoots ?? {},
1830
- ...(sandboxOwnerId !== undefined ? { sandboxOwnerId } : {}),
1831
- };
1832
- this.store.put("sessions", id, created);
1833
- this.vault.recordAttachment(id, vaultIds);
1834
- return created;
1835
- });
1836
- return json(this.view(result));
1837
- }
1838
- const s = this.session(id);
1839
- if (method === "GET" && path.length === 3)
1840
- return json(this.view(s));
1841
- const cursor = url.searchParams.get("cursor") ??
1842
- (typeof request.headers["last-event-id"] === "string"
1843
- ? request.headers["last-event-id"]
1844
- : undefined);
1845
- if (method === "GET" && path[3] === "items")
1846
- return json(this.store.history(id, cursor, url.searchParams.get("agent") ?? undefined));
1847
- if (method === "GET" && path[3] === "events") {
1848
- const history = this.store.history(id, cursor);
1849
- const set = this.observers.get(id) ?? new Set();
1850
- this.observers.set(id, set);
1851
- this.sse(request, response, set);
1852
- for (const event of history.items)
1853
- this.send(response, `id: ${event.cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
1854
- return;
1855
- }
1856
- if (method === "POST" &&
1857
- path[3] === "sandbox" &&
1858
- path[4] &&
1859
- path.length === 5) {
1860
- this.requireApplication(scope);
1861
- try {
1862
- const outcome = await handleSessionSandboxTool(this.sandboxRouteDeps(), id, path[4], await this.body(request), requestAborted(request));
1863
- return json(outcome);
1864
- }
1865
- catch (error) {
1866
- if (error instanceof SandboxRouteError)
1867
- fail(error.status, error.message);
1868
- throw error;
1869
- }
1870
- }
1871
- }
1872
- fail(404, "Route not found");
1873
- }
1874
- catch (error) {
1875
- if (response.headersSent) {
1876
- response.end();
1877
- return;
1878
- }
1879
- if (error instanceof OpaqueAuthError) {
1880
- json(error.body, error.status);
1881
- return;
1882
- }
1883
- const status = error instanceof HttpError ||
1884
- error instanceof VaultError ||
1885
- error instanceof SandboxRouteError
1886
- ? error.status
1887
- : error?.name === "ZodError" ||
1888
- error?.message === "Invalid cursor"
1889
- ? 400
1890
- : 500;
1891
- json({
1892
- status: "rejected",
1893
- code: status === 500 ? "internal_error" : "request_rejected",
1894
- message: status === 500
1895
- ? "Runtime request failed"
1896
- : error.message,
1897
- }, status);
1898
- }
233
+ authorize(sessionId, request) {
234
+ return authorize(this.ctx, sessionId, request);
1899
235
  }
1900
- view(s) {
236
+ async summary() {
237
+ const { store, live } = this.ctx;
238
+ const { counts, outbox } = await store.tx(async (t) => ({
239
+ counts: await t.counts(),
240
+ outbox: await t.outboxStats(),
241
+ }));
1901
242
  return {
1902
- id: s.id,
1903
- agentId: s.agentId,
1904
- ownerUserId: s.ownerUserId,
1905
- manifestHash: s.manifestHash,
1906
- implementationVersion: s.implementationVersion,
1907
- status: s.status,
1908
- activeTurnId: s.activeTurnId,
1909
- vaultIds: s.vaultIds ?? [],
1910
- credentialSelections: s.credentialSelections ?? [],
1911
- sandboxOwnerId: s.sandboxOwnerId ?? null,
1912
- mcpSnapshot: s.mcpSnapshot ?? null,
1913
- mcpDiagnostics: s.mcpDiagnostics ?? [],
1914
- waits: (() => {
1915
- if (isWorkflowManifest(s.manifest)) {
1916
- const aggregated = aggregateWaits({
1917
- store: this.store,
1918
- workflowSessionId: s.id,
1919
- });
1920
- if (aggregated.length > 0)
1921
- return aggregated;
1922
- }
1923
- return Array.isArray(s.waits)
1924
- ? s.waits.map((call) => ({
1925
- invocationId: call.invocationId,
1926
- interaction: call.interaction,
1927
- wait: call.wait,
1928
- status: call.status,
1929
- }))
1930
- : s.waits;
1931
- })(),
1932
- error: s.error,
1933
- actions: this.store
1934
- .all("actions")
1935
- .filter((a) => a.sessionId === s.id &&
1936
- !["completed", "cancelled"].includes(a.status)),
1937
- uncertainEffects: this.store
1938
- .all("effects")
1939
- .filter((e) => e.request.sessionId === s.id && e.status === "uncertain")
1940
- .map((e) => ({
1941
- effectId: e.request.effectId,
1942
- turnId: e.request.turnId,
1943
- kind: e.request.kind,
1944
- error: e.error,
1945
- })),
243
+ ready: !this.ctx.closing && !this.ctx.closed,
244
+ runningSessions: counts.runningSessions,
245
+ connectedExecutors: connectedExecutorCount(live),
246
+ pendingActions: counts.pendingActions,
247
+ uncertainEffects: counts.uncertainEffects,
248
+ outboxDepth: outbox.depth,
249
+ relayLagMs: outbox.oldestCreatedAt === null
250
+ ? 0
251
+ : Math.max(0, Date.now() - Date.parse(outbox.oldestCreatedAt)),
1946
252
  };
1947
253
  }
1948
- async prepareMcp(id, signal) {
1949
- const s = this.session(id);
1950
- if (serversOf(s.manifest).length === 0)
1951
- return;
1952
- if (!s.mcpSnapshot) {
1953
- const found = await this.mcp.discover({
1954
- sessionId: id,
1955
- manifest: s.manifest,
1956
- manifestHash: s.manifestHash,
1957
- pluginRoots: s.pluginRoots ?? {},
1958
- signal,
1959
- });
1960
- this.store.tx(() => {
1961
- const current = this.session(id);
1962
- if (current.mcpSnapshot)
1963
- return;
1964
- current.mcpSnapshot = found.snapshot;
1965
- current.mcpDiagnostics = found.diagnostics;
1966
- this.store.put("sessions", id, current);
1967
- });
1968
- return;
1969
- }
1970
- const diagnostics = await this.mcp.reconnect({
1971
- sessionId: id,
1972
- manifest: s.manifest,
1973
- pluginRoots: s.pluginRoots ?? {},
1974
- tools: s.mcpSnapshot.mcpTools,
1975
- signal,
1976
- });
1977
- if (diagnostics.length === 0)
1978
- return;
1979
- this.store.tx(() => {
1980
- const current = this.session(id);
1981
- const prior = [...(current.mcpDiagnostics ?? [])];
1982
- for (const item of diagnostics) {
1983
- const index = prior.findIndex((existing) => existing.capabilityId === item.capabilityId &&
1984
- existing.serverName === item.serverName);
1985
- if (index >= 0)
1986
- prior[index] = item;
1987
- else
1988
- prior.push(item);
1989
- }
1990
- current.mcpDiagnostics = prior;
1991
- this.store.put("sessions", id, current);
1992
- });
1993
- }
1994
- async callMcpTool(request) {
1995
- const s = this.session(request.sessionId);
1996
- const tool = mcpToolOf(s, request);
1997
- if (!tool)
1998
- throw new Error(`MCP tool '${request.toolName ?? ""}' is not in the session snapshot`);
1999
- return this.mcp.call({
2000
- sessionId: s.id,
2001
- ...(tool.agentId === undefined ? {} : { agentId: tool.agentId }),
2002
- capabilityId: tool.capabilityId,
2003
- serverName: tool.serverName,
2004
- serverToolName: tool.serverToolName,
2005
- args: request.input,
2006
- manifest: s.manifest,
2007
- pluginRoots: s.pluginRoots ?? {},
2008
- });
2009
- }
2010
- async callSandboxTool(request, signal) {
2011
- const s = this.session(request.sessionId);
2012
- // Agents used as tools share the session's sandbox; the tree declares one sandbox spec.
2013
- const capability = sandboxCapabilityOf(manifestFor(s.manifest, request.agent), request.capabilityId, request.toolName);
2014
- if (!capability)
2015
- throw new Error(`'${request.toolName ?? ""}' is not a sandbox tool`);
2016
- const ownerId = owningSandboxSessionId(s, (id) => this.store.get("sessions", id));
2017
- return this.sandbox.run({ id: ownerId, activeTurnId: s.activeTurnId, manifest: s.manifest }, capability, request.toolName, request.input, signal);
2018
- }
2019
- async authorize(sessionId, request) {
2020
- const s = this.session(sessionId);
2021
- const result = await this.vault.authorize({
2022
- sessionId,
2023
- vaultIds: s.vaultIds ?? [],
2024
- credentialSelections: s.credentialSelections ?? [],
2025
- url: request.url,
2026
- serverName: request.serverName,
2027
- });
2028
- if (result.status === "authorized") {
2029
- const token = result.headers.authorization.slice("Bearer ".length);
2030
- scrub({ authorization: result.headers.authorization, url: result.url }, [
2031
- token,
2032
- ]);
2033
- }
2034
- return result;
2035
- }
2036
- ensureKek() {
2037
- if (this.kek)
2038
- return this.kek;
2039
- if (!this.createKekIfMissing)
2040
- throw new QuarantineError("kek-missing", "Vault key-encryption key is required", "restore the vault-kek file beside tenant.sqlite");
2041
- this.kek = createKekFile(this.kekPath);
2042
- return this.kek;
254
+ drain(activeWork, timeoutMs = 30_000) {
255
+ return drain(this.ctx, activeWork, timeoutMs);
2043
256
  }
2044
- async dispatchTenant(scope, method, path, request) {
2045
- if (scope.kind !== "application") {
2046
- this.vault.reject(path.join("/"));
2047
- fail(403, "Application credential required");
2048
- }
2049
- if (path.length === 2 && method === "GET")
2050
- return buildTenantStatus({
2051
- envelope: this.envelope,
2052
- config: this.config,
2053
- store: this.store,
2054
- registry: this.registry,
2055
- vault: this.vault,
2056
- sandbox: this.sandbox,
2057
- closing: this.closing || this.closed,
2058
- modelConfigured: this.useVaultModel
2059
- ? this.vault.getHostModel().configured
2060
- : true,
2061
- executorStreams: this.executorStreams,
2062
- });
2063
- if (path[2] === "reset" && path.length === 3 && method === "POST") {
2064
- const body = ResetTenantRequestSchema.parse(await this.body(request));
2065
- await this.drain(body.activeWork);
2066
- await resetTenant({
2067
- store: this.store,
2068
- registry: this.registry,
2069
- sandbox: this.sandbox,
2070
- paths: this.config.paths,
2071
- clearSessionState: () => {
2072
- this.pending.clear();
2073
- for (const c of this.running.values())
2074
- c.abort();
2075
- this.running.clear();
2076
- for (const set of this.observers.values())
2077
- for (const r of set)
2078
- r.end();
2079
- this.observers.clear();
2080
- },
2081
- clearExecutorStreams: () => {
2082
- for (const streams of this.executorStreams.values())
2083
- for (const r of streams)
2084
- r.end();
2085
- this.executorStreams.clear();
2086
- },
2087
- }, body.scope);
2088
- // Reset leaves the Tenant open for new work.
2089
- this.closing = false;
2090
- return { ok: true };
2091
- }
2092
- if (path[2] === "config" &&
2093
- path[3] === "seed" &&
2094
- path.length === 4 &&
2095
- method === "PUT") {
2096
- const body = SeedTenantConfigRequestSchema.parse(await this.body(request));
2097
- return seedTenantConfig({ store: this.store, vault: this.vault }, body);
2098
- }
2099
- if (path[2] === "models" && path.length === 3 && method === "GET")
2100
- return hostModelCatalog();
2101
- if (path[2] === "sandbox" && path.length === 3 && method === "GET")
2102
- return this.sandbox.report();
2103
- if (path[2] === "providers" && path.length === 3 && method === "GET")
2104
- return this.vault.listHostProviders();
2105
- if (path[2] === "model" && path.length === 3 && method === "GET")
2106
- return this.vault.getHostModel();
2107
- if (path[2] === "model" && path.length === 3 && method === "PUT")
2108
- return this.vault.putHostModel(PutHostModelRequestSchema.parse(await this.body(request)));
2109
- if (path[2] === "model" &&
2110
- path[3] === "selection" &&
2111
- path.length === 4 &&
2112
- method === "PUT")
2113
- return this.vault.selectHostModel(SelectHostModelRequestSchema.parse(await this.body(request)));
2114
- fail(404, "Route not found");
2115
- }
2116
- async dispatchVault(scope, method, path, url, request) {
2117
- if (scope.kind !== "application") {
2118
- this.vault.reject(path.join("/"));
2119
- fail(403, "Application credential required");
2120
- }
2121
- if (path.length === 2 && method === "POST") {
2122
- const body = CreateVaultRequestSchema.parse(await this.body(request));
2123
- return this.vault.createVault(body);
2124
- }
2125
- if (path.length === 2 && method === "GET") {
2126
- const ownerUserId = url.searchParams.get("ownerUserId") ??
2127
- fail(400, "ownerUserId is required");
2128
- return { vaults: this.vault.listVaults(ownerUserId) };
2129
- }
2130
- const vaultId = path[2];
2131
- if (!vaultId)
2132
- fail(404, "Vault not found");
2133
- if (path.length === 3 && method === "GET")
2134
- return this.vault.getVault(vaultId);
2135
- if (path.length === 3 && method === "DELETE")
2136
- return this.vault.deleteVault(vaultId);
2137
- if (path[3] !== "credentials")
2138
- fail(404, "Route not found");
2139
- if (path.length === 4 && method === "POST") {
2140
- const body = CreateCredentialRequestSchema.parse(await this.body(request));
2141
- return this.vault.createCredential(vaultId, body);
2142
- }
2143
- if (path.length === 4 && method === "GET")
2144
- return { credentials: this.vault.listCredentials(vaultId) };
2145
- const credentialId = path[4];
2146
- if (!credentialId)
2147
- fail(404, "Credential not found");
2148
- if (path.length === 5 && method === "GET")
2149
- return this.vault.getCredential(vaultId, credentialId);
2150
- if (path.length === 5 && method === "POST") {
2151
- const body = RotateCredentialRequestSchema.parse(await this.body(request));
2152
- return this.vault.rotateCredential(vaultId, credentialId, body);
2153
- }
2154
- if (path.length === 5 && method === "DELETE")
2155
- return this.vault.deleteCredential(vaultId, credentialId);
2156
- fail(404, "Route not found");
257
+ /**
258
+ * Aborts the advance of `sessionId` if it runs on this process. Cancel calls it after
259
+ * committing `cancelled`; the control stream calls it for cancels made elsewhere.
260
+ */
261
+ abortLocal(sessionId) {
262
+ this.ctx.abortLocal(sessionId);
2157
263
  }
2158
- summary() {
2159
- const sessions = this.store.all("sessions");
2160
- const runningSessions = sessions.filter((sess) => sess.status === "running" || sess.status === "runnable").length;
2161
- const connectedExecutors = [...this.executorStreams.values()].filter((set) => set.size > 0).length;
2162
- const pendingActions = this.store
2163
- .all("actions")
2164
- .filter((a) => a.status === "pending" || a.status === "claimed").length;
2165
- const uncertainEffects = this.store
2166
- .all("effects")
2167
- .filter((e) => e.status === "uncertain").length;
2168
- return {
2169
- ready: !this.closing && !this.closed,
2170
- runningSessions,
2171
- connectedExecutors,
2172
- pendingActions,
2173
- uncertainEffects,
2174
- };
264
+ /**
265
+ * The Durable Streams seam's status for Tenant status and readiness: S2 reachability, the
266
+ * basin, outbox depth and relay lag (`streamsStatus` in `tenant/streams.ts`).
267
+ */
268
+ streamsStatus() {
269
+ return streamsStatus(this.ctx);
2175
270
  }
2176
- async drain(activeWork, timeoutMs = 30_000) {
2177
- this.closing = true;
2178
- if (activeWork === "cancel") {
2179
- for (const sess of this.store.all("sessions"))
2180
- if (sess.activeTurnId &&
2181
- ["running", "runnable", "paused"].includes(sess.status))
2182
- this.command(sess.id, {
2183
- type: "cancel",
2184
- requestId: randomUUID(),
2185
- idempotencyKey: `drain-cancel-${sess.id}-${sess.activeTurnId}`,
2186
- reason: "tenant drain cancel",
2187
- }, {
2188
- kind: "application",
2189
- principalId: "drain",
2190
- });
2191
- }
2192
- const deadline = Date.now() + timeoutMs;
2193
- while (this.running.size > 0 && Date.now() < deadline)
2194
- await new Promise((resolve) => setTimeout(resolve, 10));
271
+ /** Adds a callback to the Tenant sweep. Returns a function that removes it. */
272
+ onSweep(hook) {
273
+ return this.ctx.onSweep(hook);
2195
274
  }
2196
275
  async close() {
2197
- this.closing = true;
2198
- clearInterval(this.timer);
2199
- for (const c of this.running.values())
2200
- c.abort();
2201
- await this.mcp.close();
2202
- for (const streams of this.executorStreams.values())
2203
- for (const r of streams)
2204
- r.end();
2205
- for (const set of this.observers.values())
2206
- for (const r of set)
2207
- r.end();
2208
- while (this.running.size)
2209
- await new Promise((resolve) => setTimeout(resolve, 10));
2210
- await this.sandbox.close();
2211
- this.closed = true;
2212
- this.store.db.close();
2213
- if (this.lockPath && existsSync(this.lockPath))
2214
- unlinkSync(this.lockPath);
276
+ const ctx = this.ctx;
277
+ ctx.closing = true;
278
+ // A shutdown abort: the advances leave their sessions to the next advance, unsettled.
279
+ abortAll(ctx, "shutdown");
280
+ const idleBy = Date.now() + this.closeGraceMs;
281
+ await this.detach();
282
+ await ctx.mcp.close();
283
+ endAllStreams(ctx.live);
284
+ // Bounded: an advance that ignores its abort is abandoned; its lease lapses (§11.4).
285
+ await waitForIdle(ctx, Math.max(0, idleBy - Date.now()));
286
+ await ctx.sandbox.close();
287
+ ctx.closed = true;
288
+ await closeStreams(ctx);
289
+ await ctx.store.close();
2215
290
  }
2216
291
  }
2217
- function readEnvelope(config) {
2218
- if (existsSync(config.paths.envelope)) {
2219
- return JSON.parse(readFileSync(config.paths.envelope, "utf8"));
2220
- }
2221
- const now = new Date().toISOString();
2222
- return {
2223
- id: config.tenantId,
2224
- name: config.tenantId,
2225
- createdAt: now,
2226
- updatedAt: now,
2227
- schemaVersion: 1,
2228
- };
2229
- }
2230
292
  /**
2231
293
  * Open a Tenant Runtime from an explicit `TenantConfig`. No listen() (A1).
2232
294
  * Implements `OpenTenantRuntime`.
2233
295
  */
2234
- export async function openTenantRuntime(config,
2235
- // TENANTS-CCR: optional hooks for tests until TenantConfig grows them
2236
- hooks) {
2237
- return TenantRuntime.open(config, hooks ?? {});
2238
- }
2239
- /** What an action runs, for events: a tool name, or a hook point and its capabilities. */
2240
- function actionTarget(action) {
2241
- if (action.kind === "hook")
2242
- return {
2243
- hook: action.hook,
2244
- ...(action.agent ? { agent: action.agent } : {}),
2245
- };
2246
- if (action.kind === "tool" && "toolName" in action)
2247
- return {
2248
- toolName: action.toolName,
2249
- ...(action.agent ? { agent: action.agent } : {}),
2250
- };
2251
- return {
2252
- path: action.path,
2253
- key: action.key,
2254
- ...(action.agent ? { agent: action.agent } : {}),
2255
- };
296
+ export async function openTenantRuntime(config, hooks) {
297
+ return TenantRuntime.open(config, hooks);
2256
298
  }