@nylorun/runtime 0.10.0-beta → 0.11.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/README.md +53 -51
- package/dist/adapters/execution/restate.d.ts +130 -0
- package/dist/adapters/execution/restate.js +442 -0
- package/dist/adapters/sandbox/virtual.js +1 -1
- package/dist/adapters/streams/s2.d.ts +18 -0
- package/dist/adapters/streams/s2.js +361 -0
- package/dist/core/executors.d.ts +4 -1
- package/dist/core/executors.js +20 -1
- package/dist/core/flow-host.d.ts +93 -132
- package/dist/core/flow-host.js +282 -547
- package/dist/core/sandbox-routes.d.ts +8 -3
- package/dist/core/sandbox-routes.js +10 -10
- package/dist/execution/memory.d.ts +54 -0
- package/dist/execution/memory.js +217 -0
- package/dist/execution/types.d.ts +125 -0
- package/dist/execution/types.js +52 -0
- package/dist/host/config-for.d.ts +2 -1
- package/dist/host/config-for.js +3 -21
- package/dist/host/config.d.ts +0 -8
- package/dist/host/create-host.d.ts +36 -1
- package/dist/host/create-host.js +48 -24
- package/dist/host/execution.d.ts +61 -0
- package/dist/host/execution.js +69 -0
- package/dist/host/http.d.ts +4 -0
- package/dist/host/http.js +6 -0
- package/dist/host/index.d.ts +3 -1
- package/dist/host/index.js +2 -0
- package/dist/host/main.js +106 -47
- package/dist/host/stack-config.d.ts +72 -0
- package/dist/host/stack-config.js +193 -0
- package/dist/index.d.ts +1 -1
- package/dist/infra/database.d.ts +16 -0
- package/dist/infra/database.js +31 -0
- package/dist/infra/execution.d.ts +39 -0
- package/dist/infra/execution.js +68 -0
- package/dist/infra/index.d.ts +42 -0
- package/dist/infra/index.js +56 -0
- package/dist/infra/readiness.d.ts +33 -0
- package/dist/infra/readiness.js +53 -0
- package/dist/infra/streams.d.ts +17 -0
- package/dist/infra/streams.js +33 -0
- package/dist/infra/workers.d.ts +39 -0
- package/dist/infra/workers.js +31 -0
- package/dist/model/pi-model.d.ts +2 -2
- package/dist/model/pi-model.js +5 -5
- package/dist/sandbox/manager.d.ts +26 -8
- package/dist/sandbox/manager.js +95 -47
- package/dist/sandbox/select.d.ts +1 -1
- package/dist/sandbox/select.js +4 -5
- package/dist/sandbox/types.d.ts +2 -2
- package/dist/store/canonical.d.ts +5 -0
- package/dist/store/canonical.js +17 -0
- package/dist/store/cursor.d.ts +9 -0
- package/dist/store/cursor.js +18 -0
- package/dist/store/memory.d.ts +57 -0
- package/dist/store/memory.js +643 -0
- package/dist/store/ownership.d.ts +19 -0
- package/dist/store/ownership.js +33 -0
- package/dist/store/postgres/connect.d.ts +25 -0
- package/dist/store/postgres/connect.js +25 -0
- package/dist/store/postgres/locking.d.ts +16 -0
- package/dist/store/postgres/locking.js +22 -0
- package/dist/store/postgres/migrations/001_initial.d.ts +27 -0
- package/dist/store/postgres/migrations/001_initial.js +196 -0
- package/dist/store/postgres/migrations/index.d.ts +49 -0
- package/dist/store/postgres/migrations/index.js +75 -0
- package/dist/store/postgres/names.d.ts +11 -0
- package/dist/store/postgres/names.js +42 -0
- package/dist/store/postgres/store.d.ts +11 -0
- package/dist/store/postgres/store.js +790 -0
- package/dist/store/postgres/tenants.d.ts +90 -0
- package/dist/store/postgres/tenants.js +207 -0
- package/dist/store/types.d.ts +440 -0
- package/dist/store/types.js +10 -0
- package/dist/streams/basin.d.ts +23 -0
- package/dist/streams/basin.js +51 -0
- package/dist/streams/memory.d.ts +25 -0
- package/dist/streams/memory.js +124 -0
- package/dist/streams/relay.d.ts +37 -0
- package/dist/streams/relay.js +254 -0
- package/dist/streams/types.d.ts +115 -0
- package/dist/streams/types.js +86 -0
- package/dist/tenant/actions.d.ts +153 -0
- package/dist/tenant/actions.js +191 -0
- package/dist/tenant/advance.d.ts +15 -0
- package/dist/tenant/advance.js +439 -0
- package/dist/tenant/auth.d.ts +11 -0
- package/dist/tenant/auth.js +34 -0
- package/dist/tenant/commands.d.ts +3 -0
- package/dist/tenant/commands.js +384 -0
- package/dist/tenant/context.d.ts +140 -0
- package/dist/tenant/context.js +40 -0
- package/dist/tenant/effects.d.ts +20 -0
- package/dist/tenant/effects.js +539 -0
- package/dist/tenant/envelope.d.ts +0 -3
- package/dist/tenant/envelope.js +2 -34
- package/dist/tenant/ephemeral.d.ts +15 -4
- package/dist/tenant/ephemeral.js +106 -38
- package/dist/tenant/http.d.ts +32 -0
- package/dist/tenant/http.js +51 -0
- package/dist/tenant/live.d.ts +92 -0
- package/dist/tenant/live.js +290 -0
- package/dist/tenant/model-setting.d.ts +16 -0
- package/dist/tenant/model-setting.js +12 -0
- package/dist/tenant/module.d.ts +29 -8
- package/dist/tenant/module.js +162 -158
- package/dist/tenant/paths.d.ts +12 -8
- package/dist/tenant/paths.js +38 -12
- package/dist/tenant/principals.d.ts +9 -13
- package/dist/tenant/principals.js +31 -38
- package/dist/tenant/quarantine-error.d.ts +1 -1
- package/dist/tenant/quarantine-error.js +2 -8
- package/dist/tenant/quarantine.d.ts +0 -8
- package/dist/tenant/quarantine.js +8 -30
- package/dist/tenant/reset.d.ts +7 -6
- package/dist/tenant/reset.js +7 -51
- package/dist/tenant/routes-tenant.d.ts +12 -0
- package/dist/tenant/routes-tenant.js +126 -0
- package/dist/tenant/routes.d.ts +9 -0
- package/dist/tenant/routes.js +142 -0
- package/dist/tenant/runtime.d.ts +59 -58
- package/dist/tenant/runtime.js +243 -2201
- package/dist/tenant/scheduler.d.ts +25 -0
- package/dist/tenant/scheduler.js +84 -0
- package/dist/tenant/session.d.ts +60 -0
- package/dist/tenant/session.js +94 -0
- package/dist/tenant/sessions.d.ts +34 -0
- package/dist/tenant/sessions.js +149 -0
- package/dist/tenant/status.d.ts +13 -4
- package/dist/tenant/status.js +76 -41
- package/dist/tenant/store-memory.d.ts +9 -12
- package/dist/tenant/store-memory.js +79 -68
- package/dist/tenant/store-pg.d.ts +15 -0
- package/dist/tenant/store-pg.js +140 -0
- package/dist/tenant/streams.d.ts +126 -0
- package/dist/tenant/streams.js +419 -0
- package/dist/tenant/sweep.d.ts +15 -0
- package/dist/tenant/sweep.js +122 -0
- package/dist/tenant/types.d.ts +95 -15
- package/dist/tenant/types.js +23 -1
- package/dist/tenant/worker.d.ts +84 -0
- package/dist/tenant/worker.js +189 -0
- package/dist/vault/service.d.ts +37 -31
- package/dist/vault/service.js +439 -359
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +13 -13
- package/dist/adapters/sandbox/microsandbox.d.ts +0 -2
- package/dist/adapters/sandbox/microsandbox.js +0 -238
- package/dist/core/store.d.ts +0 -32
- package/dist/core/store.js +0 -143
- package/dist/launcher/commands.d.ts +0 -24
- package/dist/launcher/commands.js +0 -260
- package/dist/launcher/ensure-config.d.ts +0 -16
- package/dist/launcher/ensure-config.js +0 -79
- package/dist/launcher/errors.d.ts +0 -12
- package/dist/launcher/errors.js +0 -22
- package/dist/launcher/host-config.d.ts +0 -17
- package/dist/launcher/host-config.js +0 -64
- package/dist/launcher/host-state.d.ts +0 -17
- package/dist/launcher/host-state.js +0 -61
- package/dist/launcher/lifecycle.d.ts +0 -66
- package/dist/launcher/lifecycle.js +0 -407
- package/dist/launcher/locks.d.ts +0 -26
- package/dist/launcher/locks.js +0 -117
- package/dist/launcher/logs.d.ts +0 -18
- package/dist/launcher/logs.js +0 -117
- package/dist/launcher/main.d.ts +0 -2
- package/dist/launcher/main.js +0 -66
- package/dist/launcher/output.d.ts +0 -11
- package/dist/launcher/output.js +0 -68
- package/dist/launcher/paths.d.ts +0 -20
- package/dist/launcher/paths.js +0 -36
- package/dist/launcher/probe.d.ts +0 -23
- package/dist/launcher/probe.js +0 -72
- package/dist/launcher/protocol.d.ts +0 -62
- package/dist/launcher/protocol.js +0 -1
- package/dist/launcher/schema-guard.d.ts +0 -6
- package/dist/launcher/schema-guard.js +0 -36
- package/dist/launcher/spawn.d.ts +0 -22
- package/dist/launcher/spawn.js +0 -85
- package/dist/launcher/status.d.ts +0 -6
- package/dist/launcher/status.js +0 -65
- package/dist/tenant/migration.d.ts +0 -16
- package/dist/tenant/migration.js +0 -116
- package/dist/tenant/schema.d.ts +0 -23
- package/dist/tenant/schema.js +0 -151
- package/dist/tenant/store-fs.d.ts +0 -16
- package/dist/tenant/store-fs.js +0 -205
package/dist/tenant/runtime.js
CHANGED
|
@@ -1,228 +1,60 @@
|
|
|
1
|
-
import {} from "node:
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { dirname } from "node:path";
|
|
5
|
-
import { AgentManifestSchema, CreateCredentialRequestSchema, CreateVaultRequestSchema, PutAgentRequestSchema, PutHostModelRequestSchema, SelectHostModelRequestSchema, PutSessionRequestSchema, RegisterExecutorsRequestSchema, RotateCredentialRequestSchema, SessionCommandSchema, ActionClaimRequestSchema, ActionHeartbeatRequestSchema, ResetTenantRequestSchema, SeedTenantConfigRequestSchema, DefinitionDocumentSchema, } from "@nylorun/core/contracts";
|
|
6
|
-
import { createDurableCheckpoint, createFlowCheckpoint, runDurable, runFlowDurable, agentTurnValue, } from "@nylorun/harness/run";
|
|
7
|
-
import { aggregateWaits, cancelSiblingWork, countActiveFlowWork, deriveAgentEffectSessionId, emitLoopActionEvents, fenceWorkflowActions, foreignInteractionConflict, isFlowEffect, isFlowToolEffect, isWorkflowManifest, planCancelCascade, reconcilePendingAgentEffects, reofferOrphanedFnVerifyClaims, wakeForQueuedEffects, wakeLinkedWorkflow, } from "../core/flow-host.js";
|
|
8
|
-
import { mayDispatchMore, resolveFlowLimits, } from "../core/limits.js";
|
|
9
|
-
import { allowedManifestHashes, rebaseTurnState, resolveMessageManifest, } from "../core/turn-manifest.js";
|
|
10
|
-
import { hashManifest } from "@nylorun/core/compatibility";
|
|
11
|
-
import { delegateManifest, schemaFromJSON, } from "@nylorun/core/define";
|
|
12
|
-
import { Store, canonical } from "../core/store.js";
|
|
13
|
-
import { ExecutorRegistry, assertExecutorCredential, hashToken, } from "../core/executors.js";
|
|
14
|
-
import { hostModelCatalog } from "../model/catalog.js";
|
|
15
|
-
import { piModel } from "../model/pi-model.js";
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { resolveFlowLimits } from "../core/limits.js";
|
|
3
|
+
import { loadExecutorRegistry } from "../core/executors.js";
|
|
16
4
|
import { scriptedModel, gatewayModel, toolFixtureModel, } from "../core/provider.js";
|
|
17
|
-
import { scrub } from "../redact.js";
|
|
18
5
|
import { createKekFile, readVaultKek } from "../vault/kek.js";
|
|
19
|
-
import { QuarantineError } from "./quarantine-error.js";
|
|
20
|
-
import { applicationTokenHashes, findPrincipalByTokenHash, } from "./principals.js";
|
|
21
|
-
import { resetTenant } from "./reset.js";
|
|
22
|
-
import { buildTenantStatus, seedTenantConfig } from "./status.js";
|
|
23
|
-
import { VaultError } from "../vault/error.js";
|
|
24
6
|
import { VaultService } from "../vault/service.js";
|
|
25
|
-
import { McpPool
|
|
26
|
-
import { SandboxManager
|
|
7
|
+
import { McpPool } from "../mcp/pool.js";
|
|
8
|
+
import { SandboxManager } from "../sandbox/manager.js";
|
|
27
9
|
import { defaultSandboxBackends } from "../sandbox/select.js";
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if (!request.complete)
|
|
51
|
-
controller.abort();
|
|
52
|
-
});
|
|
53
|
-
return controller.signal;
|
|
54
|
-
}
|
|
55
|
-
class OpaqueAuthError extends Error {
|
|
56
|
-
status = 404;
|
|
57
|
-
body = OPAQUE_NOT_FOUND;
|
|
58
|
-
constructor() {
|
|
59
|
-
super(OPAQUE_NOT_FOUND.message);
|
|
60
|
-
this.name = "OpaqueAuthError";
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
const failOpaque = () => {
|
|
64
|
-
throw new OpaqueAuthError();
|
|
65
|
-
};
|
|
66
|
-
function sessionToolsOf(snapshot) {
|
|
67
|
-
if (!snapshot?.mcpTools.length)
|
|
68
|
-
return undefined;
|
|
69
|
-
return snapshot.mcpTools.map((tool) => ({
|
|
70
|
-
...(tool.agentId === undefined ? {} : { agentId: tool.agentId }),
|
|
71
|
-
capabilityId: tool.capabilityId,
|
|
72
|
-
name: tool.name,
|
|
73
|
-
...(tool.description === undefined
|
|
74
|
-
? {}
|
|
75
|
-
: { description: tool.description }),
|
|
76
|
-
inputSchema: tool.inputSchema,
|
|
77
|
-
...(tool.outputSchema === undefined
|
|
78
|
-
? {}
|
|
79
|
-
: { outputSchema: tool.outputSchema }),
|
|
80
|
-
}));
|
|
81
|
-
}
|
|
82
|
-
/** Turn-manifest store backed by the session's `variants` map (no schema change). */
|
|
83
|
-
function variantStore(session) {
|
|
84
|
-
return {
|
|
85
|
-
get(hash) {
|
|
86
|
-
return session.variants?.[hash];
|
|
87
|
-
},
|
|
88
|
-
put(hash, manifest) {
|
|
89
|
-
session.variants = { ...(session.variants ?? {}), [hash]: manifest };
|
|
90
|
-
},
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
/** Manifest this turn's checkpoint is pinned to (session pin or a stored variant). */
|
|
94
|
-
function turnManifestOf(session) {
|
|
95
|
-
const hash = session.checkpoint?.manifestHash;
|
|
96
|
-
if (!hash || hash === session.manifestHash)
|
|
97
|
-
return session.manifest;
|
|
98
|
-
return session.variants?.[hash] ?? session.manifest;
|
|
99
|
-
}
|
|
100
|
-
/** Wrap a linked agent turn so the Loop learns the turn's manifest (SD-I4 pin stays on session). */
|
|
101
|
-
function linkedAgentOutput(session, output) {
|
|
102
|
-
if (isWorkflowManifest(session.manifest))
|
|
103
|
-
return output ?? null;
|
|
104
|
-
return agentTurnValue(output ?? null, turnManifestOf(session));
|
|
105
|
-
}
|
|
106
|
-
function rebaseSessionState(session, turnHash) {
|
|
107
|
-
const allowed = allowedManifestHashes({
|
|
108
|
-
pinnedHash: session.manifestHash,
|
|
109
|
-
variantHashes: Object.keys(session.variants ?? {}),
|
|
110
|
-
});
|
|
111
|
-
const next = rebaseTurnState({
|
|
112
|
-
state: session.state,
|
|
113
|
-
turnManifestHash: turnHash,
|
|
114
|
-
isAllowedHash: (hash) => allowed.has(hash),
|
|
10
|
+
import { MemoryExecution } from "../execution/memory.js";
|
|
11
|
+
import { QuarantineError } from "./quarantine-error.js";
|
|
12
|
+
import { MemoryStreams } from "../streams/memory.js";
|
|
13
|
+
import { connectedExecutorCount, createLiveHub, endAllStreams, } from "./live.js";
|
|
14
|
+
import { closeStreams, drainOutbox, streamsStatus, wireStreams, } from "./streams.js";
|
|
15
|
+
import { abortAll, abortLocal, createWorkState, drain, waitForIdle, } from "./scheduler.js";
|
|
16
|
+
import { advance } from "./advance.js";
|
|
17
|
+
import { sweep } from "./sweep.js";
|
|
18
|
+
import { TenantWorkers, WORKER_ID, } from "./worker.js";
|
|
19
|
+
import { authorize } from "./effects.js";
|
|
20
|
+
import { handle } from "./routes.js";
|
|
21
|
+
/** Default ownership lease of an advance; the heartbeat renews it every third. */
|
|
22
|
+
const DEFAULT_OWNER_LEASE_MS = 30_000;
|
|
23
|
+
function validateConfig(config) {
|
|
24
|
+
for (const key of ["leaseMs", "ownerLeaseMs", "sweepIntervalMs"]) {
|
|
25
|
+
const value = config[key];
|
|
26
|
+
if (value !== undefined && (!Number.isFinite(value) || value <= 0))
|
|
27
|
+
throw new Error(`${key} must be finite and positive`);
|
|
28
|
+
}
|
|
29
|
+
const flowLimits = resolveFlowLimits({
|
|
30
|
+
flow: config.flow,
|
|
31
|
+
env: config.flowEnv,
|
|
115
32
|
});
|
|
116
|
-
if (
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
return session.mcpSnapshot?.mcpTools.find((tool) => tool.agentId === request.agent?.id &&
|
|
121
|
-
tool.capabilityId === request.capabilityId &&
|
|
122
|
-
tool.name === request.toolName);
|
|
33
|
+
if ((config.model.kind === "fixture" || config.model.kind === "scripted") &&
|
|
34
|
+
config.mode === "shared")
|
|
35
|
+
throw new Error("fixture/scripted models require ephemeral or test mode");
|
|
36
|
+
return flowLimits;
|
|
123
37
|
}
|
|
124
|
-
/** The manifest of the agent an effect belongs to: the root, or an agent it uses as a tool. */
|
|
125
|
-
function manifestFor(manifest, agent) {
|
|
126
|
-
return agent ? delegateManifest(manifest, agent.id) : manifest;
|
|
127
|
-
}
|
|
128
|
-
function pinnedTool(manifest, capabilityId, toolName) {
|
|
129
|
-
const capability = manifest.capabilities.find((item) => item.id === capabilityId);
|
|
130
|
-
return capability?.tools?.find((tool) => tool.name === toolName);
|
|
131
|
-
}
|
|
132
|
-
function acceptedToolResult(action, command) {
|
|
133
|
-
if (action.kind !== "tool" || !action.outputSchema)
|
|
134
|
-
return command;
|
|
135
|
-
const value = command.outcome.value;
|
|
136
|
-
if (isFailedToolValue(value))
|
|
137
|
-
return command;
|
|
138
|
-
// Executors wrap successful tool output as `{ kind: "completed", output }`.
|
|
139
|
-
// Validate the tool payload, not the outcome envelope.
|
|
140
|
-
const candidate = completedToolOutput(value);
|
|
141
|
-
let matches = false;
|
|
142
|
-
try {
|
|
143
|
-
matches = schemaFromJSON(action.outputSchema).validate(candidate).ok;
|
|
144
|
-
}
|
|
145
|
-
catch {
|
|
146
|
-
matches = false;
|
|
147
|
-
}
|
|
148
|
-
if (matches)
|
|
149
|
-
return command;
|
|
150
|
-
return {
|
|
151
|
-
...command,
|
|
152
|
-
outcome: {
|
|
153
|
-
...command.outcome,
|
|
154
|
-
value: {
|
|
155
|
-
kind: "failed",
|
|
156
|
-
code: "tool.invalid-output",
|
|
157
|
-
message: "Tool result does not match the output schema stored on the action",
|
|
158
|
-
},
|
|
159
|
-
},
|
|
160
|
-
};
|
|
161
|
-
}
|
|
162
|
-
function completedToolOutput(value) {
|
|
163
|
-
if (value &&
|
|
164
|
-
typeof value === "object" &&
|
|
165
|
-
!Array.isArray(value) &&
|
|
166
|
-
value.kind === "completed" &&
|
|
167
|
-
"output" in value) {
|
|
168
|
-
return value.output;
|
|
169
|
-
}
|
|
170
|
-
return value;
|
|
171
|
-
}
|
|
172
|
-
function isFailedToolValue(value) {
|
|
173
|
-
return (!!value &&
|
|
174
|
-
typeof value === "object" &&
|
|
175
|
-
!Array.isArray(value) &&
|
|
176
|
-
value.kind === "failed");
|
|
177
|
-
}
|
|
178
|
-
const semantic = (value) => {
|
|
179
|
-
const { requestId: _, ...body } = value;
|
|
180
|
-
return canonical(body);
|
|
181
|
-
};
|
|
182
|
-
const sessionIdentity = (value) => {
|
|
183
|
-
const { requestId: _requestId, vaultIds: _vaultIds, credentialSelections: _selections, ...body } = value ?? {};
|
|
184
|
-
return canonical(body);
|
|
185
|
-
};
|
|
186
|
-
const equals = (a, b) => {
|
|
187
|
-
const aa = Buffer.from(a), bb = Buffer.from(b);
|
|
188
|
-
return aa.length === bb.length && timingSafeEqual(aa, bb);
|
|
189
|
-
};
|
|
190
38
|
export class TenantRuntime {
|
|
191
|
-
|
|
192
|
-
store;
|
|
193
|
-
vault;
|
|
194
|
-
kek;
|
|
195
|
-
kekPath;
|
|
196
|
-
running = new Map();
|
|
197
|
-
pending = new Set();
|
|
198
|
-
observers = new Map();
|
|
199
|
-
// Keyed by token hash so a rotation can end exactly the streams that the replaced token owns.
|
|
200
|
-
executorStreams = new Map();
|
|
201
|
-
registry = new ExecutorRegistry();
|
|
202
|
-
mcp;
|
|
203
|
-
sandbox;
|
|
204
|
-
closing = false;
|
|
205
|
-
closed = false;
|
|
206
|
-
lockPath;
|
|
207
|
-
timer;
|
|
208
|
-
modelProvider;
|
|
209
|
-
useVaultModel;
|
|
210
|
-
createKekIfMissing;
|
|
211
|
-
flowLimits;
|
|
39
|
+
ctx;
|
|
212
40
|
envelope;
|
|
213
|
-
|
|
214
|
-
|
|
41
|
+
worker;
|
|
42
|
+
detach;
|
|
43
|
+
closeGraceMs;
|
|
44
|
+
constructor(ctx, envelope,
|
|
45
|
+
/** This Tenant's Worker handlers, as registered with the execution. */
|
|
46
|
+
worker, detach,
|
|
47
|
+
/** How long close waits for running advances: the execution's advance grace period. */
|
|
48
|
+
closeGraceMs) {
|
|
49
|
+
this.ctx = ctx;
|
|
215
50
|
this.envelope = envelope;
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
if ((config.model.kind === "fixture" || config.model.kind === "scripted") &&
|
|
224
|
-
config.mode === "shared")
|
|
225
|
-
throw new Error("fixture/scripted models require ephemeral or test mode");
|
|
51
|
+
this.worker = worker;
|
|
52
|
+
this.detach = detach;
|
|
53
|
+
this.closeGraceMs = closeGraceMs;
|
|
54
|
+
}
|
|
55
|
+
static async open(config, hooks) {
|
|
56
|
+
const envelope = hooks.envelope;
|
|
57
|
+
const flowLimits = validateConfig(config);
|
|
226
58
|
const paths = config.paths;
|
|
227
59
|
mkdirSync(paths.root, { recursive: true });
|
|
228
60
|
mkdirSync(paths.home, { recursive: true });
|
|
@@ -230,2027 +62,237 @@ export class TenantRuntime {
|
|
|
230
62
|
mkdirSync(paths.sandboxes, { recursive: true });
|
|
231
63
|
mkdirSync(paths.pluginData, { recursive: true });
|
|
232
64
|
mkdirSync(paths.logs, { recursive: true });
|
|
233
|
-
|
|
234
|
-
|
|
65
|
+
const store = hooks.store;
|
|
66
|
+
let wired;
|
|
67
|
+
let detach;
|
|
235
68
|
try {
|
|
236
|
-
|
|
237
|
-
if (!Number.isSafeInteger(oldPid) || oldPid < 1)
|
|
238
|
-
throw new Error("Invalid runtime lock; inspect before removing");
|
|
239
|
-
try {
|
|
240
|
-
process.kill(oldPid, 0);
|
|
241
|
-
throw new QuarantineError("locked", "Another Runtime owns this Tenant database", "nylorun tenant status", { lockPath: this.lockPath, lockPid: oldPid });
|
|
242
|
-
}
|
|
243
|
-
catch (error) {
|
|
244
|
-
if (error instanceof QuarantineError)
|
|
245
|
-
throw error;
|
|
246
|
-
if (error.code !== "ESRCH")
|
|
247
|
-
throw error;
|
|
248
|
-
unlinkSync(this.lockPath);
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
catch (error) {
|
|
252
|
-
if (error instanceof QuarantineError)
|
|
253
|
-
throw error;
|
|
254
|
-
if (error.code !== "ENOENT")
|
|
255
|
-
throw error;
|
|
256
|
-
}
|
|
257
|
-
const fd = openSync(this.lockPath, "wx");
|
|
258
|
-
writeFileSync(fd, String(process.pid));
|
|
259
|
-
closeSync(fd);
|
|
260
|
-
this.kekPath = paths.kek;
|
|
261
|
-
this.createKekIfMissing = hooks.createKekIfMissing !== false;
|
|
262
|
-
let store;
|
|
263
|
-
try {
|
|
264
|
-
store = new Store(paths.database, config.tenantId);
|
|
265
|
-
this.kek = readVaultKek({
|
|
69
|
+
let kek = readVaultKek({
|
|
266
70
|
vaultKek: hooks.vaultKek,
|
|
267
|
-
vaultKekPath:
|
|
71
|
+
vaultKekPath: paths.kek,
|
|
268
72
|
});
|
|
269
|
-
if (store.
|
|
270
|
-
throw new QuarantineError("kek-missing", "Vault key-encryption key is missing for ciphertext in this Tenant", "restore the vault-kek file
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
this.registry.seed(this.store.allExecutors().map((row) => ({
|
|
281
|
-
agentId: row.agentId,
|
|
282
|
-
implementationVersion: row.implementationVersion,
|
|
283
|
-
...(row.manifestHash === undefined
|
|
284
|
-
? {}
|
|
285
|
-
: { manifestHash: row.manifestHash }),
|
|
286
|
-
tokenHash: row.tokenHash,
|
|
287
|
-
persisted: true,
|
|
288
|
-
updatedAt: row.updatedAt,
|
|
289
|
-
...(row.principalId === undefined
|
|
290
|
-
? {}
|
|
291
|
-
: { principalId: row.principalId }),
|
|
292
|
-
})));
|
|
293
|
-
this.vault = new VaultService(this.store.db, (fn) => this.store.tx(fn), () => this.ensureKek(), config.vaultFetch ?? globalThis.fetch);
|
|
294
|
-
this.mcp = new McpPool({
|
|
295
|
-
pluginData: paths.pluginData,
|
|
296
|
-
childEnv: config.childEnv,
|
|
297
|
-
authorize: (sessionId, request) => this.authorize(sessionId, request),
|
|
298
|
-
});
|
|
299
|
-
const ephemeral = config.mode === "ephemeral";
|
|
300
|
-
this.sandbox = new SandboxManager({
|
|
301
|
-
scope: config.tenantId,
|
|
302
|
-
store: this.store,
|
|
303
|
-
backends: config.sandbox.backends ??
|
|
304
|
-
defaultSandboxBackends({ root: paths.sandboxes }),
|
|
305
|
-
preference: config.sandbox.backend,
|
|
306
|
-
ephemeral,
|
|
307
|
-
emit: (sessionId, turnId, type, payload) => {
|
|
308
|
-
if (this.closed)
|
|
309
|
-
return;
|
|
310
|
-
const event = this.store.tx(() => this.store.event(sessionId, turnId, type, payload));
|
|
311
|
-
this.publish(event);
|
|
312
|
-
},
|
|
313
|
-
});
|
|
314
|
-
if (!ephemeral && this.sandbox.hasRecords())
|
|
315
|
-
void this.sandbox
|
|
316
|
-
.reconcile((id) => !this.closed && !!this.store.get("sessions", id))
|
|
317
|
-
.catch(() => undefined);
|
|
318
|
-
this.useVaultModel = config.model.kind === "vault";
|
|
319
|
-
if (hooks.modelProvider)
|
|
320
|
-
this.modelProvider = hooks.modelProvider;
|
|
321
|
-
else if (config.model.kind === "scripted")
|
|
322
|
-
this.modelProvider = scriptedModel(config.model.output);
|
|
323
|
-
else if (config.model.kind === "fixture")
|
|
324
|
-
this.modelProvider = toolFixtureModel();
|
|
325
|
-
else if (config.model.kind === "gateway") {
|
|
326
|
-
const gateway = config.model;
|
|
327
|
-
this.modelProvider = async (effect, signal) => {
|
|
328
|
-
const secret = this.vault.readHostModel();
|
|
329
|
-
const token = typeof secret?.credential.key === "string"
|
|
330
|
-
? secret.credential.key
|
|
331
|
-
: "";
|
|
332
|
-
return gatewayModel({
|
|
333
|
-
url: gateway.url,
|
|
334
|
-
model: gateway.model,
|
|
335
|
-
token,
|
|
336
|
-
})(effect, signal);
|
|
73
|
+
if ((await store.tx((t) => t.countCredentials())) > 0 && !kek) {
|
|
74
|
+
throw new QuarantineError("kek-missing", "Vault key-encryption key is missing for ciphertext in this Tenant", "restore the vault-kek file in the Tenant directory");
|
|
75
|
+
}
|
|
76
|
+
const createKekIfMissing = hooks.createKekIfMissing !== false;
|
|
77
|
+
const ensureKek = () => {
|
|
78
|
+
if (kek)
|
|
79
|
+
return kek;
|
|
80
|
+
if (!createKekIfMissing)
|
|
81
|
+
throw new QuarantineError("kek-missing", "Vault key-encryption key is required", "restore the vault-kek file in the Tenant directory");
|
|
82
|
+
kek = createKekFile(paths.kek);
|
|
83
|
+
return kek;
|
|
337
84
|
};
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
const sess = this.store.get("sessions", effect.request.sessionId);
|
|
347
|
-
if (sess &&
|
|
348
|
-
sess.status !== "cancelled" &&
|
|
349
|
-
sess.activeTurnId === effect.request.turnId) {
|
|
350
|
-
sess.status = "uncertain";
|
|
351
|
-
this.store.put("sessions", sess.id, sess);
|
|
352
|
-
this.store.event(sess.id, sess.activeTurnId, "effect.uncertain", {
|
|
353
|
-
effectId: effect.request.effectId,
|
|
354
|
-
});
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
});
|
|
358
|
-
this.retireLegacyHooks();
|
|
359
|
-
this.expireClaims();
|
|
360
|
-
// Orphaned fn/verify claims from a prior process: re-offer immediately.
|
|
361
|
-
this.store.tx(() => {
|
|
362
|
-
reofferOrphanedFnVerifyClaims(this.store);
|
|
363
|
-
});
|
|
364
|
-
this.notify();
|
|
365
|
-
this.timer = setInterval(() => this.expireClaims(), Math.min(config.leaseMs ?? 30000, 5000));
|
|
366
|
-
this.timer.unref();
|
|
367
|
-
for (const sess of this.store.all("sessions"))
|
|
368
|
-
if (sess.status === "running" || sess.status === "runnable")
|
|
369
|
-
this.schedule(sess.id);
|
|
370
|
-
this.store.tx(() => {
|
|
371
|
-
reconcilePendingAgentEffects({
|
|
372
|
-
store: this.store,
|
|
373
|
-
schedule: (sid) => this.schedule(sid),
|
|
374
|
-
publish: (e) => this.publish(e),
|
|
85
|
+
const opened = store;
|
|
86
|
+
const registry = await loadExecutorRegistry(opened);
|
|
87
|
+
// A seeded Tenant setting wins over the Host default (A18).
|
|
88
|
+
const seededBackend = await opened.tx((t) => t.getSetting("sandbox.backend"));
|
|
89
|
+
const vault = new VaultService({
|
|
90
|
+
store: opened,
|
|
91
|
+
kek: ensureKek,
|
|
92
|
+
fetch: config.vaultFetch ?? globalThis.fetch,
|
|
375
93
|
});
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
this.send(response, 'event: work_available\ndata: {"type":"work_available"}\n\n');
|
|
397
|
-
}
|
|
398
|
-
/**
|
|
399
|
-
* Manifest v4 replaced beforeModelCall/afterModelCall with scoped hooks. Turns started under
|
|
400
|
-
* an older manifest cannot continue, and their hook actions no longer parse, so end them.
|
|
401
|
-
*/
|
|
402
|
-
retireLegacyHooks() {
|
|
403
|
-
this.store.tx(() => {
|
|
404
|
-
for (const action of this.store.all("actions"))
|
|
405
|
-
if ((action.kind === "beforeModelCall" ||
|
|
406
|
-
action.kind === "afterModelCall") &&
|
|
407
|
-
(action.status === "pending" || action.status === "claimed")) {
|
|
408
|
-
action.status = "cancelled";
|
|
409
|
-
this.store.put("actions", action.actionId, action);
|
|
410
|
-
}
|
|
411
|
-
for (const s of this.store.all("sessions"))
|
|
412
|
-
if (s.activeTurnId &&
|
|
413
|
-
!isWorkflowManifest(s.manifest) &&
|
|
414
|
-
s.manifest?.manifestSchemaVersion !== 4) {
|
|
415
|
-
this.store.event(s.id, s.activeTurnId, "turn.failed", {
|
|
416
|
-
error: {
|
|
417
|
-
code: "execution.incompatible",
|
|
418
|
-
message: "This turn started under manifest schema 3; see MIGRATION.md for before/after hooks",
|
|
419
|
-
},
|
|
420
|
-
});
|
|
421
|
-
s.status = "failed";
|
|
422
|
-
s.activeTurnId = null;
|
|
423
|
-
this.store.put("sessions", s.id, s);
|
|
424
|
-
}
|
|
425
|
-
});
|
|
426
|
-
}
|
|
427
|
-
expireClaims() {
|
|
428
|
-
const events = [];
|
|
429
|
-
let redeliver = false;
|
|
430
|
-
this.store.tx(() => {
|
|
431
|
-
for (const action of this.store.all("actions"))
|
|
432
|
-
if (action.status === "claimed" &&
|
|
433
|
-
Date.parse(action.leaseExpiresAt) <= Date.now()) {
|
|
434
|
-
if (action.kind === "hook" ||
|
|
435
|
-
action.kind === "fn" ||
|
|
436
|
-
action.kind === "verify") {
|
|
437
|
-
// Hooks, fn and verify are pure / repeat-safe; a lost claim is offered again.
|
|
438
|
-
// The next claim bumps the generation, which fences any late result.
|
|
439
|
-
action.status = "pending";
|
|
440
|
-
action.claimId = null;
|
|
441
|
-
action.leaseExpiresAt = null;
|
|
442
|
-
this.store.put("actions", action.actionId, action);
|
|
443
|
-
redeliver = true;
|
|
444
|
-
continue;
|
|
445
|
-
}
|
|
446
|
-
action.status = "uncertain";
|
|
447
|
-
this.store.put("actions", action.actionId, action);
|
|
448
|
-
const effect = this.store.get("effects", action.actionId);
|
|
449
|
-
effect.status = "uncertain";
|
|
450
|
-
this.store.put("effects", action.actionId, effect);
|
|
451
|
-
const s = this.session(action.sessionId);
|
|
452
|
-
if (s.status !== "cancelled" && s.activeTurnId === action.turnId) {
|
|
453
|
-
s.status = "uncertain";
|
|
454
|
-
this.store.put("sessions", s.id, s);
|
|
455
|
-
events.push(this.store.event(s.id, s.activeTurnId, "action.uncertain", {
|
|
456
|
-
actionId: action.actionId,
|
|
457
|
-
...(action.agent ? { agent: action.agent } : {}),
|
|
458
|
-
}));
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
|
-
});
|
|
462
|
-
events.forEach((e) => this.publish(e));
|
|
463
|
-
if (redeliver)
|
|
464
|
-
this.notify();
|
|
465
|
-
}
|
|
466
|
-
schedule(id) {
|
|
467
|
-
if (this.closing)
|
|
468
|
-
return;
|
|
469
|
-
this.pending.add(id);
|
|
470
|
-
setImmediate(() => {
|
|
471
|
-
if (!this.running.has(id) && this.pending.delete(id) && !this.closing)
|
|
472
|
-
void this.execute(id);
|
|
473
|
-
});
|
|
474
|
-
}
|
|
475
|
-
async execute(id) {
|
|
476
|
-
const s = this.session(id);
|
|
477
|
-
if (!s.checkpoint || !["running", "runnable"].includes(s.status))
|
|
478
|
-
return;
|
|
479
|
-
s.status = "running";
|
|
480
|
-
this.store.tx(() => this.store.put("sessions", id, s));
|
|
481
|
-
const controller = new AbortController();
|
|
482
|
-
this.running.set(id, controller);
|
|
483
|
-
try {
|
|
484
|
-
// prepareMcp mutates the session's mcpSnapshot; read current after it.
|
|
485
|
-
if (!isWorkflowManifest(this.session(id).manifest))
|
|
486
|
-
await this.prepareMcp(id, controller.signal);
|
|
487
|
-
const current = this.session(id);
|
|
488
|
-
const host = {
|
|
489
|
-
resolveEffect: (e) => this.resolveEffect(e, controller.signal),
|
|
490
|
-
};
|
|
491
|
-
if (!isWorkflowManifest(current.manifest) && current.checkpoint) {
|
|
492
|
-
rebaseSessionState(current, current.checkpoint.manifestHash);
|
|
493
|
-
const cp = current.checkpoint;
|
|
494
|
-
current.checkpoint = { ...cp, state: current.state };
|
|
495
|
-
this.store.tx(() => this.store.put("sessions", id, current));
|
|
496
|
-
}
|
|
497
|
-
const result = isWorkflowManifest(current.manifest)
|
|
498
|
-
? await runFlowDurable({
|
|
499
|
-
manifest: current.manifest,
|
|
500
|
-
checkpoint: current.checkpoint,
|
|
501
|
-
signal: controller.signal,
|
|
502
|
-
host,
|
|
503
|
-
limits: this.flowLimits,
|
|
504
|
-
})
|
|
505
|
-
: await runDurable({
|
|
506
|
-
manifest: turnManifestOf(current),
|
|
507
|
-
checkpoint: current.checkpoint,
|
|
508
|
-
signal: controller.signal,
|
|
509
|
-
sessionTools: sessionToolsOf(current.mcpSnapshot),
|
|
510
|
-
host,
|
|
511
|
-
});
|
|
512
|
-
let event;
|
|
513
|
-
const siblingCancelIds = [];
|
|
514
|
-
this.store.tx(() => {
|
|
515
|
-
const current = this.session(id);
|
|
516
|
-
if (current.status === "cancelled" ||
|
|
517
|
-
current.activeTurnId !== s.activeTurnId)
|
|
518
|
-
return;
|
|
519
|
-
// A result can arrive during concurrent action persistence. Preserve the runnable marker.
|
|
520
|
-
const resumeRequested = current.status === "runnable";
|
|
521
|
-
if (result.status === "waiting" || result.status === "uncertain") {
|
|
522
|
-
const linkedWaits = isWorkflowManifest(current.manifest)
|
|
523
|
-
? aggregateWaits({
|
|
524
|
-
store: this.store,
|
|
525
|
-
workflowSessionId: id,
|
|
526
|
-
})
|
|
527
|
-
: [];
|
|
528
|
-
if (linkedWaits.length > 0 && !resumeRequested) {
|
|
529
|
-
current.status = "paused";
|
|
530
|
-
current.waits = linkedWaits;
|
|
531
|
-
}
|
|
532
|
-
else {
|
|
533
|
-
current.status = resumeRequested
|
|
534
|
-
? "runnable"
|
|
535
|
-
: current.status === "uncertain"
|
|
536
|
-
? "uncertain"
|
|
537
|
-
: result.status;
|
|
538
|
-
current.waits =
|
|
539
|
-
linkedWaits.length > 0
|
|
540
|
-
? linkedWaits
|
|
541
|
-
: { effectIds: result.effectIds };
|
|
542
|
-
}
|
|
543
|
-
}
|
|
544
|
-
else if ("result" in result) {
|
|
545
|
-
const flow = isWorkflowManifest(current.manifest);
|
|
546
|
-
if (!flow) {
|
|
547
|
-
current.state =
|
|
548
|
-
result.status === "failed"
|
|
549
|
-
? current.turnStartState
|
|
550
|
-
: result.result.state;
|
|
94
|
+
// `ctx` is assigned below; these callbacks only run once the Tenant is open.
|
|
95
|
+
let ctx;
|
|
96
|
+
const mcp = new McpPool({
|
|
97
|
+
pluginData: paths.pluginData,
|
|
98
|
+
childEnv: config.childEnv,
|
|
99
|
+
authorize: (sessionId, request) => authorize(ctx, sessionId, request),
|
|
100
|
+
});
|
|
101
|
+
const ephemeral = config.mode === "ephemeral";
|
|
102
|
+
const sandbox = new SandboxManager({
|
|
103
|
+
scope: config.tenantId,
|
|
104
|
+
store: opened,
|
|
105
|
+
backends: config.sandbox.backends ??
|
|
106
|
+
defaultSandboxBackends({ root: paths.sandboxes }),
|
|
107
|
+
preference: seededBackend ?? config.sandbox.backend,
|
|
108
|
+
ephemeral,
|
|
109
|
+
emit: async (sessionId, turnId, type, payload) => {
|
|
110
|
+
if (ctx.closed)
|
|
111
|
+
return;
|
|
112
|
+
try {
|
|
113
|
+
await opened.tx((t) => t.event(sessionId, turnId, type, payload));
|
|
551
114
|
}
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
current.status = result.status;
|
|
555
|
-
current.waits =
|
|
556
|
-
result.status === "paused"
|
|
557
|
-
? result.result.pending
|
|
558
|
-
: undefined;
|
|
559
|
-
if (result.status === "completed")
|
|
560
|
-
current.lastOutput = result.result.output;
|
|
561
|
-
// Fail-fast: cancel pending Parallel/Map siblings before clearing the turn.
|
|
562
|
-
if (result.status === "failed" &&
|
|
563
|
-
flow &&
|
|
564
|
-
"cancelEffectIds" in result &&
|
|
565
|
-
Array.isArray(result.cancelEffectIds) &&
|
|
566
|
-
result.cancelEffectIds.length > 0 &&
|
|
567
|
-
s.activeTurnId) {
|
|
568
|
-
const cancelResult = cancelSiblingWork({
|
|
569
|
-
store: this.store,
|
|
570
|
-
workflowSessionId: id,
|
|
571
|
-
turnId: s.activeTurnId,
|
|
572
|
-
cancelEffectIds: result.cancelEffectIds,
|
|
573
|
-
});
|
|
574
|
-
siblingCancelIds.push(...cancelResult.agentSessionIds);
|
|
115
|
+
catch {
|
|
116
|
+
/* the session is gone (reset) or the store closed */
|
|
575
117
|
}
|
|
576
|
-
|
|
577
|
-
current.activeTurnId = null;
|
|
578
|
-
event = this.store.event(id, s.activeTurnId, `turn.${result.status}`, result.status === "completed"
|
|
579
|
-
? { output: result.result.output }
|
|
580
|
-
: result.status === "paused"
|
|
581
|
-
? {
|
|
582
|
-
interactions: (result.result.pending ?? []).map((call) => ({
|
|
583
|
-
invocationId: call.invocationId,
|
|
584
|
-
interaction: call.interaction,
|
|
585
|
-
wait: call.wait,
|
|
586
|
-
status: call.status,
|
|
587
|
-
})),
|
|
588
|
-
}
|
|
589
|
-
: result.status === "failed"
|
|
590
|
-
? {
|
|
591
|
-
error: {
|
|
592
|
-
code: result.result.error?.code,
|
|
593
|
-
message: result.result.error?.message,
|
|
594
|
-
...(result.result.error?.path
|
|
595
|
-
? { path: result.result.error.path }
|
|
596
|
-
: {}),
|
|
597
|
-
},
|
|
598
|
-
}
|
|
599
|
-
: {});
|
|
600
|
-
}
|
|
601
|
-
this.store.put("sessions", id, current);
|
|
602
|
-
});
|
|
603
|
-
for (const agentId of siblingCancelIds) {
|
|
604
|
-
try {
|
|
605
|
-
this.command(agentId, {
|
|
606
|
-
type: "cancel",
|
|
607
|
-
requestId: `flow-sibling-${id}-${agentId}`,
|
|
608
|
-
idempotencyKey: `flow-sibling-cancel:${id}:${agentId}:${s.activeTurnId}`,
|
|
609
|
-
reason: "sibling branch failed",
|
|
610
|
-
}, {
|
|
611
|
-
kind: "application",
|
|
612
|
-
principalId: "flow-host",
|
|
613
|
-
});
|
|
614
|
-
}
|
|
615
|
-
catch {
|
|
616
|
-
/* agent may already be terminal */
|
|
617
|
-
}
|
|
618
|
-
}
|
|
619
|
-
if (event) {
|
|
620
|
-
this.publish(event);
|
|
621
|
-
if (event.type === "turn.completed" || event.type === "turn.failed") {
|
|
622
|
-
this.store.tx(() => {
|
|
623
|
-
wakeLinkedWorkflow({
|
|
624
|
-
agentSessionId: id,
|
|
625
|
-
store: this.store,
|
|
626
|
-
output: event.type === "turn.completed"
|
|
627
|
-
? linkedAgentOutput(this.session(id), event.payload.output)
|
|
628
|
-
: undefined,
|
|
629
|
-
failed: event.type === "turn.failed",
|
|
630
|
-
error: event.type === "turn.failed"
|
|
631
|
-
? String(event.payload.message ?? "")
|
|
632
|
-
: undefined,
|
|
633
|
-
schedule: (sid) => this.schedule(sid),
|
|
634
|
-
publish: (e) => this.publish(e),
|
|
635
|
-
});
|
|
636
|
-
});
|
|
637
|
-
}
|
|
638
|
-
if (event.type === "turn.cancelled") {
|
|
639
|
-
this.store.tx(() => {
|
|
640
|
-
wakeLinkedWorkflow({
|
|
641
|
-
agentSessionId: id,
|
|
642
|
-
store: this.store,
|
|
643
|
-
cancelled: true,
|
|
644
|
-
error: "Agent turn was cancelled",
|
|
645
|
-
schedule: (sid) => this.schedule(sid),
|
|
646
|
-
publish: (e) => this.publish(e),
|
|
647
|
-
});
|
|
648
|
-
});
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
}
|
|
652
|
-
catch (error) {
|
|
653
|
-
let event;
|
|
654
|
-
this.store.tx(() => {
|
|
655
|
-
const current = this.session(id);
|
|
656
|
-
if (current.status === "cancelled" ||
|
|
657
|
-
current.activeTurnId !== s.activeTurnId)
|
|
658
|
-
return;
|
|
659
|
-
current.status = "failed";
|
|
660
|
-
current.state = current.turnStartState;
|
|
661
|
-
if (current.checkpoint)
|
|
662
|
-
this.store.put("checkpoints", JSON.stringify([id, s.activeTurnId, current.checkpoint.segment]), { checkpoint: current.checkpoint, status: "failed" });
|
|
663
|
-
current.error = error instanceof Error ? error.message : String(error);
|
|
664
|
-
this.store.put("sessions", id, current);
|
|
665
|
-
event = this.store.event(id, current.activeTurnId, "turn.failed", {
|
|
666
|
-
message: current.error,
|
|
667
|
-
});
|
|
668
|
-
current.activeTurnId = null;
|
|
669
|
-
this.store.put("sessions", id, current);
|
|
118
|
+
},
|
|
670
119
|
});
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
}
|
|
692
|
-
invokeModel(request, signal) {
|
|
693
|
-
if (!this.useVaultModel)
|
|
694
|
-
return this.modelProvider(request, signal);
|
|
695
|
-
const adapter = piModel({
|
|
696
|
-
root: this.config.paths.home,
|
|
697
|
-
readHostModel: () => this.vault.readHostModel(),
|
|
698
|
-
writeHostCredential: (credential) => this.vault.updateHostCredential(credential),
|
|
699
|
-
});
|
|
700
|
-
return adapter(request.input, {
|
|
701
|
-
request: request.context.request,
|
|
702
|
-
invocationId: String(request.context.invocationId),
|
|
703
|
-
signal,
|
|
704
|
-
reportPreparedCall() { },
|
|
705
|
-
});
|
|
706
|
-
}
|
|
707
|
-
async resolveEffect(request, signal) {
|
|
708
|
-
let invoke;
|
|
709
|
-
let notify = false;
|
|
710
|
-
let event;
|
|
711
|
-
const resolution = this.store.tx(() => {
|
|
712
|
-
const s = this.session(request.sessionId);
|
|
713
|
-
if (s.status === "cancelled" ||
|
|
714
|
-
s.activeTurnId !== request.turnId ||
|
|
715
|
-
signal.aborted)
|
|
716
|
-
throw new Error("Turn cancelled");
|
|
717
|
-
const existing = this.store.get("effects", request.effectId);
|
|
718
|
-
if (existing) {
|
|
719
|
-
if (canonical(existing.request) !== canonical(request))
|
|
720
|
-
throw new Error("Effect identity request drift");
|
|
721
|
-
if (existing.status === "completed")
|
|
722
|
-
return { status: "completed", outcome: existing.outcome };
|
|
723
|
-
if (request.kind === "agent" && existing.status === "pending") {
|
|
724
|
-
const agentSessionId = existing.agentSessionId;
|
|
725
|
-
if (agentSessionId) {
|
|
726
|
-
const agent = this.store.get("sessions", agentSessionId);
|
|
727
|
-
if (agent?.status === "completed") {
|
|
728
|
-
const outcome = {
|
|
729
|
-
value: linkedAgentOutput(agent, agent.lastOutput ?? null),
|
|
730
|
-
};
|
|
731
|
-
existing.status = "completed";
|
|
732
|
-
existing.outcome = outcome;
|
|
733
|
-
this.store.put("effects", request.effectId, existing);
|
|
734
|
-
return { status: "completed", outcome };
|
|
735
|
-
}
|
|
736
|
-
if (agent?.status === "failed") {
|
|
737
|
-
const outcome = {
|
|
738
|
-
value: {
|
|
739
|
-
kind: "failed",
|
|
740
|
-
code: "agent.failed",
|
|
741
|
-
message: agent.error ?? "Agent turn failed",
|
|
742
|
-
},
|
|
743
|
-
};
|
|
744
|
-
existing.status = "completed";
|
|
745
|
-
existing.outcome = outcome;
|
|
746
|
-
this.store.put("effects", request.effectId, existing);
|
|
747
|
-
return { status: "completed", outcome };
|
|
748
|
-
}
|
|
749
|
-
}
|
|
750
|
-
}
|
|
751
|
-
return {
|
|
752
|
-
status: existing.status === "uncertain" || existing.status === "invoking"
|
|
753
|
-
? "uncertain"
|
|
754
|
-
: "pending",
|
|
120
|
+
const useVaultModel = config.model.kind === "vault";
|
|
121
|
+
let modelProvider;
|
|
122
|
+
if (hooks.modelProvider)
|
|
123
|
+
modelProvider = hooks.modelProvider;
|
|
124
|
+
else if (config.model.kind === "scripted")
|
|
125
|
+
modelProvider = scriptedModel(config.model.output);
|
|
126
|
+
else if (config.model.kind === "fixture")
|
|
127
|
+
modelProvider = toolFixtureModel();
|
|
128
|
+
else if (config.model.kind === "gateway") {
|
|
129
|
+
const gateway = config.model;
|
|
130
|
+
modelProvider = async (effect, signal) => {
|
|
131
|
+
const secret = await vault.readHostModel();
|
|
132
|
+
const token = typeof secret?.credential.key === "string"
|
|
133
|
+
? secret.credential.key
|
|
134
|
+
: "";
|
|
135
|
+
return gatewayModel({
|
|
136
|
+
url: gateway.url,
|
|
137
|
+
model: gateway.model,
|
|
138
|
+
token,
|
|
139
|
+
})(effect, signal);
|
|
755
140
|
};
|
|
756
141
|
}
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
this.store.put("effects", request.effectId, {
|
|
768
|
-
request,
|
|
769
|
-
status: "completed",
|
|
770
|
-
outcome,
|
|
142
|
+
else
|
|
143
|
+
modelProvider = scriptedModel();
|
|
144
|
+
const live = createLiveHub();
|
|
145
|
+
const local = hooks.execution
|
|
146
|
+
? undefined
|
|
147
|
+
: new MemoryExecution({
|
|
148
|
+
sweepIntervalMs: config.sweepIntervalMs ?? Math.min(config.leaseMs ?? 30_000, 5000),
|
|
149
|
+
onError: (error) => config.logger.error("tenant execution failed", {
|
|
150
|
+
message: error instanceof Error ? error.message : String(error),
|
|
151
|
+
}),
|
|
771
152
|
});
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
}
|
|
776
|
-
const agentManifest = manifestFor(s.manifest, request.agent);
|
|
777
|
-
if (!agentManifest)
|
|
778
|
-
throw new Error(`Agent '${request.agent?.id ?? ""}' is not used as a tool`);
|
|
779
|
-
const mcpTool = request.kind === "tool" ? mcpToolOf(s, request) : undefined;
|
|
780
|
-
const sandboxTool = request.kind === "tool" && !mcpTool
|
|
781
|
-
? sandboxCapabilityOf(agentManifest, request.capabilityId, request.toolName)
|
|
782
|
-
: undefined;
|
|
783
|
-
this.store.put("effects", request.effectId, {
|
|
784
|
-
request,
|
|
785
|
-
status: request.kind === "model" || mcpTool || sandboxTool
|
|
786
|
-
? "invoking"
|
|
787
|
-
: "pending",
|
|
788
|
-
});
|
|
789
|
-
if (request.kind === "model" || mcpTool || sandboxTool) {
|
|
790
|
-
invoke = mcpTool ? "mcp" : sandboxTool ? "sandbox" : "model";
|
|
791
|
-
return undefined;
|
|
792
|
-
}
|
|
793
|
-
const tool = request.kind === "tool"
|
|
794
|
-
? pinnedTool(agentManifest, request.capabilityId, request.toolName)
|
|
795
|
-
: undefined;
|
|
796
|
-
const base = {
|
|
797
|
-
actionId: request.effectId,
|
|
798
|
-
sessionId: request.sessionId,
|
|
799
|
-
turnId: request.turnId,
|
|
800
|
-
agentId: request.agentId,
|
|
801
|
-
manifestHash: request.manifestHash,
|
|
802
|
-
implementationVersion: s.implementationVersion,
|
|
803
|
-
input: request.input,
|
|
804
|
-
context: request.context,
|
|
805
|
-
status: "pending",
|
|
806
|
-
generation: 0,
|
|
807
|
-
claimId: null,
|
|
808
|
-
leaseExpiresAt: null,
|
|
809
|
-
...(request.agent ? { agent: request.agent } : {}),
|
|
153
|
+
const { execution, workers } = hooks.execution ?? {
|
|
154
|
+
execution: local,
|
|
155
|
+
workers: new TenantWorkers(),
|
|
810
156
|
};
|
|
811
|
-
const
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
...
|
|
836
|
-
input: action.input,
|
|
837
|
-
});
|
|
838
|
-
notify = true;
|
|
839
|
-
return { status: "pending" };
|
|
840
|
-
});
|
|
841
|
-
if (event)
|
|
842
|
-
this.publish(event);
|
|
843
|
-
if (notify)
|
|
844
|
-
this.notify();
|
|
845
|
-
if (resolution && resolution.__flow && isFlowEffect(request)) {
|
|
846
|
-
return this.resolveNewFlowEffect(request, signal);
|
|
847
|
-
}
|
|
848
|
-
if (!invoke)
|
|
849
|
-
return resolution;
|
|
850
|
-
try {
|
|
851
|
-
const value = invoke === "mcp"
|
|
852
|
-
? await this.callMcpTool(request)
|
|
853
|
-
: invoke === "sandbox"
|
|
854
|
-
? await this.callSandboxTool(request, signal)
|
|
855
|
-
: await this.invokeModel(request, signal);
|
|
856
|
-
return this.store.tx(() => {
|
|
857
|
-
const s = this.session(request.sessionId);
|
|
858
|
-
if (s.status === "cancelled" ||
|
|
859
|
-
s.activeTurnId !== request.turnId ||
|
|
860
|
-
signal.aborted)
|
|
861
|
-
throw new Error("Turn cancelled");
|
|
862
|
-
const effect = this.store.get("effects", request.effectId);
|
|
863
|
-
effect.status = "completed";
|
|
864
|
-
effect.outcome = { value };
|
|
865
|
-
this.store.put("effects", request.effectId, effect);
|
|
866
|
-
return { status: "completed", outcome: effect.outcome };
|
|
867
|
-
});
|
|
868
|
-
}
|
|
869
|
-
catch (error) {
|
|
870
|
-
let event;
|
|
871
|
-
this.store.tx(() => {
|
|
872
|
-
const effect = this.store.get("effects", request.effectId);
|
|
873
|
-
effect.status = "uncertain";
|
|
874
|
-
effect.error = error instanceof Error ? error.message : String(error);
|
|
875
|
-
this.store.put("effects", request.effectId, effect);
|
|
876
|
-
const s = this.session(request.sessionId);
|
|
877
|
-
if (s.status !== "cancelled" && s.activeTurnId === request.turnId)
|
|
878
|
-
event = this.store.event(s.id, request.turnId, "effect.uncertain", {
|
|
879
|
-
effectId: request.effectId,
|
|
880
|
-
message: effect.error,
|
|
881
|
-
});
|
|
882
|
-
});
|
|
883
|
-
if (event)
|
|
884
|
-
this.publish(event);
|
|
885
|
-
return { status: "uncertain" };
|
|
886
|
-
}
|
|
887
|
-
}
|
|
888
|
-
/** Journal and dispatch a new flow effect (agent / tool node / fn / verify). */
|
|
889
|
-
async resolveNewFlowEffect(request, signal) {
|
|
890
|
-
if (signal.aborted)
|
|
891
|
-
throw new Error("Turn cancelled");
|
|
892
|
-
const existing = this.store.get("effects", request.effectId);
|
|
893
|
-
if (existing) {
|
|
894
|
-
if (existing.status === "completed")
|
|
895
|
-
return { status: "completed", outcome: existing.outcome };
|
|
896
|
-
if (existing.status === "queued") {
|
|
897
|
-
// Fall through to dispatch when a concurrency slot is free.
|
|
898
|
-
}
|
|
899
|
-
else if (request.kind === "agent") {
|
|
900
|
-
const agentSessionId = existing.agentSessionId;
|
|
901
|
-
if (agentSessionId) {
|
|
902
|
-
const agent = this.store.get("sessions", agentSessionId);
|
|
903
|
-
if (agent?.status === "completed") {
|
|
904
|
-
const outcome = {
|
|
905
|
-
value: linkedAgentOutput(agent, agent.lastOutput ?? null),
|
|
906
|
-
};
|
|
907
|
-
this.store.tx(() => {
|
|
908
|
-
existing.status = "completed";
|
|
909
|
-
existing.outcome = outcome;
|
|
910
|
-
this.store.put("effects", request.effectId, existing);
|
|
911
|
-
});
|
|
912
|
-
return { status: "completed", outcome };
|
|
913
|
-
}
|
|
914
|
-
if (agent?.status === "cancelled") {
|
|
915
|
-
const outcome = {
|
|
916
|
-
value: {
|
|
917
|
-
kind: "failed",
|
|
918
|
-
code: "agent.cancelled",
|
|
919
|
-
message: agent.error ?? "Agent turn was cancelled",
|
|
920
|
-
},
|
|
921
|
-
};
|
|
922
|
-
this.store.tx(() => {
|
|
923
|
-
existing.status = "completed";
|
|
924
|
-
existing.outcome = outcome;
|
|
925
|
-
this.store.put("effects", request.effectId, existing);
|
|
926
|
-
});
|
|
927
|
-
return { status: "completed", outcome };
|
|
928
|
-
}
|
|
929
|
-
if (agent?.status === "failed") {
|
|
930
|
-
const outcome = {
|
|
931
|
-
value: {
|
|
932
|
-
kind: "failed",
|
|
933
|
-
code: "agent.failed",
|
|
934
|
-
message: agent.error ?? "Agent turn failed",
|
|
935
|
-
},
|
|
936
|
-
};
|
|
937
|
-
this.store.tx(() => {
|
|
938
|
-
existing.status = "completed";
|
|
939
|
-
existing.outcome = outcome;
|
|
940
|
-
this.store.put("effects", request.effectId, existing);
|
|
941
|
-
});
|
|
942
|
-
return { status: "completed", outcome };
|
|
943
|
-
}
|
|
944
|
-
}
|
|
945
|
-
return { status: "pending" };
|
|
946
|
-
}
|
|
947
|
-
else {
|
|
948
|
-
return { status: "pending" };
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
const active = countActiveFlowWork(this.store, request.sessionId, request.turnId);
|
|
952
|
-
if (!mayDispatchMore(active, this.flowLimits)) {
|
|
953
|
-
if (!existing || existing.status !== "queued") {
|
|
954
|
-
this.store.tx(() => {
|
|
955
|
-
this.store.put("effects", request.effectId, {
|
|
956
|
-
request,
|
|
957
|
-
status: "queued",
|
|
958
|
-
});
|
|
959
|
-
});
|
|
960
|
-
}
|
|
961
|
-
return { status: "pending" };
|
|
962
|
-
}
|
|
963
|
-
if (request.kind === "fn" ||
|
|
964
|
-
request.kind === "verify" ||
|
|
965
|
-
isFlowToolEffect(request)) {
|
|
966
|
-
let event;
|
|
967
|
-
let started;
|
|
968
|
-
this.store.tx(() => {
|
|
969
|
-
const s = this.session(request.sessionId);
|
|
970
|
-
this.store.put("effects", request.effectId, {
|
|
971
|
-
request,
|
|
972
|
-
status: "pending",
|
|
973
|
-
});
|
|
974
|
-
const action = request.kind === "fn" || request.kind === "verify"
|
|
157
|
+
const sweepHooks = new Set();
|
|
158
|
+
ctx = {
|
|
159
|
+
config,
|
|
160
|
+
envelope,
|
|
161
|
+
store: opened,
|
|
162
|
+
vault,
|
|
163
|
+
registry,
|
|
164
|
+
mcp,
|
|
165
|
+
sandbox,
|
|
166
|
+
flowLimits,
|
|
167
|
+
modelProvider,
|
|
168
|
+
useVaultModel,
|
|
169
|
+
closing: false,
|
|
170
|
+
closed: false,
|
|
171
|
+
work: createWorkState(),
|
|
172
|
+
live,
|
|
173
|
+
workerId: hooks.workerId ?? WORKER_ID,
|
|
174
|
+
ownerLeaseMs: config.ownerLeaseMs ?? DEFAULT_OWNER_LEASE_MS,
|
|
175
|
+
wake: async (sessionId, wake) => {
|
|
176
|
+
if (ctx.closing || ctx.closed)
|
|
177
|
+
return;
|
|
178
|
+
await execution.wake(config.tenantId, sessionId, wake);
|
|
179
|
+
},
|
|
180
|
+
abortLocal: (sessionId) => abortLocal(ctx, sessionId),
|
|
181
|
+
...(hooks.execution?.stuckInvocations
|
|
975
182
|
? {
|
|
976
|
-
|
|
977
|
-
sessionId: request.sessionId,
|
|
978
|
-
turnId: request.turnId,
|
|
979
|
-
agentId: request.agentId,
|
|
980
|
-
manifestHash: request.manifestHash,
|
|
981
|
-
implementationVersion: s.implementationVersion,
|
|
982
|
-
input: request.input,
|
|
983
|
-
context: request.context,
|
|
984
|
-
status: "pending",
|
|
985
|
-
generation: 0,
|
|
986
|
-
claimId: null,
|
|
987
|
-
leaseExpiresAt: null,
|
|
988
|
-
kind: request.kind,
|
|
989
|
-
path: request.path,
|
|
990
|
-
key: request.key,
|
|
183
|
+
stuckInvocations: () => hooks.execution.stuckInvocations(config.tenantId),
|
|
991
184
|
}
|
|
992
|
-
: {
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
manifestHash: request.manifestHash,
|
|
998
|
-
implementationVersion: s.implementationVersion,
|
|
999
|
-
input: request.input,
|
|
1000
|
-
context: request.context,
|
|
1001
|
-
status: "pending",
|
|
1002
|
-
generation: 0,
|
|
1003
|
-
claimId: null,
|
|
1004
|
-
leaseExpiresAt: null,
|
|
1005
|
-
kind: "tool",
|
|
1006
|
-
path: request.path,
|
|
1007
|
-
key: request.key,
|
|
1008
|
-
};
|
|
1009
|
-
this.store.put("actions", action.actionId, action);
|
|
1010
|
-
if (isFlowToolEffect(request)) {
|
|
1011
|
-
started = this.store.event(s.id, s.activeTurnId, "node.started", {
|
|
1012
|
-
path: request.path,
|
|
1013
|
-
kind: "tool",
|
|
1014
|
-
key: request.key,
|
|
1015
|
-
...(request.iterations !== undefined
|
|
1016
|
-
? { iterations: request.iterations }
|
|
1017
|
-
: {}),
|
|
1018
|
-
});
|
|
1019
|
-
}
|
|
1020
|
-
event = this.store.event(s.id, s.activeTurnId, "action.pending", {
|
|
1021
|
-
actionId: action.actionId,
|
|
1022
|
-
kind: action.kind,
|
|
1023
|
-
path: action.path,
|
|
1024
|
-
key: action.key,
|
|
1025
|
-
input: action.input,
|
|
1026
|
-
});
|
|
1027
|
-
});
|
|
1028
|
-
if (started)
|
|
1029
|
-
this.publish(started);
|
|
1030
|
-
if (event)
|
|
1031
|
-
this.publish(event);
|
|
1032
|
-
this.notify();
|
|
1033
|
-
return { status: "pending" };
|
|
1034
|
-
}
|
|
1035
|
-
// agent effect: create linked session + message via the public contract path
|
|
1036
|
-
const body = request.input;
|
|
1037
|
-
const path = body.path ?? request.path;
|
|
1038
|
-
const workflow = this.session(request.sessionId);
|
|
1039
|
-
const agentSessionId = deriveAgentEffectSessionId(workflow.id, path, request);
|
|
1040
|
-
const iterations = request.iterations ?? "-";
|
|
1041
|
-
const n = Number(request.context.n ?? iterations.split(".")[0] ?? 1);
|
|
1042
|
-
this.store.tx(() => {
|
|
1043
|
-
this.store.put("effects", request.effectId, {
|
|
1044
|
-
request,
|
|
1045
|
-
status: "pending",
|
|
1046
|
-
agentSessionId,
|
|
1047
|
-
});
|
|
1048
|
-
this.store.put("links", agentSessionId, {
|
|
1049
|
-
workflowSessionId: workflow.id,
|
|
1050
|
-
path,
|
|
1051
|
-
effectId: request.effectId,
|
|
1052
|
-
turnId: request.turnId,
|
|
1053
|
-
});
|
|
1054
|
-
});
|
|
1055
|
-
if (!this.store.get("sessions", agentSessionId)) {
|
|
1056
|
-
const definition = this.store.get("definitions", body.agentId) ??
|
|
1057
|
-
fail(404, "Definition not found");
|
|
1058
|
-
const sandboxOwnerId = sandboxSpecOf(workflow.manifest) || workflow.sandboxOwnerId
|
|
1059
|
-
? owningSandboxSessionId(workflow, (sid) => this.store.get("sessions", sid))
|
|
1060
|
-
: undefined;
|
|
1061
|
-
const created = {
|
|
1062
|
-
id: agentSessionId,
|
|
1063
|
-
agentId: body.agentId,
|
|
1064
|
-
ownerUserId: workflow.ownerUserId,
|
|
1065
|
-
manifest: definition.manifest,
|
|
1066
|
-
manifestHash: definition.manifestHash,
|
|
1067
|
-
implementationVersion: definition.implementationVersion,
|
|
1068
|
-
status: "idle",
|
|
1069
|
-
activeTurnId: null,
|
|
1070
|
-
creation: {
|
|
1071
|
-
requestId: `flow-put-${request.effectId}`,
|
|
1072
|
-
agentId: body.agentId,
|
|
1073
|
-
ownerUserId: workflow.ownerUserId,
|
|
1074
|
-
...(sandboxOwnerId ? { sandbox: { session: sandboxOwnerId } } : {}),
|
|
185
|
+
: {}),
|
|
186
|
+
sweepHooks,
|
|
187
|
+
onSweep: (hook) => {
|
|
188
|
+
sweepHooks.add(hook);
|
|
189
|
+
return () => sweepHooks.delete(hook);
|
|
1075
190
|
},
|
|
1076
|
-
vaultIds: workflow.vaultIds,
|
|
1077
|
-
credentialSelections: workflow.credentialSelections,
|
|
1078
|
-
pluginRoots: definition.pluginRoots ?? {},
|
|
1079
|
-
...(sandboxOwnerId ? { sandboxOwnerId } : {}),
|
|
1080
|
-
};
|
|
1081
|
-
this.store.tx(() => this.store.put("sessions", agentSessionId, created));
|
|
1082
|
-
}
|
|
1083
|
-
const idempotencyKey = `${request.turnId}:${path}:${iterations}`;
|
|
1084
|
-
const messageInput = body.input;
|
|
1085
|
-
const messageCommand = typeof messageInput === "string"
|
|
1086
|
-
? {
|
|
1087
|
-
type: "message",
|
|
1088
|
-
content: messageInput,
|
|
1089
|
-
requestId: `flow-${request.effectId}`,
|
|
1090
|
-
idempotencyKey,
|
|
1091
|
-
...(body.manifest ? { manifest: body.manifest } : {}),
|
|
1092
|
-
}
|
|
1093
|
-
: {
|
|
1094
|
-
type: "message",
|
|
1095
|
-
data: messageInput,
|
|
1096
|
-
requestId: `flow-${request.effectId}`,
|
|
1097
|
-
idempotencyKey,
|
|
1098
|
-
...(body.manifest ? { manifest: body.manifest } : {}),
|
|
1099
|
-
};
|
|
1100
|
-
const accepted = this.command(agentSessionId, messageCommand, {
|
|
1101
|
-
kind: "application",
|
|
1102
|
-
principalId: "flow-host",
|
|
1103
|
-
});
|
|
1104
|
-
const events = [];
|
|
1105
|
-
this.store.tx(() => {
|
|
1106
|
-
const s = this.session(request.sessionId);
|
|
1107
|
-
const agentSession = this.session(agentSessionId);
|
|
1108
|
-
events.push(this.store.event(s.id, s.activeTurnId, "loop.iteration", {
|
|
1109
|
-
path: String(request.context.loopPath ?? path.split("/")[0]),
|
|
1110
|
-
n,
|
|
1111
|
-
sessionId: agentSessionId,
|
|
1112
|
-
turnId: accepted.turnId ?? undefined,
|
|
1113
|
-
manifestHash: agentSession.checkpoint?.manifestHash,
|
|
1114
|
-
}));
|
|
1115
|
-
events.push(this.store.event(s.id, s.activeTurnId, "node.agent", {
|
|
1116
|
-
path,
|
|
1117
|
-
iterations,
|
|
1118
|
-
sessionId: agentSessionId,
|
|
1119
|
-
turnId: accepted.turnId,
|
|
1120
|
-
}));
|
|
1121
|
-
});
|
|
1122
|
-
for (const e of events)
|
|
1123
|
-
this.publish(e);
|
|
1124
|
-
// May already be settled if the agent was fast / replayed.
|
|
1125
|
-
const agent = this.session(agentSessionId);
|
|
1126
|
-
if (agent.status === "completed") {
|
|
1127
|
-
const outcome = {
|
|
1128
|
-
value: linkedAgentOutput(agent, agent.lastOutput ?? null),
|
|
1129
191
|
};
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
agentSessionId,
|
|
1136
|
-
});
|
|
192
|
+
wired = await wireStreams(ctx, {
|
|
193
|
+
store: opened,
|
|
194
|
+
streams: hooks.streams ?? new MemoryStreams(),
|
|
195
|
+
ownsStreams: !hooks.streams,
|
|
196
|
+
tenantId: config.tenantId,
|
|
1137
197
|
});
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
}
|
|
1142
|
-
scope(request) {
|
|
1143
|
-
const header = request.headers.authorization;
|
|
1144
|
-
const token = header?.startsWith("Bearer ") ? header.slice(7) : "";
|
|
1145
|
-
if (!token || !header?.startsWith("Bearer ")) {
|
|
1146
|
-
this.config.logger.warn("credential rejected", {
|
|
1147
|
-
reason: "missing_bearer",
|
|
1148
|
-
});
|
|
1149
|
-
return failOpaque();
|
|
1150
|
-
}
|
|
1151
|
-
const tokenHash = hashToken(token);
|
|
1152
|
-
const principal = findPrincipalByTokenHash(this.store.db, tokenHash);
|
|
1153
|
-
if (principal)
|
|
1154
|
-
return { kind: "application", principalId: principal.id };
|
|
1155
|
-
const executor = this.registry.find(tokenHash);
|
|
1156
|
-
if (!executor) {
|
|
1157
|
-
this.config.logger.warn("credential rejected", {
|
|
1158
|
-
reason: "unknown_token",
|
|
198
|
+
// Outbox rows a lost relay step left behind are appended by the sweep.
|
|
199
|
+
sweepHooks.add(async () => {
|
|
200
|
+
await drainOutbox(ctx);
|
|
1159
201
|
});
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
return scope.principalId;
|
|
1171
|
-
return fail(403, "Application credential required");
|
|
1172
|
-
}
|
|
1173
|
-
sandboxRouteDeps() {
|
|
1174
|
-
return {
|
|
1175
|
-
sandbox: this.sandbox,
|
|
1176
|
-
session: (id) => this.session(id),
|
|
1177
|
-
lookup: (id) => this.store.get("sessions", id),
|
|
1178
|
-
getAction: (id) => this.store.get("actions", id),
|
|
1179
|
-
};
|
|
1180
|
-
}
|
|
1181
|
-
async body(request) {
|
|
1182
|
-
let data = "";
|
|
1183
|
-
for await (const chunk of request) {
|
|
1184
|
-
data += chunk;
|
|
1185
|
-
if (Buffer.byteLength(data) > 1024 * 1024)
|
|
1186
|
-
fail(413, "Request too large");
|
|
1187
|
-
}
|
|
1188
|
-
try {
|
|
1189
|
-
return JSON.parse(data);
|
|
1190
|
-
}
|
|
1191
|
-
catch {
|
|
1192
|
-
return fail(400, "Invalid JSON");
|
|
1193
|
-
}
|
|
1194
|
-
}
|
|
1195
|
-
command(id, input, scope) {
|
|
1196
|
-
let event;
|
|
1197
|
-
const extraEvents = [];
|
|
1198
|
-
const cascadeCancelIds = [];
|
|
1199
|
-
let schedule = false;
|
|
1200
|
-
const response = this.store.tx(() => {
|
|
1201
|
-
const s = this.session(id);
|
|
1202
|
-
let command = input;
|
|
1203
|
-
if (command.type === "action_result") {
|
|
1204
|
-
const a = this.store.get("actions", command.actionId) ??
|
|
1205
|
-
fail(404, "Action not found");
|
|
1206
|
-
if (a.sessionId !== id)
|
|
1207
|
-
fail(403, "Action belongs to another session");
|
|
1208
|
-
this.scoped(scope, a);
|
|
1209
|
-
command = acceptedToolResult(a, command);
|
|
1210
|
-
}
|
|
1211
|
-
else if (scope.kind !== "application")
|
|
1212
|
-
fail(403, "Application credential required");
|
|
1213
|
-
const key = JSON.stringify([id, command.idempotencyKey]);
|
|
1214
|
-
const existing = this.store.get("commands", key);
|
|
1215
|
-
if (existing) {
|
|
1216
|
-
if (semantic(existing.command) !== semantic(command))
|
|
1217
|
-
fail(409, "Idempotency key already binds another command");
|
|
1218
|
-
return existing.response;
|
|
1219
|
-
}
|
|
1220
|
-
if (command.type === "action_result") {
|
|
1221
|
-
const action = this.store.get("actions", command.actionId);
|
|
1222
|
-
const prior = this.store.get("effects", command.actionId);
|
|
1223
|
-
if (action.status === "completed") {
|
|
1224
|
-
if (action.claimId !== command.claimId ||
|
|
1225
|
-
action.generation !== command.generation ||
|
|
1226
|
-
canonical(prior.outcome) !== canonical(command.outcome))
|
|
1227
|
-
fail(409, "Conflicting action result");
|
|
1228
|
-
this.store.put("commands", key, { command, response: prior.receipt });
|
|
1229
|
-
return prior.receipt;
|
|
1230
|
-
}
|
|
1231
|
-
if (s.status === "cancelled" ||
|
|
1232
|
-
s.activeTurnId !== action.turnId ||
|
|
1233
|
-
action.status !== "claimed" ||
|
|
1234
|
-
action.claimId !== command.claimId ||
|
|
1235
|
-
action.generation !== command.generation ||
|
|
1236
|
-
Date.parse(action.leaseExpiresAt) <= Date.now())
|
|
1237
|
-
fail(409, "Stale, expired, or cancelled claim");
|
|
1238
|
-
action.status = "completed";
|
|
1239
|
-
this.store.put("actions", action.actionId, action);
|
|
1240
|
-
prior.status = "completed";
|
|
1241
|
-
prior.outcome = command.outcome;
|
|
1242
|
-
s.status = "runnable";
|
|
1243
|
-
schedule = true;
|
|
1244
|
-
event = this.store.event(id, s.activeTurnId, "action.completed", {
|
|
1245
|
-
actionId: action.actionId,
|
|
1246
|
-
...actionTarget(action),
|
|
1247
|
-
kind: action.kind,
|
|
1248
|
-
result: command.outcome.value,
|
|
1249
|
-
});
|
|
1250
|
-
if (action.kind === "verify") {
|
|
1251
|
-
const verdict = command.outcome.value;
|
|
1252
|
-
if (verdict?.kind !== "failed") {
|
|
1253
|
-
extraEvents.push(this.store.event(id, s.activeTurnId, "loop.verified", {
|
|
1254
|
-
path: String(action.context?.loopPath ?? action.path ?? ""),
|
|
1255
|
-
n: Number(action.context?.n ?? 1),
|
|
1256
|
-
pass: Boolean(verdict?.pass),
|
|
1257
|
-
...(verdict?.feedback !== undefined
|
|
1258
|
-
? { feedback: verdict.feedback }
|
|
1259
|
-
: {}),
|
|
1260
|
-
...(verdict?.data !== undefined ? { data: verdict.data } : {}),
|
|
1261
|
-
}));
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
else if (action.kind === "fn" && action.context?.role === "decide") {
|
|
1265
|
-
const decision = command.outcome.value;
|
|
1266
|
-
extraEvents.push(this.store.event(id, s.activeTurnId, "loop.decided", {
|
|
1267
|
-
path: String(action.context?.loopPath ?? action.path ?? ""),
|
|
1268
|
-
n: Number(action.context?.n ?? 1),
|
|
1269
|
-
next: "output" in decision && !("input" in decision)
|
|
1270
|
-
? "output"
|
|
1271
|
-
: "input",
|
|
1272
|
-
patched: Boolean(decision.agent),
|
|
1273
|
-
}));
|
|
1274
|
-
}
|
|
1275
|
-
const receipt = {
|
|
1276
|
-
status: "accepted",
|
|
1277
|
-
turnId: s.activeTurnId,
|
|
1278
|
-
cursor: event.cursor,
|
|
1279
|
-
requestId: command.requestId,
|
|
1280
|
-
};
|
|
1281
|
-
prior.receipt = receipt;
|
|
1282
|
-
this.store.put("effects", action.actionId, prior);
|
|
1283
|
-
if (isWorkflowManifest(s.manifest) && s.activeTurnId) {
|
|
1284
|
-
wakeForQueuedEffects({
|
|
1285
|
-
store: this.store,
|
|
1286
|
-
workflowSessionId: id,
|
|
1287
|
-
turnId: s.activeTurnId,
|
|
1288
|
-
limits: this.flowLimits,
|
|
1289
|
-
schedule: (sid) => {
|
|
1290
|
-
schedule = true;
|
|
1291
|
-
void sid;
|
|
1292
|
-
},
|
|
1293
|
-
});
|
|
1294
|
-
}
|
|
1295
|
-
}
|
|
1296
|
-
else if (command.type === "cancel") {
|
|
1297
|
-
const cancelledTurnId = s.activeTurnId;
|
|
1298
|
-
const workflowCancel = isWorkflowManifest(s.manifest);
|
|
1299
|
-
const cascade = workflowCancel
|
|
1300
|
-
? planCancelCascade({
|
|
1301
|
-
store: this.store,
|
|
1302
|
-
workflowSessionId: id,
|
|
1303
|
-
turnId: cancelledTurnId,
|
|
1304
|
-
})
|
|
1305
|
-
: null;
|
|
1306
|
-
s.status = "cancelled";
|
|
1307
|
-
if (workflowCancel) {
|
|
1308
|
-
fenceWorkflowActions({
|
|
1309
|
-
store: this.store,
|
|
1310
|
-
workflowSessionId: id,
|
|
1311
|
-
turnId: cancelledTurnId,
|
|
1312
|
-
});
|
|
1313
|
-
}
|
|
1314
|
-
else {
|
|
1315
|
-
for (const a of this.store.all("actions"))
|
|
1316
|
-
if (a.sessionId === id &&
|
|
1317
|
-
a.turnId === cancelledTurnId &&
|
|
1318
|
-
["pending", "claimed"].includes(a.status)) {
|
|
1319
|
-
// Claimed work may already have an external effect. Preserve it for reconciliation.
|
|
1320
|
-
a.status = a.status === "claimed" ? "uncertain" : "cancelled";
|
|
1321
|
-
this.store.put("actions", a.actionId, a);
|
|
1322
|
-
const effect = this.store.get("effects", a.actionId);
|
|
1323
|
-
effect.status = a.status;
|
|
1324
|
-
this.store.put("effects", a.actionId, effect);
|
|
1325
|
-
}
|
|
1326
|
-
}
|
|
1327
|
-
for (const effect of this.store.all("effects"))
|
|
1328
|
-
if (effect.request.sessionId === id &&
|
|
1329
|
-
effect.request.turnId === cancelledTurnId &&
|
|
1330
|
-
effect.status === "invoking") {
|
|
1331
|
-
effect.status = "uncertain";
|
|
1332
|
-
effect.error =
|
|
1333
|
-
"Turn cancelled before model outcome was durably recorded";
|
|
1334
|
-
this.store.put("effects", effect.request.effectId, effect);
|
|
1335
|
-
}
|
|
1336
|
-
event = this.store.event(id, cancelledTurnId, "turn.cancelled", {
|
|
1337
|
-
reason: command.reason,
|
|
1338
|
-
});
|
|
1339
|
-
s.activeTurnId = null;
|
|
1340
|
-
// The next turn starts from the state preceding the cancelled turn, never its paused plan.
|
|
1341
|
-
if (cancelledTurnId !== null)
|
|
1342
|
-
s.state = s.turnStartState;
|
|
1343
|
-
s.checkpoint = undefined;
|
|
1344
|
-
s.waits = undefined;
|
|
1345
|
-
s.error = undefined;
|
|
1346
|
-
// Cascade: cancel linked agent sessions deepest-first (after fencing this session).
|
|
1347
|
-
if (cascade) {
|
|
1348
|
-
for (const agentId of cascade.agentSessionIds) {
|
|
1349
|
-
const agent = this.store.get("sessions", agentId);
|
|
1350
|
-
if (!agent ||
|
|
1351
|
-
!agent.activeTurnId ||
|
|
1352
|
-
["idle", "completed", "failed", "cancelled"].includes(agent.status))
|
|
1353
|
-
continue;
|
|
1354
|
-
cascadeCancelIds.push(agentId);
|
|
1355
|
-
}
|
|
1356
|
-
}
|
|
1357
|
-
}
|
|
1358
|
-
else {
|
|
1359
|
-
if (command.type === "message") {
|
|
1360
|
-
if (!["idle", "completed", "failed", "cancelled"].includes(s.status))
|
|
1361
|
-
fail(409, "Session has active or unresolved work");
|
|
1362
|
-
s.turnStartState = s.state;
|
|
1363
|
-
s.activeTurnId = randomUUID();
|
|
1364
|
-
if (isWorkflowManifest(s.manifest)) {
|
|
1365
|
-
const input = "content" in command
|
|
1366
|
-
? command.content
|
|
1367
|
-
: command.data;
|
|
1368
|
-
s.checkpoint = createFlowCheckpoint({
|
|
1369
|
-
manifest: s.manifest,
|
|
1370
|
-
sessionId: id,
|
|
1371
|
-
turnId: s.activeTurnId,
|
|
1372
|
-
input,
|
|
1373
|
-
});
|
|
1374
|
-
}
|
|
1375
|
-
else {
|
|
1376
|
-
// Optional message.manifest: validate as variant, pin by hash, apply this turn only.
|
|
1377
|
-
const resolved = resolveMessageManifest({
|
|
1378
|
-
pinned: s.manifest,
|
|
1379
|
-
pinnedHash: s.manifestHash,
|
|
1380
|
-
messageManifest: command.manifest,
|
|
1381
|
-
store: variantStore(s),
|
|
1382
|
-
});
|
|
1383
|
-
if (!resolved.ok) {
|
|
1384
|
-
fail(400, resolved.message);
|
|
1385
|
-
}
|
|
1386
|
-
else {
|
|
1387
|
-
rebaseSessionState(s, resolved.hash);
|
|
1388
|
-
s.checkpoint = createDurableCheckpoint({
|
|
1389
|
-
manifest: resolved.manifest,
|
|
1390
|
-
sessionId: id,
|
|
1391
|
-
turnId: s.activeTurnId,
|
|
1392
|
-
input: "content" in command
|
|
1393
|
-
? command.content
|
|
1394
|
-
: typeof command.data === "string"
|
|
1395
|
-
? command.data
|
|
1396
|
-
: JSON.stringify(command.data),
|
|
1397
|
-
state: s.state,
|
|
1398
|
-
info: s.info,
|
|
1399
|
-
});
|
|
1400
|
-
}
|
|
1401
|
-
}
|
|
1402
|
-
}
|
|
1403
|
-
else {
|
|
1404
|
-
if (s.status !== "paused" || !s.state || !s.activeTurnId)
|
|
1405
|
-
fail(409, "Session is not awaiting a response");
|
|
1406
|
-
const pending = s.state.plan?.calls ?? [];
|
|
1407
|
-
if (!pending.some((c) => c.status === "interaction" &&
|
|
1408
|
-
c.interaction?.id === command.interactionId &&
|
|
1409
|
-
(command.type === "approve") ===
|
|
1410
|
-
(c.interaction?.kind === "approval")))
|
|
1411
|
-
fail(409, "Unknown interaction");
|
|
1412
|
-
const input = command.type === "approve"
|
|
1413
|
-
? {
|
|
1414
|
-
kind: "approve",
|
|
1415
|
-
interactionId: command.interactionId,
|
|
1416
|
-
approved: command.approved,
|
|
1417
|
-
}
|
|
1418
|
-
: {
|
|
1419
|
-
kind: "respond",
|
|
1420
|
-
interactionId: command.interactionId,
|
|
1421
|
-
value: command.value,
|
|
1422
|
-
};
|
|
1423
|
-
if (isWorkflowManifest(s.manifest)) {
|
|
1424
|
-
// Approvals owned by linked agent sessions must be answered there (WF-R52).
|
|
1425
|
-
const conflict = foreignInteractionConflict({
|
|
1426
|
-
store: this.store,
|
|
1427
|
-
workflowSessionId: id,
|
|
1428
|
-
interactionId: command.interactionId,
|
|
1429
|
-
});
|
|
1430
|
-
if (conflict)
|
|
1431
|
-
fail(409, conflict.message);
|
|
1432
|
-
// Workflow-owned interactions (tool-node / verify) resume on this session once
|
|
1433
|
-
// the flow engine supports pause segments; tracer root Loop has none yet.
|
|
1434
|
-
fail(409, "Workflow sessions do not accept approve/respond on the root without a pending interaction");
|
|
1435
|
-
}
|
|
1436
|
-
s.checkpoint = createDurableCheckpoint({
|
|
1437
|
-
manifest: turnManifestOf(s),
|
|
1438
|
-
sessionId: id,
|
|
1439
|
-
turnId: s.activeTurnId,
|
|
1440
|
-
input: input,
|
|
1441
|
-
state: s.state,
|
|
1442
|
-
info: s.info,
|
|
1443
|
-
segment: (s.checkpoint?.segment ?? 0) + 1,
|
|
1444
|
-
});
|
|
1445
|
-
}
|
|
1446
|
-
this.store.put("checkpoints", JSON.stringify([id, s.activeTurnId, s.checkpoint.segment]), { checkpoint: s.checkpoint, status: "runnable" });
|
|
1447
|
-
s.status = "runnable";
|
|
1448
|
-
s.waits = undefined;
|
|
1449
|
-
schedule = true;
|
|
1450
|
-
event = this.store.event(id, s.activeTurnId, `command.${command.type}`, command);
|
|
1451
|
-
}
|
|
1452
|
-
this.store.put("sessions", id, s);
|
|
1453
|
-
const response = {
|
|
1454
|
-
status: "accepted",
|
|
1455
|
-
turnId: s.activeTurnId,
|
|
1456
|
-
cursor: event?.cursor ?? this.store.history(id).cursor,
|
|
1457
|
-
requestId: command.requestId,
|
|
202
|
+
// Register the handlers, then arm the sweep: its first pass runs at once and re-wakes
|
|
203
|
+
// sessions a previous process left runnable or running (takeover handles the rest).
|
|
204
|
+
let afterOpen = true;
|
|
205
|
+
const worker = {
|
|
206
|
+
advance: (sessionId, signal) => advance(ctx, sessionId, signal),
|
|
207
|
+
sweep: async () => {
|
|
208
|
+
const first = afterOpen;
|
|
209
|
+
afterOpen = false;
|
|
210
|
+
await sweep(ctx, { afterOpen: first });
|
|
211
|
+
},
|
|
1458
212
|
};
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
// Cascade cancel linked agents deepest-first under existing fencing (WF-R53).
|
|
1469
|
-
for (const agentId of cascadeCancelIds) {
|
|
1470
|
-
try {
|
|
1471
|
-
this.command(agentId, {
|
|
1472
|
-
type: "cancel",
|
|
1473
|
-
requestId: `flow-cascade-${id}-${agentId}`,
|
|
1474
|
-
idempotencyKey: `flow-cascade-cancel:${id}:${agentId}`,
|
|
1475
|
-
reason: "workflow cancelled",
|
|
1476
|
-
}, scope);
|
|
1477
|
-
}
|
|
1478
|
-
catch {
|
|
1479
|
-
/* agent may already be terminal */
|
|
1480
|
-
}
|
|
213
|
+
const unregister = workers.register(config.tenantId, worker);
|
|
214
|
+
detach = async () => {
|
|
215
|
+
unregister();
|
|
216
|
+
await local?.stop();
|
|
217
|
+
};
|
|
218
|
+
if (local)
|
|
219
|
+
await local.start(workers.handlers);
|
|
220
|
+
await execution.armSweep(config.tenantId);
|
|
221
|
+
return new TenantRuntime(ctx, envelope, worker, detach, workers.graceMs);
|
|
1481
222
|
}
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
wakeLinkedWorkflow({
|
|
1488
|
-
agentSessionId: id,
|
|
1489
|
-
store: this.store,
|
|
1490
|
-
cancelled: true,
|
|
1491
|
-
error: "Agent turn was cancelled",
|
|
1492
|
-
schedule: (sid) => this.schedule(sid),
|
|
1493
|
-
publish: (e) => this.publish(e),
|
|
1494
|
-
});
|
|
1495
|
-
});
|
|
1496
|
-
}
|
|
223
|
+
catch (error) {
|
|
224
|
+
await detach?.().catch(() => undefined);
|
|
225
|
+
await wired?.close().catch(() => undefined);
|
|
226
|
+
await store.close().catch(() => undefined);
|
|
227
|
+
throw error;
|
|
1497
228
|
}
|
|
1498
|
-
if (schedule)
|
|
1499
|
-
this.schedule(id);
|
|
1500
|
-
return response;
|
|
1501
229
|
}
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
"content-type": "text/event-stream",
|
|
1505
|
-
"cache-control": "no-cache",
|
|
1506
|
-
connection: "keep-alive",
|
|
1507
|
-
"x-accel-buffering": "no",
|
|
1508
|
-
});
|
|
1509
|
-
response.flushHeaders();
|
|
1510
|
-
set.add(response);
|
|
1511
|
-
const timer = setInterval(() => this.send(response, ": keepalive\n\n"), 15000);
|
|
1512
|
-
timer.unref();
|
|
1513
|
-
request.on("close", () => {
|
|
1514
|
-
clearInterval(timer);
|
|
1515
|
-
set.delete(response);
|
|
1516
|
-
if (!set.size)
|
|
1517
|
-
whenEmpty?.();
|
|
1518
|
-
});
|
|
230
|
+
handle(request, response, url) {
|
|
231
|
+
return handle(this.ctx, request, response, url);
|
|
1519
232
|
}
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
const payload = JSON.stringify(value);
|
|
1523
|
-
response.writeHead(status, {
|
|
1524
|
-
"content-type": "application/json",
|
|
1525
|
-
"content-length": Buffer.byteLength(payload),
|
|
1526
|
-
});
|
|
1527
|
-
response.end(payload);
|
|
1528
|
-
};
|
|
1529
|
-
try {
|
|
1530
|
-
const url = _url ?? new URL(request.url ?? "/", "http://runtime");
|
|
1531
|
-
const path = url.pathname
|
|
1532
|
-
.split("/")
|
|
1533
|
-
.filter(Boolean)
|
|
1534
|
-
.map(decodeURIComponent);
|
|
1535
|
-
const method = request.method;
|
|
1536
|
-
const scope = this.scope(request);
|
|
1537
|
-
if (path[0] !== "v1")
|
|
1538
|
-
fail(404, "Route not found");
|
|
1539
|
-
if (path[1] === "executors" &&
|
|
1540
|
-
path[2] === "connect" &&
|
|
1541
|
-
method === "GET") {
|
|
1542
|
-
if (scope.kind !== "executor")
|
|
1543
|
-
return fail(403, "Executor credential required");
|
|
1544
|
-
const tokenHash = scope.executor.tokenHash;
|
|
1545
|
-
let streams = this.executorStreams.get(tokenHash);
|
|
1546
|
-
if (!streams)
|
|
1547
|
-
this.executorStreams.set(tokenHash, (streams = new Set()));
|
|
1548
|
-
this.sse(request, response, streams, () => this.executorStreams.delete(tokenHash));
|
|
1549
|
-
this.send(response, 'event: work_available\ndata: {"type":"work_available"}\n\n');
|
|
1550
|
-
return;
|
|
1551
|
-
}
|
|
1552
|
-
if (path[1] === "actions") {
|
|
1553
|
-
if (scope.kind !== "executor")
|
|
1554
|
-
return fail(403, "Executor credential required");
|
|
1555
|
-
this.expireClaims();
|
|
1556
|
-
if (path.length === 2 && method === "GET")
|
|
1557
|
-
return json({
|
|
1558
|
-
actions: this.store
|
|
1559
|
-
.all("actions")
|
|
1560
|
-
.filter((a) => a.status === "pending" && a.agentId === scope.executor.agentId),
|
|
1561
|
-
});
|
|
1562
|
-
const actionId = path[2];
|
|
1563
|
-
if (!actionId)
|
|
1564
|
-
fail(404, "Action not found");
|
|
1565
|
-
if (method === "POST" &&
|
|
1566
|
-
path[3] === "sandbox" &&
|
|
1567
|
-
path[4] &&
|
|
1568
|
-
path.length === 5) {
|
|
1569
|
-
const action = this.store.get("actions", actionId) ??
|
|
1570
|
-
fail(404, "Action not found");
|
|
1571
|
-
this.scoped(scope, action);
|
|
1572
|
-
try {
|
|
1573
|
-
const outcome = await handleActionSandboxTool(this.sandboxRouteDeps(), actionId, path[4], await this.body(request), requestAborted(request));
|
|
1574
|
-
return json(outcome);
|
|
1575
|
-
}
|
|
1576
|
-
catch (error) {
|
|
1577
|
-
if (error instanceof SandboxRouteError)
|
|
1578
|
-
fail(error.status, error.message);
|
|
1579
|
-
throw error;
|
|
1580
|
-
}
|
|
1581
|
-
}
|
|
1582
|
-
const body = await this.body(request);
|
|
1583
|
-
let event;
|
|
1584
|
-
const result = this.store.tx(() => {
|
|
1585
|
-
const action = this.store.get("actions", actionId) ??
|
|
1586
|
-
fail(404, "Action not found");
|
|
1587
|
-
this.scoped(scope, action);
|
|
1588
|
-
if (method === "POST" && path[3] === "claim") {
|
|
1589
|
-
ActionClaimRequestSchema.parse(body);
|
|
1590
|
-
if (action.status !== "pending" ||
|
|
1591
|
-
this.session(action.sessionId).status === "cancelled" ||
|
|
1592
|
-
this.session(action.sessionId).activeTurnId !== action.turnId)
|
|
1593
|
-
fail(409, "Action unavailable");
|
|
1594
|
-
action.status = "claimed";
|
|
1595
|
-
action.generation++;
|
|
1596
|
-
action.claimId = randomUUID();
|
|
1597
|
-
action.leaseExpiresAt = new Date(Date.now() + (this.config.leaseMs ?? 30000)).toISOString();
|
|
1598
|
-
this.store.put("actions", actionId, action);
|
|
1599
|
-
event = this.store.event(action.sessionId, action.turnId, "action.claimed", {
|
|
1600
|
-
actionId,
|
|
1601
|
-
generation: action.generation,
|
|
1602
|
-
...(action.agent ? { agent: action.agent } : {}),
|
|
1603
|
-
});
|
|
1604
|
-
return {
|
|
1605
|
-
action,
|
|
1606
|
-
claimId: action.claimId,
|
|
1607
|
-
generation: action.generation,
|
|
1608
|
-
leaseExpiresAt: action.leaseExpiresAt,
|
|
1609
|
-
};
|
|
1610
|
-
}
|
|
1611
|
-
if (method === "POST" && path[3] === "heartbeat") {
|
|
1612
|
-
const beat = ActionHeartbeatRequestSchema.parse(body);
|
|
1613
|
-
if (this.session(action.sessionId).activeTurnId !== action.turnId ||
|
|
1614
|
-
action.status !== "claimed" ||
|
|
1615
|
-
action.claimId !== beat.claimId ||
|
|
1616
|
-
action.generation !== beat.generation ||
|
|
1617
|
-
Date.parse(action.leaseExpiresAt) <= Date.now())
|
|
1618
|
-
fail(409, "Stale or expired claim");
|
|
1619
|
-
action.leaseExpiresAt = new Date(Date.now() + (this.config.leaseMs ?? 30000)).toISOString();
|
|
1620
|
-
this.store.put("actions", actionId, action);
|
|
1621
|
-
return { leaseExpiresAt: action.leaseExpiresAt };
|
|
1622
|
-
}
|
|
1623
|
-
return fail(404, "Route not found");
|
|
1624
|
-
});
|
|
1625
|
-
if (event)
|
|
1626
|
-
this.publish(event);
|
|
1627
|
-
return json(result);
|
|
1628
|
-
}
|
|
1629
|
-
if (path[1] === "sessions" &&
|
|
1630
|
-
path[2] &&
|
|
1631
|
-
path[3] === "commands" &&
|
|
1632
|
-
method === "POST")
|
|
1633
|
-
return json(this.command(path[2], SessionCommandSchema.parse(await this.body(request)), scope));
|
|
1634
|
-
if (path[1] === "vaults")
|
|
1635
|
-
return json(await this.dispatchVault(scope, method, path, url, request));
|
|
1636
|
-
if (path[1] === "tenant")
|
|
1637
|
-
return json(await this.dispatchTenant(scope, method, path, request));
|
|
1638
|
-
const principalId = this.requireApplication(scope);
|
|
1639
|
-
if (path[1] === "executors" && path.length === 2 && method === "GET")
|
|
1640
|
-
return json({
|
|
1641
|
-
executors: this.registry.list().map((e) => ({
|
|
1642
|
-
agentId: e.agentId,
|
|
1643
|
-
implementationVersion: e.implementationVersion,
|
|
1644
|
-
...(e.manifestHash === undefined
|
|
1645
|
-
? {}
|
|
1646
|
-
: { manifestHash: e.manifestHash }),
|
|
1647
|
-
connected: (this.executorStreams.get(e.tokenHash)?.size ?? 0) > 0,
|
|
1648
|
-
updatedAt: e.updatedAt,
|
|
1649
|
-
})),
|
|
1650
|
-
});
|
|
1651
|
-
// The length guard matters: the connect branch above only matches GET, so without it a
|
|
1652
|
-
// PUT to /v1/executors/connect would register an agent literally named "connect".
|
|
1653
|
-
if (path[1] === "executors" && path.length === 2 && method === "PUT") {
|
|
1654
|
-
const body = RegisterExecutorsRequestSchema.parse(await this.body(request));
|
|
1655
|
-
// Credential rules are shared with startup validation, which throws plainly; on the
|
|
1656
|
-
// wire a rejected registration is a bad request, not a server fault.
|
|
1657
|
-
const applicationHashes = applicationTokenHashes(this.store.db);
|
|
1658
|
-
try {
|
|
1659
|
-
for (const executor of body.executors)
|
|
1660
|
-
assertExecutorCredential(executor, applicationHashes);
|
|
1661
|
-
}
|
|
1662
|
-
catch (error) {
|
|
1663
|
-
fail(400, error instanceof Error ? error.message : String(error));
|
|
1664
|
-
}
|
|
1665
|
-
if (new Set(body.executors.map((e) => e.agentId)).size !==
|
|
1666
|
-
body.executors.length ||
|
|
1667
|
-
new Set(body.executors.map((e) => e.token)).size !==
|
|
1668
|
-
body.executors.length)
|
|
1669
|
-
fail(400, "Executor registrations must be unique per agent and token");
|
|
1670
|
-
const updatedAt = new Date().toISOString();
|
|
1671
|
-
const records = body.executors.map((executor) => ({
|
|
1672
|
-
agentId: executor.agentId,
|
|
1673
|
-
implementationVersion: executor.implementationVersion,
|
|
1674
|
-
...(executor.manifestHash === undefined
|
|
1675
|
-
? {}
|
|
1676
|
-
: { manifestHash: executor.manifestHash }),
|
|
1677
|
-
tokenHash: hashToken(executor.token),
|
|
1678
|
-
persisted: true,
|
|
1679
|
-
updatedAt,
|
|
1680
|
-
principalId,
|
|
1681
|
-
}));
|
|
1682
|
-
for (const record of records) {
|
|
1683
|
-
const collision = this.registry.find(record.tokenHash);
|
|
1684
|
-
if (collision && collision.agentId !== record.agentId)
|
|
1685
|
-
fail(409, "Executor tokens must be unique");
|
|
1686
|
-
// A token hash may not appear in both principals and executors (D3).
|
|
1687
|
-
if (applicationHashes.includes(record.tokenHash))
|
|
1688
|
-
fail(400, "Executor tokens require independent credentials and an agent id");
|
|
1689
|
-
}
|
|
1690
|
-
// Persist the whole batch first; the in-memory registry must never run ahead of SQLite.
|
|
1691
|
-
this.store.tx(() => {
|
|
1692
|
-
for (const record of records)
|
|
1693
|
-
this.store.putExecutor({
|
|
1694
|
-
agentId: record.agentId,
|
|
1695
|
-
tokenHash: record.tokenHash,
|
|
1696
|
-
implementationVersion: record.implementationVersion,
|
|
1697
|
-
...(record.manifestHash === undefined
|
|
1698
|
-
? {}
|
|
1699
|
-
: { manifestHash: record.manifestHash }),
|
|
1700
|
-
principalId: record.principalId,
|
|
1701
|
-
updatedAt,
|
|
1702
|
-
});
|
|
1703
|
-
});
|
|
1704
|
-
const results = records.map((record) => {
|
|
1705
|
-
const previous = this.registry.get(record.agentId);
|
|
1706
|
-
const { rotated, previousHash } = this.registry.upsert(record);
|
|
1707
|
-
if (rotated && previousHash) {
|
|
1708
|
-
for (const r of this.executorStreams.get(previousHash) ?? [])
|
|
1709
|
-
r.end();
|
|
1710
|
-
this.executorStreams.delete(previousHash);
|
|
1711
|
-
}
|
|
1712
|
-
const replacedByDifferent = previous?.principalId !== undefined &&
|
|
1713
|
-
previous.principalId !== principalId;
|
|
1714
|
-
if (replacedByDifferent)
|
|
1715
|
-
this.config.logger.warn("executor registration replaced by different application credential", {
|
|
1716
|
-
agentId: record.agentId,
|
|
1717
|
-
previousPrincipalId: previous.principalId,
|
|
1718
|
-
});
|
|
1719
|
-
return {
|
|
1720
|
-
agentId: record.agentId,
|
|
1721
|
-
implementationVersion: record.implementationVersion,
|
|
1722
|
-
rotated,
|
|
1723
|
-
...(replacedByDifferent
|
|
1724
|
-
? { replacedBy: "different-credential" }
|
|
1725
|
-
: {}),
|
|
1726
|
-
};
|
|
1727
|
-
});
|
|
1728
|
-
return json({ executors: results });
|
|
1729
|
-
}
|
|
1730
|
-
if (path[1] === "executors" &&
|
|
1731
|
-
path.length === 3 &&
|
|
1732
|
-
path[2] &&
|
|
1733
|
-
method === "DELETE") {
|
|
1734
|
-
const agentId = path[2];
|
|
1735
|
-
const existing = this.registry.get(agentId) ?? fail(404, "Executor not found");
|
|
1736
|
-
this.store.tx(() => this.store.deleteExecutor(agentId));
|
|
1737
|
-
this.registry.remove(agentId);
|
|
1738
|
-
for (const r of this.executorStreams.get(existing.tokenHash) ?? [])
|
|
1739
|
-
r.end();
|
|
1740
|
-
this.executorStreams.delete(existing.tokenHash);
|
|
1741
|
-
return json({ agentId, deleted: true });
|
|
1742
|
-
}
|
|
1743
|
-
if (path[1] === "agents" && path.length === 2 && method === "GET")
|
|
1744
|
-
return json({
|
|
1745
|
-
agents: this.store.all("definitions").map((d) => ({
|
|
1746
|
-
agentId: d.manifest.id,
|
|
1747
|
-
manifest: d.manifest,
|
|
1748
|
-
manifestHash: d.manifestHash,
|
|
1749
|
-
implementationVersion: d.implementationVersion,
|
|
1750
|
-
})),
|
|
1751
|
-
});
|
|
1752
|
-
if (path[1] === "sessions" && path.length === 2 && method === "GET")
|
|
1753
|
-
return json({
|
|
1754
|
-
sessions: this.store
|
|
1755
|
-
.all("sessions")
|
|
1756
|
-
.filter((s) => !url.searchParams.has("agentId") ||
|
|
1757
|
-
s.agentId === url.searchParams.get("agentId"))
|
|
1758
|
-
.map((s) => ({
|
|
1759
|
-
id: s.id,
|
|
1760
|
-
agentId: s.agentId,
|
|
1761
|
-
ownerUserId: s.ownerUserId,
|
|
1762
|
-
status: s.status,
|
|
1763
|
-
activeTurnId: s.activeTurnId,
|
|
1764
|
-
})),
|
|
1765
|
-
});
|
|
1766
|
-
if (path[1] === "agents" &&
|
|
1767
|
-
path[2] &&
|
|
1768
|
-
path.length === 3 &&
|
|
1769
|
-
method === "PUT") {
|
|
1770
|
-
const body = PutAgentRequestSchema.parse(await this.body(request));
|
|
1771
|
-
if (body.manifest.id !== path[2])
|
|
1772
|
-
fail(400, "Agent id mismatch");
|
|
1773
|
-
DefinitionDocumentSchema.parse(body.manifest);
|
|
1774
|
-
const definition = {
|
|
1775
|
-
...body,
|
|
1776
|
-
manifestHash: hashManifest(body.manifest),
|
|
1777
|
-
};
|
|
1778
|
-
this.store.tx(() => {
|
|
1779
|
-
this.store.put("definitions", path[2], definition);
|
|
1780
|
-
});
|
|
1781
|
-
return json({
|
|
1782
|
-
agentId: path[2],
|
|
1783
|
-
manifestHash: definition.manifestHash,
|
|
1784
|
-
implementationVersion: body.implementationVersion,
|
|
1785
|
-
});
|
|
1786
|
-
}
|
|
1787
|
-
if (path[1] === "sessions" && path[2]) {
|
|
1788
|
-
const id = path[2];
|
|
1789
|
-
if (method === "PUT" && path.length === 3) {
|
|
1790
|
-
const body = PutSessionRequestSchema.parse(await this.body(request));
|
|
1791
|
-
const vaultIds = body.vaultIds ?? [];
|
|
1792
|
-
const credentialSelections = body.credentialSelections ?? [];
|
|
1793
|
-
const prior = this.store.get("sessions", id);
|
|
1794
|
-
const definition = prior === undefined || body.sandbox
|
|
1795
|
-
? this.store.get("definitions", body.agentId) ??
|
|
1796
|
-
fail(404, "Definition not found")
|
|
1797
|
-
: undefined;
|
|
1798
|
-
const sandboxOwnerId = body.sandbox
|
|
1799
|
-
? validateSandboxAttach(body, (definition ?? prior).manifest, (sid) => this.store.get("sessions", sid))
|
|
1800
|
-
: undefined;
|
|
1801
|
-
const result = this.store.tx(() => {
|
|
1802
|
-
this.vault.assertAttachment(body.ownerUserId, vaultIds, credentialSelections);
|
|
1803
|
-
const existing = this.store.get("sessions", id);
|
|
1804
|
-
if (existing) {
|
|
1805
|
-
if (sessionIdentity(existing.creation) !== sessionIdentity(body))
|
|
1806
|
-
fail(409, "Session already exists with different creation parameters");
|
|
1807
|
-
existing.vaultIds = vaultIds;
|
|
1808
|
-
existing.credentialSelections = credentialSelections;
|
|
1809
|
-
existing.creation = body;
|
|
1810
|
-
if (sandboxOwnerId !== undefined)
|
|
1811
|
-
existing.sandboxOwnerId = sandboxOwnerId;
|
|
1812
|
-
this.store.put("sessions", id, existing);
|
|
1813
|
-
this.vault.recordAttachment(id, vaultIds);
|
|
1814
|
-
return existing;
|
|
1815
|
-
}
|
|
1816
|
-
const created = {
|
|
1817
|
-
id,
|
|
1818
|
-
agentId: body.agentId,
|
|
1819
|
-
ownerUserId: body.ownerUserId,
|
|
1820
|
-
manifest: definition.manifest,
|
|
1821
|
-
manifestHash: definition.manifestHash,
|
|
1822
|
-
implementationVersion: definition.implementationVersion,
|
|
1823
|
-
info: body.info,
|
|
1824
|
-
status: "idle",
|
|
1825
|
-
activeTurnId: null,
|
|
1826
|
-
creation: body,
|
|
1827
|
-
vaultIds,
|
|
1828
|
-
credentialSelections,
|
|
1829
|
-
pluginRoots: definition.pluginRoots ?? {},
|
|
1830
|
-
...(sandboxOwnerId !== undefined ? { sandboxOwnerId } : {}),
|
|
1831
|
-
};
|
|
1832
|
-
this.store.put("sessions", id, created);
|
|
1833
|
-
this.vault.recordAttachment(id, vaultIds);
|
|
1834
|
-
return created;
|
|
1835
|
-
});
|
|
1836
|
-
return json(this.view(result));
|
|
1837
|
-
}
|
|
1838
|
-
const s = this.session(id);
|
|
1839
|
-
if (method === "GET" && path.length === 3)
|
|
1840
|
-
return json(this.view(s));
|
|
1841
|
-
const cursor = url.searchParams.get("cursor") ??
|
|
1842
|
-
(typeof request.headers["last-event-id"] === "string"
|
|
1843
|
-
? request.headers["last-event-id"]
|
|
1844
|
-
: undefined);
|
|
1845
|
-
if (method === "GET" && path[3] === "items")
|
|
1846
|
-
return json(this.store.history(id, cursor, url.searchParams.get("agent") ?? undefined));
|
|
1847
|
-
if (method === "GET" && path[3] === "events") {
|
|
1848
|
-
const history = this.store.history(id, cursor);
|
|
1849
|
-
const set = this.observers.get(id) ?? new Set();
|
|
1850
|
-
this.observers.set(id, set);
|
|
1851
|
-
this.sse(request, response, set);
|
|
1852
|
-
for (const event of history.items)
|
|
1853
|
-
this.send(response, `id: ${event.cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
|
|
1854
|
-
return;
|
|
1855
|
-
}
|
|
1856
|
-
if (method === "POST" &&
|
|
1857
|
-
path[3] === "sandbox" &&
|
|
1858
|
-
path[4] &&
|
|
1859
|
-
path.length === 5) {
|
|
1860
|
-
this.requireApplication(scope);
|
|
1861
|
-
try {
|
|
1862
|
-
const outcome = await handleSessionSandboxTool(this.sandboxRouteDeps(), id, path[4], await this.body(request), requestAborted(request));
|
|
1863
|
-
return json(outcome);
|
|
1864
|
-
}
|
|
1865
|
-
catch (error) {
|
|
1866
|
-
if (error instanceof SandboxRouteError)
|
|
1867
|
-
fail(error.status, error.message);
|
|
1868
|
-
throw error;
|
|
1869
|
-
}
|
|
1870
|
-
}
|
|
1871
|
-
}
|
|
1872
|
-
fail(404, "Route not found");
|
|
1873
|
-
}
|
|
1874
|
-
catch (error) {
|
|
1875
|
-
if (response.headersSent) {
|
|
1876
|
-
response.end();
|
|
1877
|
-
return;
|
|
1878
|
-
}
|
|
1879
|
-
if (error instanceof OpaqueAuthError) {
|
|
1880
|
-
json(error.body, error.status);
|
|
1881
|
-
return;
|
|
1882
|
-
}
|
|
1883
|
-
const status = error instanceof HttpError ||
|
|
1884
|
-
error instanceof VaultError ||
|
|
1885
|
-
error instanceof SandboxRouteError
|
|
1886
|
-
? error.status
|
|
1887
|
-
: error?.name === "ZodError" ||
|
|
1888
|
-
error?.message === "Invalid cursor"
|
|
1889
|
-
? 400
|
|
1890
|
-
: 500;
|
|
1891
|
-
json({
|
|
1892
|
-
status: "rejected",
|
|
1893
|
-
code: status === 500 ? "internal_error" : "request_rejected",
|
|
1894
|
-
message: status === 500
|
|
1895
|
-
? "Runtime request failed"
|
|
1896
|
-
: error.message,
|
|
1897
|
-
}, status);
|
|
1898
|
-
}
|
|
233
|
+
authorize(sessionId, request) {
|
|
234
|
+
return authorize(this.ctx, sessionId, request);
|
|
1899
235
|
}
|
|
1900
|
-
|
|
236
|
+
async summary() {
|
|
237
|
+
const { store, live } = this.ctx;
|
|
238
|
+
const { counts, outbox } = await store.tx(async (t) => ({
|
|
239
|
+
counts: await t.counts(),
|
|
240
|
+
outbox: await t.outboxStats(),
|
|
241
|
+
}));
|
|
1901
242
|
return {
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
sandboxOwnerId: s.sandboxOwnerId ?? null,
|
|
1912
|
-
mcpSnapshot: s.mcpSnapshot ?? null,
|
|
1913
|
-
mcpDiagnostics: s.mcpDiagnostics ?? [],
|
|
1914
|
-
waits: (() => {
|
|
1915
|
-
if (isWorkflowManifest(s.manifest)) {
|
|
1916
|
-
const aggregated = aggregateWaits({
|
|
1917
|
-
store: this.store,
|
|
1918
|
-
workflowSessionId: s.id,
|
|
1919
|
-
});
|
|
1920
|
-
if (aggregated.length > 0)
|
|
1921
|
-
return aggregated;
|
|
1922
|
-
}
|
|
1923
|
-
return Array.isArray(s.waits)
|
|
1924
|
-
? s.waits.map((call) => ({
|
|
1925
|
-
invocationId: call.invocationId,
|
|
1926
|
-
interaction: call.interaction,
|
|
1927
|
-
wait: call.wait,
|
|
1928
|
-
status: call.status,
|
|
1929
|
-
}))
|
|
1930
|
-
: s.waits;
|
|
1931
|
-
})(),
|
|
1932
|
-
error: s.error,
|
|
1933
|
-
actions: this.store
|
|
1934
|
-
.all("actions")
|
|
1935
|
-
.filter((a) => a.sessionId === s.id &&
|
|
1936
|
-
!["completed", "cancelled"].includes(a.status)),
|
|
1937
|
-
uncertainEffects: this.store
|
|
1938
|
-
.all("effects")
|
|
1939
|
-
.filter((e) => e.request.sessionId === s.id && e.status === "uncertain")
|
|
1940
|
-
.map((e) => ({
|
|
1941
|
-
effectId: e.request.effectId,
|
|
1942
|
-
turnId: e.request.turnId,
|
|
1943
|
-
kind: e.request.kind,
|
|
1944
|
-
error: e.error,
|
|
1945
|
-
})),
|
|
243
|
+
ready: !this.ctx.closing && !this.ctx.closed,
|
|
244
|
+
runningSessions: counts.runningSessions,
|
|
245
|
+
connectedExecutors: connectedExecutorCount(live),
|
|
246
|
+
pendingActions: counts.pendingActions,
|
|
247
|
+
uncertainEffects: counts.uncertainEffects,
|
|
248
|
+
outboxDepth: outbox.depth,
|
|
249
|
+
relayLagMs: outbox.oldestCreatedAt === null
|
|
250
|
+
? 0
|
|
251
|
+
: Math.max(0, Date.now() - Date.parse(outbox.oldestCreatedAt)),
|
|
1946
252
|
};
|
|
1947
253
|
}
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
if (serversOf(s.manifest).length === 0)
|
|
1951
|
-
return;
|
|
1952
|
-
if (!s.mcpSnapshot) {
|
|
1953
|
-
const found = await this.mcp.discover({
|
|
1954
|
-
sessionId: id,
|
|
1955
|
-
manifest: s.manifest,
|
|
1956
|
-
manifestHash: s.manifestHash,
|
|
1957
|
-
pluginRoots: s.pluginRoots ?? {},
|
|
1958
|
-
signal,
|
|
1959
|
-
});
|
|
1960
|
-
this.store.tx(() => {
|
|
1961
|
-
const current = this.session(id);
|
|
1962
|
-
if (current.mcpSnapshot)
|
|
1963
|
-
return;
|
|
1964
|
-
current.mcpSnapshot = found.snapshot;
|
|
1965
|
-
current.mcpDiagnostics = found.diagnostics;
|
|
1966
|
-
this.store.put("sessions", id, current);
|
|
1967
|
-
});
|
|
1968
|
-
return;
|
|
1969
|
-
}
|
|
1970
|
-
const diagnostics = await this.mcp.reconnect({
|
|
1971
|
-
sessionId: id,
|
|
1972
|
-
manifest: s.manifest,
|
|
1973
|
-
pluginRoots: s.pluginRoots ?? {},
|
|
1974
|
-
tools: s.mcpSnapshot.mcpTools,
|
|
1975
|
-
signal,
|
|
1976
|
-
});
|
|
1977
|
-
if (diagnostics.length === 0)
|
|
1978
|
-
return;
|
|
1979
|
-
this.store.tx(() => {
|
|
1980
|
-
const current = this.session(id);
|
|
1981
|
-
const prior = [...(current.mcpDiagnostics ?? [])];
|
|
1982
|
-
for (const item of diagnostics) {
|
|
1983
|
-
const index = prior.findIndex((existing) => existing.capabilityId === item.capabilityId &&
|
|
1984
|
-
existing.serverName === item.serverName);
|
|
1985
|
-
if (index >= 0)
|
|
1986
|
-
prior[index] = item;
|
|
1987
|
-
else
|
|
1988
|
-
prior.push(item);
|
|
1989
|
-
}
|
|
1990
|
-
current.mcpDiagnostics = prior;
|
|
1991
|
-
this.store.put("sessions", id, current);
|
|
1992
|
-
});
|
|
1993
|
-
}
|
|
1994
|
-
async callMcpTool(request) {
|
|
1995
|
-
const s = this.session(request.sessionId);
|
|
1996
|
-
const tool = mcpToolOf(s, request);
|
|
1997
|
-
if (!tool)
|
|
1998
|
-
throw new Error(`MCP tool '${request.toolName ?? ""}' is not in the session snapshot`);
|
|
1999
|
-
return this.mcp.call({
|
|
2000
|
-
sessionId: s.id,
|
|
2001
|
-
...(tool.agentId === undefined ? {} : { agentId: tool.agentId }),
|
|
2002
|
-
capabilityId: tool.capabilityId,
|
|
2003
|
-
serverName: tool.serverName,
|
|
2004
|
-
serverToolName: tool.serverToolName,
|
|
2005
|
-
args: request.input,
|
|
2006
|
-
manifest: s.manifest,
|
|
2007
|
-
pluginRoots: s.pluginRoots ?? {},
|
|
2008
|
-
});
|
|
2009
|
-
}
|
|
2010
|
-
async callSandboxTool(request, signal) {
|
|
2011
|
-
const s = this.session(request.sessionId);
|
|
2012
|
-
// Agents used as tools share the session's sandbox; the tree declares one sandbox spec.
|
|
2013
|
-
const capability = sandboxCapabilityOf(manifestFor(s.manifest, request.agent), request.capabilityId, request.toolName);
|
|
2014
|
-
if (!capability)
|
|
2015
|
-
throw new Error(`'${request.toolName ?? ""}' is not a sandbox tool`);
|
|
2016
|
-
const ownerId = owningSandboxSessionId(s, (id) => this.store.get("sessions", id));
|
|
2017
|
-
return this.sandbox.run({ id: ownerId, activeTurnId: s.activeTurnId, manifest: s.manifest }, capability, request.toolName, request.input, signal);
|
|
2018
|
-
}
|
|
2019
|
-
async authorize(sessionId, request) {
|
|
2020
|
-
const s = this.session(sessionId);
|
|
2021
|
-
const result = await this.vault.authorize({
|
|
2022
|
-
sessionId,
|
|
2023
|
-
vaultIds: s.vaultIds ?? [],
|
|
2024
|
-
credentialSelections: s.credentialSelections ?? [],
|
|
2025
|
-
url: request.url,
|
|
2026
|
-
serverName: request.serverName,
|
|
2027
|
-
});
|
|
2028
|
-
if (result.status === "authorized") {
|
|
2029
|
-
const token = result.headers.authorization.slice("Bearer ".length);
|
|
2030
|
-
scrub({ authorization: result.headers.authorization, url: result.url }, [
|
|
2031
|
-
token,
|
|
2032
|
-
]);
|
|
2033
|
-
}
|
|
2034
|
-
return result;
|
|
2035
|
-
}
|
|
2036
|
-
ensureKek() {
|
|
2037
|
-
if (this.kek)
|
|
2038
|
-
return this.kek;
|
|
2039
|
-
if (!this.createKekIfMissing)
|
|
2040
|
-
throw new QuarantineError("kek-missing", "Vault key-encryption key is required", "restore the vault-kek file beside tenant.sqlite");
|
|
2041
|
-
this.kek = createKekFile(this.kekPath);
|
|
2042
|
-
return this.kek;
|
|
254
|
+
drain(activeWork, timeoutMs = 30_000) {
|
|
255
|
+
return drain(this.ctx, activeWork, timeoutMs);
|
|
2043
256
|
}
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
return buildTenantStatus({
|
|
2051
|
-
envelope: this.envelope,
|
|
2052
|
-
config: this.config,
|
|
2053
|
-
store: this.store,
|
|
2054
|
-
registry: this.registry,
|
|
2055
|
-
vault: this.vault,
|
|
2056
|
-
sandbox: this.sandbox,
|
|
2057
|
-
closing: this.closing || this.closed,
|
|
2058
|
-
modelConfigured: this.useVaultModel
|
|
2059
|
-
? this.vault.getHostModel().configured
|
|
2060
|
-
: true,
|
|
2061
|
-
executorStreams: this.executorStreams,
|
|
2062
|
-
});
|
|
2063
|
-
if (path[2] === "reset" && path.length === 3 && method === "POST") {
|
|
2064
|
-
const body = ResetTenantRequestSchema.parse(await this.body(request));
|
|
2065
|
-
await this.drain(body.activeWork);
|
|
2066
|
-
await resetTenant({
|
|
2067
|
-
store: this.store,
|
|
2068
|
-
registry: this.registry,
|
|
2069
|
-
sandbox: this.sandbox,
|
|
2070
|
-
paths: this.config.paths,
|
|
2071
|
-
clearSessionState: () => {
|
|
2072
|
-
this.pending.clear();
|
|
2073
|
-
for (const c of this.running.values())
|
|
2074
|
-
c.abort();
|
|
2075
|
-
this.running.clear();
|
|
2076
|
-
for (const set of this.observers.values())
|
|
2077
|
-
for (const r of set)
|
|
2078
|
-
r.end();
|
|
2079
|
-
this.observers.clear();
|
|
2080
|
-
},
|
|
2081
|
-
clearExecutorStreams: () => {
|
|
2082
|
-
for (const streams of this.executorStreams.values())
|
|
2083
|
-
for (const r of streams)
|
|
2084
|
-
r.end();
|
|
2085
|
-
this.executorStreams.clear();
|
|
2086
|
-
},
|
|
2087
|
-
}, body.scope);
|
|
2088
|
-
// Reset leaves the Tenant open for new work.
|
|
2089
|
-
this.closing = false;
|
|
2090
|
-
return { ok: true };
|
|
2091
|
-
}
|
|
2092
|
-
if (path[2] === "config" &&
|
|
2093
|
-
path[3] === "seed" &&
|
|
2094
|
-
path.length === 4 &&
|
|
2095
|
-
method === "PUT") {
|
|
2096
|
-
const body = SeedTenantConfigRequestSchema.parse(await this.body(request));
|
|
2097
|
-
return seedTenantConfig({ store: this.store, vault: this.vault }, body);
|
|
2098
|
-
}
|
|
2099
|
-
if (path[2] === "models" && path.length === 3 && method === "GET")
|
|
2100
|
-
return hostModelCatalog();
|
|
2101
|
-
if (path[2] === "sandbox" && path.length === 3 && method === "GET")
|
|
2102
|
-
return this.sandbox.report();
|
|
2103
|
-
if (path[2] === "providers" && path.length === 3 && method === "GET")
|
|
2104
|
-
return this.vault.listHostProviders();
|
|
2105
|
-
if (path[2] === "model" && path.length === 3 && method === "GET")
|
|
2106
|
-
return this.vault.getHostModel();
|
|
2107
|
-
if (path[2] === "model" && path.length === 3 && method === "PUT")
|
|
2108
|
-
return this.vault.putHostModel(PutHostModelRequestSchema.parse(await this.body(request)));
|
|
2109
|
-
if (path[2] === "model" &&
|
|
2110
|
-
path[3] === "selection" &&
|
|
2111
|
-
path.length === 4 &&
|
|
2112
|
-
method === "PUT")
|
|
2113
|
-
return this.vault.selectHostModel(SelectHostModelRequestSchema.parse(await this.body(request)));
|
|
2114
|
-
fail(404, "Route not found");
|
|
2115
|
-
}
|
|
2116
|
-
async dispatchVault(scope, method, path, url, request) {
|
|
2117
|
-
if (scope.kind !== "application") {
|
|
2118
|
-
this.vault.reject(path.join("/"));
|
|
2119
|
-
fail(403, "Application credential required");
|
|
2120
|
-
}
|
|
2121
|
-
if (path.length === 2 && method === "POST") {
|
|
2122
|
-
const body = CreateVaultRequestSchema.parse(await this.body(request));
|
|
2123
|
-
return this.vault.createVault(body);
|
|
2124
|
-
}
|
|
2125
|
-
if (path.length === 2 && method === "GET") {
|
|
2126
|
-
const ownerUserId = url.searchParams.get("ownerUserId") ??
|
|
2127
|
-
fail(400, "ownerUserId is required");
|
|
2128
|
-
return { vaults: this.vault.listVaults(ownerUserId) };
|
|
2129
|
-
}
|
|
2130
|
-
const vaultId = path[2];
|
|
2131
|
-
if (!vaultId)
|
|
2132
|
-
fail(404, "Vault not found");
|
|
2133
|
-
if (path.length === 3 && method === "GET")
|
|
2134
|
-
return this.vault.getVault(vaultId);
|
|
2135
|
-
if (path.length === 3 && method === "DELETE")
|
|
2136
|
-
return this.vault.deleteVault(vaultId);
|
|
2137
|
-
if (path[3] !== "credentials")
|
|
2138
|
-
fail(404, "Route not found");
|
|
2139
|
-
if (path.length === 4 && method === "POST") {
|
|
2140
|
-
const body = CreateCredentialRequestSchema.parse(await this.body(request));
|
|
2141
|
-
return this.vault.createCredential(vaultId, body);
|
|
2142
|
-
}
|
|
2143
|
-
if (path.length === 4 && method === "GET")
|
|
2144
|
-
return { credentials: this.vault.listCredentials(vaultId) };
|
|
2145
|
-
const credentialId = path[4];
|
|
2146
|
-
if (!credentialId)
|
|
2147
|
-
fail(404, "Credential not found");
|
|
2148
|
-
if (path.length === 5 && method === "GET")
|
|
2149
|
-
return this.vault.getCredential(vaultId, credentialId);
|
|
2150
|
-
if (path.length === 5 && method === "POST") {
|
|
2151
|
-
const body = RotateCredentialRequestSchema.parse(await this.body(request));
|
|
2152
|
-
return this.vault.rotateCredential(vaultId, credentialId, body);
|
|
2153
|
-
}
|
|
2154
|
-
if (path.length === 5 && method === "DELETE")
|
|
2155
|
-
return this.vault.deleteCredential(vaultId, credentialId);
|
|
2156
|
-
fail(404, "Route not found");
|
|
257
|
+
/**
|
|
258
|
+
* Aborts the advance of `sessionId` if it runs on this process. Cancel calls it after
|
|
259
|
+
* committing `cancelled`; the control stream calls it for cancels made elsewhere.
|
|
260
|
+
*/
|
|
261
|
+
abortLocal(sessionId) {
|
|
262
|
+
this.ctx.abortLocal(sessionId);
|
|
2157
263
|
}
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
.filter((a) => a.status === "pending" || a.status === "claimed").length;
|
|
2165
|
-
const uncertainEffects = this.store
|
|
2166
|
-
.all("effects")
|
|
2167
|
-
.filter((e) => e.status === "uncertain").length;
|
|
2168
|
-
return {
|
|
2169
|
-
ready: !this.closing && !this.closed,
|
|
2170
|
-
runningSessions,
|
|
2171
|
-
connectedExecutors,
|
|
2172
|
-
pendingActions,
|
|
2173
|
-
uncertainEffects,
|
|
2174
|
-
};
|
|
264
|
+
/**
|
|
265
|
+
* The Durable Streams seam's status for Tenant status and readiness: S2 reachability, the
|
|
266
|
+
* basin, outbox depth and relay lag (`streamsStatus` in `tenant/streams.ts`).
|
|
267
|
+
*/
|
|
268
|
+
streamsStatus() {
|
|
269
|
+
return streamsStatus(this.ctx);
|
|
2175
270
|
}
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
for (const sess of this.store.all("sessions"))
|
|
2180
|
-
if (sess.activeTurnId &&
|
|
2181
|
-
["running", "runnable", "paused"].includes(sess.status))
|
|
2182
|
-
this.command(sess.id, {
|
|
2183
|
-
type: "cancel",
|
|
2184
|
-
requestId: randomUUID(),
|
|
2185
|
-
idempotencyKey: `drain-cancel-${sess.id}-${sess.activeTurnId}`,
|
|
2186
|
-
reason: "tenant drain cancel",
|
|
2187
|
-
}, {
|
|
2188
|
-
kind: "application",
|
|
2189
|
-
principalId: "drain",
|
|
2190
|
-
});
|
|
2191
|
-
}
|
|
2192
|
-
const deadline = Date.now() + timeoutMs;
|
|
2193
|
-
while (this.running.size > 0 && Date.now() < deadline)
|
|
2194
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
271
|
+
/** Adds a callback to the Tenant sweep. Returns a function that removes it. */
|
|
272
|
+
onSweep(hook) {
|
|
273
|
+
return this.ctx.onSweep(hook);
|
|
2195
274
|
}
|
|
2196
275
|
async close() {
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
await
|
|
2211
|
-
this.closed = true;
|
|
2212
|
-
this.store.db.close();
|
|
2213
|
-
if (this.lockPath && existsSync(this.lockPath))
|
|
2214
|
-
unlinkSync(this.lockPath);
|
|
276
|
+
const ctx = this.ctx;
|
|
277
|
+
ctx.closing = true;
|
|
278
|
+
// A shutdown abort: the advances leave their sessions to the next advance, unsettled.
|
|
279
|
+
abortAll(ctx, "shutdown");
|
|
280
|
+
const idleBy = Date.now() + this.closeGraceMs;
|
|
281
|
+
await this.detach();
|
|
282
|
+
await ctx.mcp.close();
|
|
283
|
+
endAllStreams(ctx.live);
|
|
284
|
+
// Bounded: an advance that ignores its abort is abandoned; its lease lapses (§11.4).
|
|
285
|
+
await waitForIdle(ctx, Math.max(0, idleBy - Date.now()));
|
|
286
|
+
await ctx.sandbox.close();
|
|
287
|
+
ctx.closed = true;
|
|
288
|
+
await closeStreams(ctx);
|
|
289
|
+
await ctx.store.close();
|
|
2215
290
|
}
|
|
2216
291
|
}
|
|
2217
|
-
function readEnvelope(config) {
|
|
2218
|
-
if (existsSync(config.paths.envelope)) {
|
|
2219
|
-
return JSON.parse(readFileSync(config.paths.envelope, "utf8"));
|
|
2220
|
-
}
|
|
2221
|
-
const now = new Date().toISOString();
|
|
2222
|
-
return {
|
|
2223
|
-
id: config.tenantId,
|
|
2224
|
-
name: config.tenantId,
|
|
2225
|
-
createdAt: now,
|
|
2226
|
-
updatedAt: now,
|
|
2227
|
-
schemaVersion: 1,
|
|
2228
|
-
};
|
|
2229
|
-
}
|
|
2230
292
|
/**
|
|
2231
293
|
* Open a Tenant Runtime from an explicit `TenantConfig`. No listen() (A1).
|
|
2232
294
|
* Implements `OpenTenantRuntime`.
|
|
2233
295
|
*/
|
|
2234
|
-
export async function openTenantRuntime(config,
|
|
2235
|
-
|
|
2236
|
-
hooks) {
|
|
2237
|
-
return TenantRuntime.open(config, hooks ?? {});
|
|
2238
|
-
}
|
|
2239
|
-
/** What an action runs, for events: a tool name, or a hook point and its capabilities. */
|
|
2240
|
-
function actionTarget(action) {
|
|
2241
|
-
if (action.kind === "hook")
|
|
2242
|
-
return {
|
|
2243
|
-
hook: action.hook,
|
|
2244
|
-
...(action.agent ? { agent: action.agent } : {}),
|
|
2245
|
-
};
|
|
2246
|
-
if (action.kind === "tool" && "toolName" in action)
|
|
2247
|
-
return {
|
|
2248
|
-
toolName: action.toolName,
|
|
2249
|
-
...(action.agent ? { agent: action.agent } : {}),
|
|
2250
|
-
};
|
|
2251
|
-
return {
|
|
2252
|
-
path: action.path,
|
|
2253
|
-
key: action.key,
|
|
2254
|
-
...(action.agent ? { agent: action.agent } : {}),
|
|
2255
|
-
};
|
|
296
|
+
export async function openTenantRuntime(config, hooks) {
|
|
297
|
+
return TenantRuntime.open(config, hooks);
|
|
2256
298
|
}
|