@nylorun/runtime 0.21.1-beta → 0.23.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +17 -3
- package/dist/adapters/execution/restate.d.ts +15 -3
- package/dist/adapters/execution/restate.js +77 -83
- package/dist/adapters/sandbox/local.js +1 -1
- package/dist/adapters/sandbox/virtual.js +1 -1
- package/dist/adapters/streams/s2.js +1 -13
- package/dist/api/components.d.ts +42 -2
- package/dist/core/flow-host.d.ts +1 -19
- package/dist/core/flow-host.js +8 -37
- package/dist/execution/types.d.ts +10 -8
- package/dist/execution/types.js +5 -7
- package/dist/gates/egress-credentials.d.ts +48 -0
- package/dist/gates/egress-credentials.js +147 -0
- package/dist/gates/egress.d.ts +3 -0
- package/dist/gates/egress.js +14 -1
- package/dist/gates/tenant-vaults.d.ts +4 -0
- package/dist/gates/tenant-vaults.js +2 -0
- package/dist/harness/calls.d.ts +2 -0
- package/dist/harness/calls.js +1 -0
- package/dist/harness/executors.js +1 -0
- package/dist/harness/main.js +4 -1
- package/dist/harness/pod.d.ts +2 -0
- package/dist/harness/pod.js +36 -1
- package/dist/harness-api/start.d.ts +2 -0
- package/dist/harness-api/start.js +4 -2
- package/dist/harness-api/ws-server.js +2 -2
- package/dist/host/execution.d.ts +8 -6
- package/dist/host/gates.js +11 -0
- package/dist/host/main.js +10 -16
- package/dist/host/stack-config.d.ts +6 -0
- package/dist/host/stack-config.js +6 -0
- package/dist/infra/execution.d.ts +18 -1
- package/dist/infra/execution.js +28 -4
- package/dist/infra/index.d.ts +1 -1
- package/dist/keys/egress-ca.d.ts +15 -0
- package/dist/keys/egress-ca.js +66 -0
- package/dist/keys/keys.d.ts +8 -1
- package/dist/keys/keys.js +6 -0
- package/dist/keys/x509.d.ts +29 -0
- package/dist/keys/x509.js +160 -0
- package/dist/management-openapi.json +195 -9
- package/dist/openapi.json +2 -1
- package/dist/retry.d.ts +21 -0
- package/dist/retry.js +34 -0
- package/dist/sandbox/join.d.ts +2 -0
- package/dist/sandbox/join.js +14 -10
- package/dist/sandbox/manager.d.ts +2 -0
- package/dist/sandbox/manager.js +1 -1
- package/dist/sandbox/pods/reconcile.js +3 -3
- package/dist/sandbox/tools.d.ts +3 -1
- package/dist/sandbox/tools.js +4 -2
- package/dist/sandbox/types.d.ts +5 -0
- package/dist/store/postgres/control.js +8 -1
- package/dist/store/postgres/drizzle/0018_wake_outbox_retries.sql +3 -0
- package/dist/store/postgres/drizzle/0019_sandbox_signal_outbox.sql +18 -0
- package/dist/store/postgres/drizzle/meta/_journal.json +14 -0
- package/dist/store/postgres/schema.d.ts +265 -4
- package/dist/store/postgres/schema.js +35 -4
- package/dist/store/postgres/store.js +165 -52
- package/dist/store/types.d.ts +96 -39
- package/dist/streams/relay/core.js +1 -13
- package/dist/tenant/advance.js +6 -7
- package/dist/tenant/commands.js +1 -2
- package/dist/tenant/context.d.ts +19 -11
- package/dist/tenant/control.d.ts +8 -1
- package/dist/tenant/control.js +2 -0
- package/dist/tenant/runtime.js +23 -3
- package/dist/tenant/sandboxes.d.ts +2 -1
- package/dist/tenant/sandboxes.js +23 -19
- package/dist/tenant/scheduler.d.ts +7 -0
- package/dist/tenant/scheduler.js +38 -7
- package/dist/tenant/session-streams.d.ts +0 -1
- package/dist/tenant/session-streams.js +1 -13
- package/dist/tenant/sessions.js +26 -0
- package/dist/tenant/streams.js +14 -29
- package/dist/tenant/sweep.d.ts +39 -12
- package/dist/tenant/sweep.js +157 -58
- package/dist/tenant/transcript.d.ts +8 -8
- package/dist/vault/service.d.ts +34 -0
- package/dist/vault/service.js +148 -15
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +4 -3
package/dist/tenant/sweep.js
CHANGED
|
@@ -4,26 +4,34 @@
|
|
|
4
4
|
* and startup scan the Tenant runtime used to keep. It reconciles the Session Store with what
|
|
5
5
|
* Durable Session Execution was asked to do; it never decides when a session advances:
|
|
6
6
|
*
|
|
7
|
-
* 1. **
|
|
8
|
-
* after commit that failed, or that a crash between the commit and the send cut
|
|
9
|
-
* first pass on this process takes every row, so a previous process's are not
|
|
10
|
-
* waiting; later passes take
|
|
11
|
-
*
|
|
12
|
-
*
|
|
7
|
+
* 1. **Outbox.** Outbox rows (`Tx.wake`, `Tx.signalSandbox`) are delivered again and deleted:
|
|
8
|
+
* a delivery after commit that failed, or that a crash between the commit and the send cut
|
|
9
|
+
* off. The first pass on this process takes every row, so a previous process's are not
|
|
10
|
+
* left waiting; later passes take those whose delivery after commit failed and those older
|
|
11
|
+
* than `OUTBOX_GRACE_MS` (the database's clock), batch after batch within
|
|
12
|
+
* `DRAIN_BUDGET_MS`. Each row carries the idempotency key it was first sent with, so a
|
|
13
|
+
* request the execution did accept before the crash, or one still being sent after its
|
|
14
|
+
* commit, runs once. Rows are delivered one by one: a row that fails is retried with
|
|
15
|
+
* backoff and parked after `MAX_OUTBOX_ATTEMPTS` (`deliverOutbox`), and no delivery fails
|
|
16
|
+
* the step.
|
|
13
17
|
* 2. **Linked agents.** Pending workflow `agent` effects whose linked turn already settled
|
|
14
|
-
* are completed and the workflow is woken
|
|
15
|
-
*
|
|
18
|
+
* are completed and the workflow is woken. A linked turn's settle does both in its own
|
|
19
|
+
* transaction; this is the backstop for one that settle left pending, such as an effect
|
|
20
|
+
* an older Runtime wrote.
|
|
16
21
|
* 3. **Orphaned sessions.** On this process's first pass for the Tenant, then at most every
|
|
17
22
|
* `ORPHAN_SCAN_MS`: `running` or `runnable` sessions with no owner, or an owner whose lease
|
|
18
|
-
* expired, are woken with reason `recover
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
+
* expired, are woken with reason `recover`, page by page within `DRAIN_BUDGET_MS`; a scan
|
|
24
|
+
* the budget cut short goes on at the next pass. Neither a lost wake (step 1) nor a Worker
|
|
25
|
+
* that died mid-advance (the execution retries that advance, which takes over, §11.4)
|
|
26
|
+
* needs it: it is the backstop for what the execution itself lost or ended without an
|
|
27
|
+
* advance, such as an in-process execution's queue when its process stopped, an advance
|
|
28
|
+
* for a Tenant not open on its Worker, an invocation an operator killed, Restate's state
|
|
29
|
+
* wiped (§17.11), or a parked outbox row.
|
|
23
30
|
* 4. **Sandboxes.** Idle sandboxes are stopped, records of compute no longer held are marked
|
|
24
31
|
* stopped, and sandboxes whose session or sandbox resource is gone are removed, by the
|
|
25
32
|
* workspace capability (`harness-api/workspace.ts`: here, or in the harness that serves
|
|
26
|
-
* workspaces). Idle MCP connections of the in-process harness are closed.
|
|
33
|
+
* workspaces). Idle MCP connections of the in-process harness are closed. As often as the
|
|
34
|
+
* orphan scan, pod sandboxes whose change stalled get a reconcile (`reconcileStalePods`).
|
|
27
35
|
* 5. **Control signals.** Signals on the control bus older than `SIGNAL_RETENTION_MS` are
|
|
28
36
|
* deleted: followers read back only the last two minutes (`store/postgres/control.ts`).
|
|
29
37
|
* 6. **Hooks.** Callbacks registered with `ctx.onSweep`.
|
|
@@ -38,14 +46,28 @@ const BATCH = 100;
|
|
|
38
46
|
/** How long a control signal is kept after it was written. */
|
|
39
47
|
const SIGNAL_RETENTION_MS = 60 * 60 * 1000;
|
|
40
48
|
/**
|
|
41
|
-
* How old
|
|
42
|
-
*
|
|
43
|
-
*
|
|
49
|
+
* How old an outbox row is before the sweep delivers it, unless its delivery after commit
|
|
50
|
+
* already failed (then it is due at once): longer than that delivery can take with its
|
|
51
|
+
* retries, so the sweep does not send again, and double the load on, an execution that is
|
|
52
|
+
* only slow. Sending one twice is harmless either way (the idempotency key).
|
|
53
|
+
*/
|
|
54
|
+
export const OUTBOX_GRACE_MS = 30_000;
|
|
55
|
+
/**
|
|
56
|
+
* How often the orphan scan, and the stale pod sandbox check, run after the first pass of a
|
|
57
|
+
* Tenant opened on this process.
|
|
44
58
|
*/
|
|
45
|
-
export const WAKE_GRACE_MS = 2000;
|
|
46
|
-
/** How often the orphan scan runs after the first pass of a Tenant opened on this process. */
|
|
47
59
|
export const ORPHAN_SCAN_MS = 60_000;
|
|
48
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* How long one pass may spend draining the outbox, and paging through orphaned sessions.
|
|
62
|
+
* A backlog is worked off over several passes rather than holding one (and with it the
|
|
63
|
+
* Tenant's sweep object in the execution) for long.
|
|
64
|
+
*/
|
|
65
|
+
export const DRAIN_BUDGET_MS = 2000;
|
|
66
|
+
/**
|
|
67
|
+
* Each open Tenant's sweep state on this process: when its last orphan scan finished, the
|
|
68
|
+
* last session id of one that ran out of budget, which the next pass goes on from, and when
|
|
69
|
+
* the stale pod sandbox check last ran.
|
|
70
|
+
*/
|
|
49
71
|
const passes = new WeakMap();
|
|
50
72
|
export async function sweep(ctx) {
|
|
51
73
|
if (ctx.closing || ctx.closed)
|
|
@@ -54,20 +76,22 @@ export async function sweep(ctx) {
|
|
|
54
76
|
const first = !passes.has(ctx);
|
|
55
77
|
const pass = passes.get(ctx) ?? {};
|
|
56
78
|
passes.set(ctx, pass);
|
|
57
|
-
const scanOrphans = pass.orphansAt === undefined ||
|
|
79
|
+
const scanOrphans = pass.orphansAt === undefined ||
|
|
80
|
+
pass.orphansAfter !== undefined ||
|
|
81
|
+
now.getTime() - pass.orphansAt >= ORPHAN_SCAN_MS;
|
|
82
|
+
const checkPods = pass.podsAt === undefined || now.getTime() - pass.podsAt >= ORPHAN_SCAN_MS;
|
|
58
83
|
const steps = [
|
|
59
|
-
[
|
|
60
|
-
"wakes",
|
|
61
|
-
() => deliverPendingWakes(ctx, first ? now : new Date(now.getTime() - WAKE_GRACE_MS)),
|
|
62
|
-
],
|
|
84
|
+
["outbox", () => deliverOutbox(ctx, first ? 0 : OUTBOX_GRACE_MS)],
|
|
63
85
|
["linked", () => reconcileLinkedAgents(ctx)],
|
|
64
86
|
...(scanOrphans
|
|
65
87
|
? [
|
|
66
88
|
[
|
|
67
89
|
"orphans",
|
|
68
90
|
async () => {
|
|
69
|
-
await wakeOrphanedSessions(ctx, now);
|
|
70
|
-
pass.
|
|
91
|
+
const scan = await wakeOrphanedSessions(ctx, now, pass.orphansAfter);
|
|
92
|
+
pass.orphansAfter = scan.next;
|
|
93
|
+
if (scan.next === undefined)
|
|
94
|
+
pass.orphansAt = now.getTime();
|
|
71
95
|
},
|
|
72
96
|
],
|
|
73
97
|
]
|
|
@@ -80,7 +104,17 @@ export async function sweep(ctx) {
|
|
|
80
104
|
sandboxExists: async (id) => (await ctx.store.tx((t) => t.sandboxResource(id))) !== undefined,
|
|
81
105
|
}),
|
|
82
106
|
],
|
|
83
|
-
|
|
107
|
+
...(checkPods
|
|
108
|
+
? [
|
|
109
|
+
[
|
|
110
|
+
"pods",
|
|
111
|
+
async () => {
|
|
112
|
+
await reconcileStalePods(ctx, now);
|
|
113
|
+
pass.podsAt = now.getTime();
|
|
114
|
+
},
|
|
115
|
+
],
|
|
116
|
+
]
|
|
117
|
+
: []),
|
|
84
118
|
// A harness elsewhere closes its own idle MCP connections.
|
|
85
119
|
["mcp", async () => ctx.mcp?.sweep()],
|
|
86
120
|
[
|
|
@@ -107,53 +141,118 @@ export async function sweep(ctx) {
|
|
|
107
141
|
if (failure)
|
|
108
142
|
throw failure.error;
|
|
109
143
|
}
|
|
144
|
+
/** Failed deliveries after which the sweep parks an outbox row (`Tx.failOutbox`). */
|
|
145
|
+
export const MAX_OUTBOX_ATTEMPTS = 10;
|
|
146
|
+
/** How long after its first failed delivery the sweep tries a row again; doubled each time. */
|
|
147
|
+
const RETRY_MS = 5000;
|
|
148
|
+
const RETRY_MAX_MS = 10 * 60_000;
|
|
110
149
|
/**
|
|
111
|
-
* Deliver the
|
|
112
|
-
*
|
|
113
|
-
*
|
|
150
|
+
* Deliver the outbox rows that are due (`Tx.pendingOutbox`: older than `graceMs`, or whose
|
|
151
|
+
* delivery failed), oldest first, batch after batch until none is left or `budgetMs` is
|
|
152
|
+
* spent, and delete the delivered ones. Each row is delivered on its own: one that fails
|
|
153
|
+
* (Restate refuses it, or a Runtime that does not know a wake's reason) is tried again later,
|
|
154
|
+
* with backoff, so it never holds back the rows behind it, and after `MAX_OUTBOX_ATTEMPTS`
|
|
155
|
+
* failures it is parked: logged once, at error, and kept in the outbox for an operator. A
|
|
156
|
+
* parked wake's session is still re-woken by the orphan scan while it is runnable, and a
|
|
157
|
+
* parked sandbox signal's pod by the stale pod check. A row the Tenant declines (it is
|
|
158
|
+
* closing) stays as it is, and ends the drain. Never throws for a delivery; returns the rows
|
|
159
|
+
* delivered.
|
|
114
160
|
*/
|
|
115
|
-
export async function
|
|
116
|
-
const
|
|
161
|
+
export async function deliverOutbox(ctx, graceMs = 0, budgetMs = DRAIN_BUDGET_MS) {
|
|
162
|
+
const deadline = Date.now() + budgetMs;
|
|
117
163
|
const delivered = [];
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
164
|
+
for (;;) {
|
|
165
|
+
const pending = await ctx.store.tx((t) => t.pendingOutbox(graceMs, BATCH));
|
|
166
|
+
const batch = [];
|
|
167
|
+
let done = pending.length < BATCH;
|
|
168
|
+
try {
|
|
169
|
+
for (const row of pending) {
|
|
170
|
+
if (Date.now() >= deadline) {
|
|
171
|
+
done = true;
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
let accepted;
|
|
175
|
+
try {
|
|
176
|
+
accepted = await ctx.send(row.request);
|
|
177
|
+
}
|
|
178
|
+
catch (error) {
|
|
179
|
+
await failed(ctx, row, error);
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
if (accepted)
|
|
183
|
+
batch.push(row);
|
|
184
|
+
else
|
|
185
|
+
done = true;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
finally {
|
|
189
|
+
// Delivered ones go even when a later step fails, so they are not sent again.
|
|
190
|
+
if (batch.length > 0)
|
|
191
|
+
await ctx.store.tx((t) => t.deleteOutbox(batch));
|
|
192
|
+
}
|
|
193
|
+
delivered.push(...batch);
|
|
194
|
+
if (done)
|
|
195
|
+
return delivered;
|
|
127
196
|
}
|
|
128
|
-
|
|
197
|
+
}
|
|
198
|
+
/** Records a failed delivery of `row`: retried with backoff, parked after the last attempt. */
|
|
199
|
+
async function failed(ctx, row, error) {
|
|
200
|
+
const attempts = row.attempts + 1;
|
|
201
|
+
const park = attempts >= MAX_OUTBOX_ATTEMPTS;
|
|
202
|
+
const retryInMs = Math.min(RETRY_MS * 2 ** (attempts - 1), RETRY_MAX_MS);
|
|
203
|
+
await ctx.store.tx((t) => t.failOutbox(row, { retryInMs, park }));
|
|
204
|
+
const { request } = row;
|
|
205
|
+
const fields = {
|
|
206
|
+
rowId: row.id,
|
|
207
|
+
...(request.kind === "wake"
|
|
208
|
+
? { sessionId: request.sessionId, reason: request.wake.reason }
|
|
209
|
+
: { sandboxId: request.sandboxId, signal: request.signal.kind }),
|
|
210
|
+
attempts,
|
|
211
|
+
message: error instanceof Error ? error.message : String(error),
|
|
212
|
+
};
|
|
213
|
+
if (park)
|
|
214
|
+
ctx.config.logger.error("outbox row parked: its delivery keeps failing", fields);
|
|
215
|
+
else
|
|
216
|
+
ctx.config.logger.warn("outbox delivery failed; the sweep tries it again", fields);
|
|
129
217
|
}
|
|
130
218
|
/** Settle pending workflow `agent` effects whose linked turn already finished. */
|
|
131
219
|
export async function reconcileLinkedAgents(ctx) {
|
|
132
220
|
const { store } = ctx;
|
|
133
221
|
const effects = await store.tx((t) => pendingAgentEffects(t));
|
|
134
222
|
for (const effect of effects)
|
|
135
|
-
await store.tx((t) => reconcilePendingAgentEffect({
|
|
136
|
-
t,
|
|
137
|
-
effectId: effect.request.effectId,
|
|
138
|
-
schedule: ctx.wake,
|
|
139
|
-
}));
|
|
223
|
+
await store.tx((t) => reconcilePendingAgentEffect({ t, effectId: effect.request.effectId }));
|
|
140
224
|
}
|
|
141
|
-
/**
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
225
|
+
/**
|
|
226
|
+
* Wake `running`/`runnable` sessions without a live owner, page after page from the first id
|
|
227
|
+
* after `after`, until none is left or `budgetMs` is spent. Returns the ids woken, and the last
|
|
228
|
+
* id when the budget ran out first (`next`, where the next scan goes on).
|
|
229
|
+
*/
|
|
230
|
+
export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetMs = DRAIN_BUDGET_MS) {
|
|
231
|
+
const deadline = Date.now() + budgetMs;
|
|
232
|
+
const woken = [];
|
|
233
|
+
let cursor = after;
|
|
234
|
+
for (;;) {
|
|
235
|
+
const page = await ctx.store.tx((t) => t.orphanedSessions(now, BATCH, cursor));
|
|
236
|
+
for (const session of page) {
|
|
237
|
+
await ctx.wake(session.id, { reason: "recover" });
|
|
238
|
+
woken.push(session.id);
|
|
239
|
+
cursor = session.id;
|
|
240
|
+
}
|
|
241
|
+
if (page.length < BATCH)
|
|
242
|
+
return { woken };
|
|
243
|
+
if (Date.now() >= deadline)
|
|
244
|
+
return { woken, next: cursor };
|
|
245
|
+
}
|
|
148
246
|
}
|
|
149
247
|
/** A pod sandbox in a change this long without a reconcile gets one from the sweep. */
|
|
150
248
|
const STALE_POD_MS = 60_000;
|
|
151
249
|
/**
|
|
152
250
|
* Pod sandboxes (F7.2) whose change has not finished (creating, deleting, an old volume still
|
|
153
251
|
* to delete, an expiry not yet recorded), or that should run but whose engine is not connected,
|
|
154
|
-
* and whose row has not moved for a minute
|
|
155
|
-
*
|
|
156
|
-
*
|
|
252
|
+
* and whose row has not moved for a minute, get a reconcile: a pod or volume deleted outside
|
|
253
|
+
* the Runtime is found, and a change goes on whose reconcile the execution itself lost (its
|
|
254
|
+
* retry timer, an invocation an operator killed, Restate's state wiped) or whose outbox row
|
|
255
|
+
* was parked. A reconcile a commit asked for is never lost: it is in the outbox.
|
|
157
256
|
*/
|
|
158
257
|
async function reconcileStalePods(ctx, now) {
|
|
159
258
|
if (!ctx.pods)
|
|
@@ -103,6 +103,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
|
103
103
|
capabilityId: string;
|
|
104
104
|
toolName: string;
|
|
105
105
|
} | {
|
|
106
|
+
redacted?: undefined;
|
|
106
107
|
agent: import("@nylorun/core/define").AgentRef;
|
|
107
108
|
error: {
|
|
108
109
|
code: string;
|
|
@@ -112,10 +113,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
|
112
113
|
retryable?: boolean | undefined;
|
|
113
114
|
};
|
|
114
115
|
output?: undefined;
|
|
115
|
-
redacted?: undefined;
|
|
116
116
|
capabilityId: string;
|
|
117
117
|
toolName: string;
|
|
118
118
|
} | {
|
|
119
|
+
redacted?: undefined;
|
|
119
120
|
agent?: undefined;
|
|
120
121
|
error: {
|
|
121
122
|
code: string;
|
|
@@ -125,7 +126,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
|
125
126
|
retryable?: boolean | undefined;
|
|
126
127
|
};
|
|
127
128
|
output?: undefined;
|
|
128
|
-
redacted?: undefined;
|
|
129
129
|
capabilityId: string;
|
|
130
130
|
toolName: string;
|
|
131
131
|
} | {
|
|
@@ -143,17 +143,17 @@ export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
|
143
143
|
capabilityId: string;
|
|
144
144
|
toolName: string;
|
|
145
145
|
} | {
|
|
146
|
+
redacted?: undefined;
|
|
146
147
|
agent: import("@nylorun/core/define").AgentRef;
|
|
147
148
|
error?: undefined;
|
|
148
149
|
output: unknown;
|
|
149
|
-
redacted?: undefined;
|
|
150
150
|
capabilityId: string;
|
|
151
151
|
toolName: string;
|
|
152
152
|
} | {
|
|
153
|
+
redacted?: undefined;
|
|
153
154
|
agent?: undefined;
|
|
154
155
|
error?: undefined;
|
|
155
156
|
output: unknown;
|
|
156
|
-
redacted?: undefined;
|
|
157
157
|
capabilityId: string;
|
|
158
158
|
toolName: string;
|
|
159
159
|
} | {
|
|
@@ -187,6 +187,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
|
187
187
|
capabilityId: string;
|
|
188
188
|
toolName: string;
|
|
189
189
|
} | {
|
|
190
|
+
redacted?: undefined;
|
|
190
191
|
callId: string;
|
|
191
192
|
invocationId: string;
|
|
192
193
|
agent: import("@nylorun/core/define").AgentRef;
|
|
@@ -198,10 +199,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
|
198
199
|
retryable?: boolean | undefined;
|
|
199
200
|
};
|
|
200
201
|
output?: undefined;
|
|
201
|
-
redacted?: undefined;
|
|
202
202
|
capabilityId: string;
|
|
203
203
|
toolName: string;
|
|
204
204
|
} | {
|
|
205
|
+
redacted?: undefined;
|
|
205
206
|
callId: string;
|
|
206
207
|
invocationId: string;
|
|
207
208
|
agent?: undefined;
|
|
@@ -213,7 +214,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
|
213
214
|
retryable?: boolean | undefined;
|
|
214
215
|
};
|
|
215
216
|
output?: undefined;
|
|
216
|
-
redacted?: undefined;
|
|
217
217
|
capabilityId: string;
|
|
218
218
|
toolName: string;
|
|
219
219
|
} | {
|
|
@@ -235,21 +235,21 @@ export declare function toolCompleted(request: HostEffect, value: unknown): {
|
|
|
235
235
|
capabilityId: string;
|
|
236
236
|
toolName: string;
|
|
237
237
|
} | {
|
|
238
|
+
redacted?: undefined;
|
|
238
239
|
callId: string;
|
|
239
240
|
invocationId: string;
|
|
240
241
|
agent: import("@nylorun/core/define").AgentRef;
|
|
241
242
|
error?: undefined;
|
|
242
243
|
output: unknown;
|
|
243
|
-
redacted?: undefined;
|
|
244
244
|
capabilityId: string;
|
|
245
245
|
toolName: string;
|
|
246
246
|
} | {
|
|
247
|
+
redacted?: undefined;
|
|
247
248
|
callId: string;
|
|
248
249
|
invocationId: string;
|
|
249
250
|
agent?: undefined;
|
|
250
251
|
error?: undefined;
|
|
251
252
|
output: unknown;
|
|
252
|
-
redacted?: undefined;
|
|
253
253
|
capabilityId: string;
|
|
254
254
|
toolName: string;
|
|
255
255
|
} | undefined;
|
package/dist/vault/service.d.ts
CHANGED
|
@@ -76,6 +76,40 @@ export declare class VaultService {
|
|
|
76
76
|
}): Promise<void>;
|
|
77
77
|
/** Writes the `attach` audit row. Run in the session PUT transaction. */
|
|
78
78
|
recordAttachment(t: Tx, sessionId: string, vaultIds: readonly string[]): Promise<void>;
|
|
79
|
+
/**
|
|
80
|
+
* The variables a session's sandbox commands get (R2c, D50): each `environment_secret`'s name
|
|
81
|
+
* set to the sentinel, each `environment_variable`'s value, and each host a secret is bound to
|
|
82
|
+
* with that secret's credential id. `conflict` says why they are ambiguous: two credentials with one variable name, or two
|
|
83
|
+
* secrets bound to one host. Reads rows only; no plaintext. Run in the caller's transaction.
|
|
84
|
+
*/
|
|
85
|
+
sessionEnvironment(t: Tx, vaultIds: readonly string[]): Promise<{
|
|
86
|
+
environment: Record<string, string>;
|
|
87
|
+
secretHosts: Record<string, string>;
|
|
88
|
+
conflict?: string;
|
|
89
|
+
}>;
|
|
90
|
+
/**
|
|
91
|
+
* The header egress-gate sets on a request from `sandboxId` to `host` (R2c, D50): the
|
|
92
|
+
* `environment_secret` bound to `host` among the vaults of the sessions using the sandbox, its
|
|
93
|
+
* value unsealed and rendered into its format. `none` when no secret is bound to the host;
|
|
94
|
+
* `refused` when two are, or the sealed value cannot be read. One query and one unseal per
|
|
95
|
+
* call; nothing is cached (E3). Plaintext exists only in the returned header.
|
|
96
|
+
*/
|
|
97
|
+
releaseEnvironmentSecret(input: {
|
|
98
|
+
sandboxId: string;
|
|
99
|
+
host: string;
|
|
100
|
+
}): Promise<{
|
|
101
|
+
status: "none";
|
|
102
|
+
} | {
|
|
103
|
+
status: "released";
|
|
104
|
+
header: string;
|
|
105
|
+
value: string;
|
|
106
|
+
credentialId: string;
|
|
107
|
+
} | {
|
|
108
|
+
status: "refused";
|
|
109
|
+
reason: "ambiguous" | "unreadable";
|
|
110
|
+
}>;
|
|
111
|
+
/** Whether any `environment_secret` of the sessions using `sandboxId` is bound to `host`. */
|
|
112
|
+
environmentSecretBound(sandboxId: string, host: string): Promise<boolean>;
|
|
79
113
|
authorize(input: {
|
|
80
114
|
/** The session the request is made for; absent for a tool preview (R2b C12). */
|
|
81
115
|
sessionId?: string;
|
package/dist/vault/service.js
CHANGED
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
* credential is chosen.
|
|
75
75
|
*/
|
|
76
76
|
import { createHash, randomUUID } from "node:crypto";
|
|
77
|
-
import { INSTALLATION_OWNER } from "@nylorun/core/contracts";
|
|
77
|
+
import { ENVIRONMENT_SECRET_DEFAULT_INJECT, ENVIRONMENT_SECRET_SENTINEL, INSTALLATION_OWNER, renderInjectFormat, } from "@nylorun/core/contracts";
|
|
78
78
|
import { canonical } from "../store/canonical.js";
|
|
79
79
|
import { decryptSecret, encryptSecret, VaultCryptoError } from "./crypto.js";
|
|
80
80
|
import { VaultError } from "./error.js";
|
|
@@ -156,11 +156,18 @@ export class VaultService {
|
|
|
156
156
|
return this.store.tx((t) => this.replay(t, `create-credential:${vaultId}:${body.idempotencyKey}`, body, async () => {
|
|
157
157
|
await this.vaultInfo(t, vaultId);
|
|
158
158
|
const id = randomUUID();
|
|
159
|
-
const url = normalizeVaultUrl(body.auth.url);
|
|
160
159
|
const type = body.auth.type;
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
160
|
+
let payload;
|
|
161
|
+
let binding;
|
|
162
|
+
if (body.auth.type === "environment_secret" || body.auth.type === "environment_variable") {
|
|
163
|
+
({ payload, binding } = environmentCredential(body.auth));
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
const url = normalizeVaultUrl(body.auth.url);
|
|
167
|
+
payload = payloadOf(body.auth);
|
|
168
|
+
binding = bindingOf(url, payload, body.auth);
|
|
169
|
+
}
|
|
170
|
+
const sealed = encryptSecret(kek, credentialAad(vaultId, id, type, binding), Buffer.from(JSON.stringify(payload), "utf8"));
|
|
164
171
|
const row = {
|
|
165
172
|
id,
|
|
166
173
|
vaultId,
|
|
@@ -201,11 +208,23 @@ export class VaultService {
|
|
|
201
208
|
if (row.type !== body.auth.type)
|
|
202
209
|
throw new VaultError(409, "Credential type cannot change");
|
|
203
210
|
const current = credentialBinding(row);
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
211
|
+
let next;
|
|
212
|
+
let binding;
|
|
213
|
+
if (body.auth.type === "environment_secret") {
|
|
214
|
+
next = { value: body.auth.secretValue };
|
|
215
|
+
binding = current;
|
|
216
|
+
}
|
|
217
|
+
else if (body.auth.type === "environment_variable") {
|
|
218
|
+
next = {};
|
|
219
|
+
binding = { ...current, variableValue: body.auth.variableValue };
|
|
220
|
+
}
|
|
221
|
+
else {
|
|
222
|
+
next = payloadOf(body.auth);
|
|
223
|
+
binding = bindingOf(current.url, next, {
|
|
224
|
+
via: body.auth.via === undefined ? current.via : body.auth.via,
|
|
225
|
+
identity: body.auth.identity === undefined ? current.identity : body.auth.identity,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
209
228
|
const updated = await this.writePayload(t, kek, row, next, binding, new Date().toISOString());
|
|
210
229
|
await this.audit(t, {
|
|
211
230
|
actor: "application",
|
|
@@ -271,6 +290,65 @@ export class VaultService {
|
|
|
271
290
|
outcome: "attached",
|
|
272
291
|
});
|
|
273
292
|
}
|
|
293
|
+
/**
|
|
294
|
+
* The variables a session's sandbox commands get (R2c, D50): each `environment_secret`'s name
|
|
295
|
+
* set to the sentinel, each `environment_variable`'s value, and each host a secret is bound to
|
|
296
|
+
* with that secret's credential id. `conflict` says why they are ambiguous: two credentials with one variable name, or two
|
|
297
|
+
* secrets bound to one host. Reads rows only; no plaintext. Run in the caller's transaction.
|
|
298
|
+
*/
|
|
299
|
+
async sessionEnvironment(t, vaultIds) {
|
|
300
|
+
const environment = {};
|
|
301
|
+
const hosts = new Map();
|
|
302
|
+
let conflict;
|
|
303
|
+
for (const vaultId of vaultIds)
|
|
304
|
+
for (const row of await t.credentialsForVault(vaultId)) {
|
|
305
|
+
if (row.type !== "environment_secret" && row.type !== "environment_variable")
|
|
306
|
+
continue;
|
|
307
|
+
const binding = JSON.parse(row.bindingJson);
|
|
308
|
+
const name = row.type === "environment_secret" ? binding.secretName : binding.variableName;
|
|
309
|
+
if (name in environment)
|
|
310
|
+
conflict ??= `Two attached credentials set the variable ${name}`;
|
|
311
|
+
environment[name] = row.type === "environment_secret" ? ENVIRONMENT_SECRET_SENTINEL : binding.variableValue;
|
|
312
|
+
for (const host of row.type === "environment_secret" ? binding.allowedHosts ?? [] : []) {
|
|
313
|
+
if (hosts.has(host) && hosts.get(host) !== row.id)
|
|
314
|
+
conflict ??= `Two attached environment_secret credentials are bound to ${host}`;
|
|
315
|
+
hosts.set(host, row.id);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
return { environment, secretHosts: Object.fromEntries([...hosts].sort()), ...(conflict ? { conflict } : {}) };
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* The header egress-gate sets on a request from `sandboxId` to `host` (R2c, D50): the
|
|
322
|
+
* `environment_secret` bound to `host` among the vaults of the sessions using the sandbox, its
|
|
323
|
+
* value unsealed and rendered into its format. `none` when no secret is bound to the host;
|
|
324
|
+
* `refused` when two are, or the sealed value cannot be read. One query and one unseal per
|
|
325
|
+
* call; nothing is cached (E3). Plaintext exists only in the returned header.
|
|
326
|
+
*/
|
|
327
|
+
async releaseEnvironmentSecret(input) {
|
|
328
|
+
const found = await this.store.tx((t) => sandboxSecretsFor(t, input.sandboxId, input.host));
|
|
329
|
+
if (found.length === 0)
|
|
330
|
+
return { status: "none" };
|
|
331
|
+
if (found.length > 1)
|
|
332
|
+
return { status: "refused", reason: "ambiguous" };
|
|
333
|
+
const row = found[0];
|
|
334
|
+
let payload;
|
|
335
|
+
try {
|
|
336
|
+
payload = readPayload(this.kek(), row);
|
|
337
|
+
}
|
|
338
|
+
catch (error) {
|
|
339
|
+
if (!(error instanceof VaultCryptoError))
|
|
340
|
+
throw error;
|
|
341
|
+
return { status: "refused", reason: "unreadable" };
|
|
342
|
+
}
|
|
343
|
+
if (!payload.value)
|
|
344
|
+
return { status: "refused", reason: "unreadable" };
|
|
345
|
+
const inject = credentialBinding(row).inject ?? ENVIRONMENT_SECRET_DEFAULT_INJECT;
|
|
346
|
+
return { status: "released", header: inject.header, value: renderInjectFormat(inject.format, payload.value), credentialId: row.id };
|
|
347
|
+
}
|
|
348
|
+
/** Whether any `environment_secret` of the sessions using `sandboxId` is bound to `host`. */
|
|
349
|
+
async environmentSecretBound(sandboxId, host) {
|
|
350
|
+
return (await this.store.tx((t) => sandboxSecretsFor(t, sandboxId, host))).length > 0;
|
|
351
|
+
}
|
|
274
352
|
// --- use ---------------------------------------------------------------------
|
|
275
353
|
async authorize(input) {
|
|
276
354
|
const url = normalizeVaultUrl(input.url);
|
|
@@ -502,7 +580,7 @@ export class VaultService {
|
|
|
502
580
|
}
|
|
503
581
|
/** Re-seals a user credential's payload with its new binding and returns the updated row. */
|
|
504
582
|
async writePayload(t, kek, row, payload, binding, rotatedAt) {
|
|
505
|
-
const sealed = encryptSecret(kek, credentialAad(row.vaultId, row.id, row.type,
|
|
583
|
+
const sealed = encryptSecret(kek, credentialAad(row.vaultId, row.id, row.type, binding), Buffer.from(JSON.stringify(payload), "utf8"));
|
|
506
584
|
const patch = { rotatedAt, bindingJson: JSON.stringify(binding), ...sealed };
|
|
507
585
|
if (!(await t.updateCredential(row.vaultId, row.id, patch)))
|
|
508
586
|
throw new VaultError(404, "Credential not found");
|
|
@@ -649,7 +727,15 @@ function credentialInfoOf(row) {
|
|
|
649
727
|
vaultId: row.vaultId,
|
|
650
728
|
name: row.name,
|
|
651
729
|
type: row.type,
|
|
652
|
-
binding:
|
|
730
|
+
binding: row.type === "environment_secret"
|
|
731
|
+
? {
|
|
732
|
+
secretName: binding.secretName,
|
|
733
|
+
allowedHosts: binding.allowedHosts,
|
|
734
|
+
inject: binding.inject ?? { ...ENVIRONMENT_SECRET_DEFAULT_INJECT },
|
|
735
|
+
}
|
|
736
|
+
: row.type === "environment_variable"
|
|
737
|
+
? { variableName: binding.variableName, variableValue: binding.variableValue }
|
|
738
|
+
: { url: binding.url },
|
|
653
739
|
...(binding.headerNames === undefined ? {} : { headerNames: binding.headerNames }),
|
|
654
740
|
...(binding.via === undefined ? {} : { via: binding.via }),
|
|
655
741
|
...(binding.identity === undefined ? {} : { identity: binding.identity }),
|
|
@@ -658,7 +744,7 @@ function credentialInfoOf(row) {
|
|
|
658
744
|
};
|
|
659
745
|
}
|
|
660
746
|
function readPayload(kek, row) {
|
|
661
|
-
const aad = credentialAad(row.vaultId, row.id, row.type,
|
|
747
|
+
const aad = credentialAad(row.vaultId, row.id, row.type, credentialBinding(row));
|
|
662
748
|
const plaintext = decryptSecret(kek, aad, Buffer.from(row.nonce), Buffer.from(row.ciphertext), Buffer.from(row.wrappedDek), row.kekId);
|
|
663
749
|
try {
|
|
664
750
|
return JSON.parse(plaintext.toString("utf8"));
|
|
@@ -804,10 +890,57 @@ export function hostModelAad(provider, model) {
|
|
|
804
890
|
model,
|
|
805
891
|
}), "utf8");
|
|
806
892
|
}
|
|
807
|
-
|
|
808
|
-
|
|
893
|
+
/**
|
|
894
|
+
* What a user credential's ciphertext is bound to: its vault, id, type and URL; for an
|
|
895
|
+
* `environment_secret` (R2c) also its variable, hosts and header, so the value can never be sent
|
|
896
|
+
* anywhere else without a new seal.
|
|
897
|
+
*/
|
|
898
|
+
function credentialAad(vaultId, credentialId, type, binding) {
|
|
899
|
+
return Buffer.from(canonical(type === "environment_secret"
|
|
900
|
+
? {
|
|
901
|
+
vaultId,
|
|
902
|
+
credentialId,
|
|
903
|
+
type,
|
|
904
|
+
secretName: binding.secretName,
|
|
905
|
+
allowedHosts: binding.allowedHosts,
|
|
906
|
+
inject: binding.inject ?? null,
|
|
907
|
+
}
|
|
908
|
+
: { vaultId, credentialId, type, url: binding.url }), "utf8");
|
|
909
|
+
}
|
|
910
|
+
/** The sealed payload and unsealed binding of an environment credential (R2c). */
|
|
911
|
+
function environmentCredential(auth) {
|
|
912
|
+
if (auth.type === "environment_variable")
|
|
913
|
+
return { payload: {}, binding: { variableName: auth.variableName, variableValue: auth.variableValue } };
|
|
914
|
+
if (auth.inject)
|
|
915
|
+
checkCredentialHeaders([auth.inject.header.toLowerCase()], undefined);
|
|
916
|
+
return {
|
|
917
|
+
payload: { value: auth.secretValue },
|
|
918
|
+
binding: {
|
|
919
|
+
secretName: auth.secretName,
|
|
920
|
+
allowedHosts: auth.allowedHosts.map((host) => host.toLowerCase()),
|
|
921
|
+
...(auth.inject ? { inject: { header: auth.inject.header, format: auth.inject.format } } : {}),
|
|
922
|
+
},
|
|
923
|
+
};
|
|
809
924
|
}
|
|
810
925
|
function requestHash(body) {
|
|
811
926
|
const { requestId: _requestId, ...rest } = body;
|
|
812
927
|
return canonical(rest);
|
|
813
928
|
}
|
|
929
|
+
/**
|
|
930
|
+
* The `environment_secret` credentials bound to `host` in the vaults of every session on the
|
|
931
|
+
* sandbox (R2c). A session between turns counts: it can run again, and what it started may
|
|
932
|
+
* still run. Session open keeps them to one owner and one credential per host.
|
|
933
|
+
*/
|
|
934
|
+
async function sandboxSecretsFor(t, sandboxId, host) {
|
|
935
|
+
const lower = host.toLowerCase();
|
|
936
|
+
const vaultIds = new Set();
|
|
937
|
+
for (const session of await t.sessionsOnSandbox(sandboxId))
|
|
938
|
+
for (const vaultId of session.vaultIds ?? [])
|
|
939
|
+
vaultIds.add(vaultId);
|
|
940
|
+
const matches = [];
|
|
941
|
+
for (const vaultId of vaultIds)
|
|
942
|
+
for (const row of await t.credentialsForVault(vaultId))
|
|
943
|
+
if (row.type === "environment_secret" && (credentialBinding(row).allowedHosts ?? []).includes(lower))
|
|
944
|
+
matches.push(row);
|
|
945
|
+
return matches;
|
|
946
|
+
}
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const RUNTIME_VERSION = "0.
|
|
1
|
+
export declare const RUNTIME_VERSION = "0.23.0-beta";
|
package/dist/version.js
CHANGED