@tangle-network/agent-app 0.63.3 → 0.63.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat-routes/cancellation.d.ts +12 -1
- package/dist/chat-routes/index.js +3 -3
- package/dist/chat-routes/stale-turn-lock.d.ts +17 -1
- package/dist/chat-routes/turn-ownership.d.ts +1 -0
- package/dist/chat-routes/turn-route-contracts.d.ts +13 -3
- package/dist/{chunk-POIVNKYH.js → chunk-72ASTUPN.js} +2 -2
- package/dist/{chunk-464WRGJ3.js → chunk-MT6ZTQGD.js} +315 -114
- package/dist/chunk-MT6ZTQGD.js.map +1 -0
- package/dist/{chunk-PGWKYLHC.js → chunk-UXK5EBQT.js} +34 -4
- package/dist/chunk-UXK5EBQT.js.map +1 -0
- package/dist/preset-cloudflare/index.js +1 -1
- package/dist/public-consultation/index.d.ts +1 -1
- package/dist/public-consultation/index.js +3 -3
- package/dist/sandbox/index.js +2 -2
- package/dist/turn-stream/adapters.d.ts +13 -11
- package/dist/turn-stream/cancellation-do.d.ts +31 -0
- package/dist/turn-stream/cancellation.d.ts +23 -0
- package/dist/turn-stream/core.d.ts +23 -4
- package/dist/turn-stream/do.d.ts +4 -2
- package/dist/turn-stream/index.d.ts +1 -0
- package/dist/turn-stream/index.js +186 -14
- package/dist/turn-stream/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-464WRGJ3.js.map +0 -1
- package/dist/chunk-PGWKYLHC.js.map +0 -1
- /package/dist/{chunk-POIVNKYH.js.map → chunk-72ASTUPN.js.map} +0 -0
|
@@ -4,118 +4,48 @@ import {
|
|
|
4
4
|
hmacSha256Base64Url
|
|
5
5
|
} from "./chunk-S5SRJJQG.js";
|
|
6
6
|
|
|
7
|
-
// src/chat-routes/stale-turn-lock.ts
|
|
8
|
-
var DEFAULT_STALE_TURN_LOCK_GRACE_MS = 5 * 60 * 1e3;
|
|
9
|
-
var DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS = 60 * 1e3;
|
|
10
|
-
function messageOf(err) {
|
|
11
|
-
return err instanceof Error ? err.message : String(err);
|
|
12
|
-
}
|
|
13
|
-
async function reconcileStaleTurnLock(options) {
|
|
14
|
-
let sandbox;
|
|
15
|
-
try {
|
|
16
|
-
sandbox = await options.probeSandbox();
|
|
17
|
-
} catch (err) {
|
|
18
|
-
return forceReleaseUnreachable(options, {
|
|
19
|
-
unreachableReason: "SANDBOX_PROBE_FAILED",
|
|
20
|
-
unreachableDetail: messageOf(err)
|
|
21
|
-
});
|
|
22
|
-
}
|
|
23
|
-
if (sandbox.status !== "running") {
|
|
24
|
-
return forceReleaseUnreachable(options, {
|
|
25
|
-
unreachableReason: sandbox.status === "absent" ? "SANDBOX_ABSENT" : "SANDBOX_NOT_RUNNING",
|
|
26
|
-
...sandbox.status === "not-running" && sandbox.state !== void 0 ? { sandboxState: sandbox.state } : {}
|
|
27
|
-
});
|
|
28
|
-
}
|
|
29
|
-
const observedAt = (options.now ?? Date.now)();
|
|
30
|
-
let session;
|
|
31
|
-
try {
|
|
32
|
-
session = await options.probeSession();
|
|
33
|
-
} catch (err) {
|
|
34
|
-
return forceReleaseUnreachable(options, {
|
|
35
|
-
unreachableReason: "SESSION_PROBE_FAILED",
|
|
36
|
-
unreachableDetail: messageOf(err)
|
|
37
|
-
});
|
|
38
|
-
}
|
|
39
|
-
if (!session.reachable) {
|
|
40
|
-
return forceReleaseUnreachable(options, {
|
|
41
|
-
unreachableReason: "SESSION_UNREACHABLE",
|
|
42
|
-
...session.reason !== void 0 ? { sessionProbeError: session.reason } : {}
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
const diagnostics = {
|
|
46
|
-
sandboxReachable: true,
|
|
47
|
-
sessionTerminal: session.terminal,
|
|
48
|
-
...session.diagnostics ?? {}
|
|
49
|
-
};
|
|
50
|
-
if (!session.terminal) return { released: false, diagnostics };
|
|
51
|
-
const terminalGraceMs = options.terminalGraceMs ?? DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS;
|
|
52
|
-
const lockAgeMs = observedAt - options.lockStartedAt;
|
|
53
|
-
if (lockAgeMs < terminalGraceMs) {
|
|
54
|
-
const log = options.log ?? ((message, meta) => console.warn(message, meta));
|
|
55
|
-
const withheld = {
|
|
56
|
-
...diagnostics,
|
|
57
|
-
lockAgeMs,
|
|
58
|
-
terminalReleaseGraceMs: terminalGraceMs,
|
|
59
|
-
terminalReleaseWithheld: "LOCK_WITHIN_TERMINAL_GRACE_PERIOD"
|
|
60
|
-
};
|
|
61
|
-
log("[chat-routes] stale turn lock held: terminal verdict but lock is younger than the registration window", {
|
|
62
|
-
...options.context ?? {},
|
|
63
|
-
...withheld
|
|
64
|
-
});
|
|
65
|
-
return { released: false, diagnostics: withheld };
|
|
66
|
-
}
|
|
67
|
-
const released = await options.release({ observedAt });
|
|
68
|
-
return {
|
|
69
|
-
released,
|
|
70
|
-
diagnostics: { ...diagnostics, lockAgeMs, terminalReleaseGraceMs: terminalGraceMs, released, observedAt }
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
async function forceReleaseUnreachable(options, reason) {
|
|
74
|
-
const log = options.log ?? ((message, meta) => console.warn(message, meta));
|
|
75
|
-
const graceMs = options.graceMs ?? DEFAULT_STALE_TURN_LOCK_GRACE_MS;
|
|
76
|
-
const at = (options.now ?? Date.now)();
|
|
77
|
-
const lockAgeMs = at - options.lockStartedAt;
|
|
78
|
-
const diagnostics = {
|
|
79
|
-
...reason,
|
|
80
|
-
sandboxReachable: false,
|
|
81
|
-
lockAgeMs,
|
|
82
|
-
forceReleaseGraceMs: graceMs
|
|
83
|
-
};
|
|
84
|
-
if (lockAgeMs < graceMs) {
|
|
85
|
-
log("[chat-routes] stale turn lock held: sandbox unreachable but lock is inside the grace period", {
|
|
86
|
-
...options.context ?? {},
|
|
87
|
-
...diagnostics
|
|
88
|
-
});
|
|
89
|
-
return { released: false, diagnostics: { ...diagnostics, forceReleaseWithheld: "LOCK_WITHIN_GRACE_PERIOD" } };
|
|
90
|
-
}
|
|
91
|
-
const released = await options.release({ observedAt: at });
|
|
92
|
-
log("[chat-routes] force-released stale turn lock: sandbox unreachable, no turn can be executing", {
|
|
93
|
-
...options.context ?? {},
|
|
94
|
-
...diagnostics,
|
|
95
|
-
released
|
|
96
|
-
});
|
|
97
|
-
return { released, diagnostics: { ...diagnostics, forceReleased: released } };
|
|
98
|
-
}
|
|
99
|
-
|
|
100
7
|
// src/turn-stream/core.ts
|
|
101
8
|
var TURN_STREAM_TOKEN_TTL_MS = 6e4;
|
|
102
9
|
var TURN_STREAM_TOKEN_HEADER = "x-turn-stream-token";
|
|
103
|
-
|
|
10
|
+
function encodeTurnStreamClaims(claims) {
|
|
11
|
+
const { identity, purpose } = claims;
|
|
12
|
+
if (!["turn-control", "turn-cancel", "turn-recovery"].includes(purpose) || !identity || !["thread", "workspace"].includes(identity.scope) || [identity.workspaceId, identity.threadId, identity.executionId, identity.lockId, identity.turnStreamId].some((value) => typeof value !== "string" || !value.trim())) throw new Error("Invalid turn-stream control claims");
|
|
13
|
+
return base64UrlEncodeText(JSON.stringify([
|
|
14
|
+
purpose,
|
|
15
|
+
identity.workspaceId,
|
|
16
|
+
identity.threadId,
|
|
17
|
+
identity.scope,
|
|
18
|
+
identity.executionId,
|
|
19
|
+
identity.lockId,
|
|
20
|
+
identity.turnStreamId
|
|
21
|
+
]));
|
|
22
|
+
}
|
|
23
|
+
async function mintTurnStreamToken(channelName, secret, now = Date.now, ttlMs = TURN_STREAM_TOKEN_TTL_MS, claims) {
|
|
104
24
|
if (!secret || secret.length < 32) throw new Error("turn-stream token secret must be at least 32 characters");
|
|
105
25
|
const expiresAt = now() + ttlMs;
|
|
106
|
-
const payload =
|
|
26
|
+
const payload = `${claims ? "v2" : "v1"}.${base64UrlEncodeText(channelName)}.${expiresAt.toString(36)}${claims ? `.${encodeTurnStreamClaims(claims)}` : ""}`;
|
|
107
27
|
return `${payload}.${await hmacSha256Base64Url(payload, secret)}`;
|
|
108
28
|
}
|
|
109
|
-
async function verifyTurnStreamToken(channelName, token, secret, now = Date.now) {
|
|
29
|
+
async function verifyTurnStreamToken(channelName, token, secret, now = Date.now, claims) {
|
|
110
30
|
if (!secret || secret.length < 32) return false;
|
|
111
31
|
const parts = token.split(".");
|
|
112
|
-
|
|
113
|
-
|
|
32
|
+
const version = parts[0];
|
|
33
|
+
if (!(version === "v1" && parts.length === 4 || version === "v2" && parts.length === 5)) return false;
|
|
34
|
+
const name = parts[1];
|
|
35
|
+
const expires36 = parts[2];
|
|
36
|
+
const mac = parts.at(-1);
|
|
37
|
+
if (claims) {
|
|
38
|
+
try {
|
|
39
|
+
if (version !== "v2" || parts[3] !== encodeTurnStreamClaims(claims)) return false;
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
114
44
|
if (name !== base64UrlEncodeText(channelName)) return false;
|
|
115
45
|
if (!/^[0-9a-z]+$/.test(expires36)) return false;
|
|
116
46
|
const expiresAt = parseInt(expires36, 36);
|
|
117
47
|
if (!Number.isSafeInteger(expiresAt) || expiresAt <= now()) return false;
|
|
118
|
-
const expected = await hmacSha256Base64Url(
|
|
48
|
+
const expected = await hmacSha256Base64Url(parts.slice(0, -1).join("."), secret);
|
|
119
49
|
return constantTimeEqual(mac, expected);
|
|
120
50
|
}
|
|
121
51
|
function threadChannelKey(workspaceId, threadId) {
|
|
@@ -163,7 +93,8 @@ function createTurnLock(input, now, ttlMs = TURN_LOCK_TTL_MS) {
|
|
|
163
93
|
lockId: input.lockId,
|
|
164
94
|
startedAt: now,
|
|
165
95
|
expiresAt: now + ttlMs,
|
|
166
|
-
...input.turnId ? { turnId: input.turnId } : {}
|
|
96
|
+
...input.turnId ? { turnId: input.turnId } : {},
|
|
97
|
+
...input.cancellation ? { cancellation: { turnStreamId: input.cancellation.turnStreamId } } : {}
|
|
167
98
|
};
|
|
168
99
|
}
|
|
169
100
|
function turnLockMatchesRelease(active, input) {
|
|
@@ -207,23 +138,261 @@ function turnEventStorageKey(seq) {
|
|
|
207
138
|
return `${TURN_STREAM_STORAGE_KEYS.turnEventPrefix}${String(seq).padStart(10, "0")}`;
|
|
208
139
|
}
|
|
209
140
|
|
|
141
|
+
// src/turn-stream/cancellation.ts
|
|
142
|
+
var TurnCancellationRequestedError = class extends Error {
|
|
143
|
+
code = "turn_cancel_requested";
|
|
144
|
+
constructor() {
|
|
145
|
+
super("Cancellation was requested for this turn");
|
|
146
|
+
this.name = "TurnCancellationRequestedError";
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
var TurnControlUnavailableError = class extends Error {
|
|
150
|
+
code = "turn_control_unavailable";
|
|
151
|
+
constructor(message = "The durable turn control connection was lost") {
|
|
152
|
+
super(message);
|
|
153
|
+
this.name = "TurnControlUnavailableError";
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
async function cancellationRequest(namespace, identity, auth, upgrade) {
|
|
157
|
+
const key = turnLockChannelKey(identity.workspaceId, identity.threadId, identity.scope);
|
|
158
|
+
const url = new URL(`https://turn-stream.internal${upgrade ? "/" : TURN_STREAM_PATHS.lockReleaseInterrupted}`);
|
|
159
|
+
const headers = new Headers({ [TURN_STREAM_TOKEN_HEADER]: await mintTurnStreamToken(
|
|
160
|
+
key,
|
|
161
|
+
auth.secret,
|
|
162
|
+
void 0,
|
|
163
|
+
void 0,
|
|
164
|
+
{ purpose: upgrade ? "turn-control" : "turn-cancel", identity }
|
|
165
|
+
) });
|
|
166
|
+
if (upgrade) {
|
|
167
|
+
for (const [name, value] of Object.entries(identity)) url.searchParams.set(name, value);
|
|
168
|
+
url.searchParams.set("sessionId", identity.threadId);
|
|
169
|
+
url.searchParams.set("cancellation", "1");
|
|
170
|
+
headers.set("Upgrade", "websocket");
|
|
171
|
+
} else headers.set("Content-Type", "application/json");
|
|
172
|
+
return namespace.get(namespace.idFromName(key)).fetch(url.href, {
|
|
173
|
+
method: upgrade ? "GET" : "POST",
|
|
174
|
+
headers,
|
|
175
|
+
...upgrade ? {} : { body: JSON.stringify({ ...identity, cancel: true }) }
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
async function requestDurableTurnCancellation(namespace, identity, auth) {
|
|
179
|
+
const response = await cancellationRequest(namespace, identity, auth, false);
|
|
180
|
+
if (!response.ok) throw new Error(`Turn cancellation failed (HTTP ${response.status})`);
|
|
181
|
+
return response.json();
|
|
182
|
+
}
|
|
183
|
+
async function connectDurableTurnCancellation(namespace, identity, auth) {
|
|
184
|
+
const response = await cancellationRequest(namespace, identity, auth, true);
|
|
185
|
+
if (response.status !== 101 || !response.webSocket) throw new TurnControlUnavailableError(`Turn control connection failed (HTTP ${response.status})`);
|
|
186
|
+
const socket = response.webSocket;
|
|
187
|
+
const controller = new AbortController();
|
|
188
|
+
let closing = false;
|
|
189
|
+
let ready = false;
|
|
190
|
+
let resolveReady;
|
|
191
|
+
let rejectReady;
|
|
192
|
+
const connected = new Promise((resolve, reject) => {
|
|
193
|
+
resolveReady = resolve;
|
|
194
|
+
rejectReady = reject;
|
|
195
|
+
});
|
|
196
|
+
const fail = () => {
|
|
197
|
+
if (closing) return;
|
|
198
|
+
const error = new TurnControlUnavailableError();
|
|
199
|
+
if (!ready) rejectReady(error);
|
|
200
|
+
controller.abort(error);
|
|
201
|
+
};
|
|
202
|
+
const close = () => {
|
|
203
|
+
closing = true;
|
|
204
|
+
clearTimeout(timer);
|
|
205
|
+
try {
|
|
206
|
+
socket.close(1e3, "Owner observation ended");
|
|
207
|
+
} catch {
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
const timer = setTimeout(() => {
|
|
211
|
+
if (!ready) {
|
|
212
|
+
fail();
|
|
213
|
+
close();
|
|
214
|
+
}
|
|
215
|
+
}, 1e4);
|
|
216
|
+
socket.addEventListener("message", (event) => {
|
|
217
|
+
let message;
|
|
218
|
+
try {
|
|
219
|
+
message = JSON.parse(String(event.data));
|
|
220
|
+
} catch {
|
|
221
|
+
fail();
|
|
222
|
+
close();
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
if (message.type === "turn.control.released") {
|
|
226
|
+
fail();
|
|
227
|
+
close();
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
if (message.type === "turn.control.ownership-lost") {
|
|
231
|
+
fail();
|
|
232
|
+
close();
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
if (message.type !== "turn.control" || typeof message.cancellationRequested !== "boolean") {
|
|
236
|
+
fail();
|
|
237
|
+
close();
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
if (message.cancellationRequested) controller.abort(new TurnCancellationRequestedError());
|
|
241
|
+
ready = true;
|
|
242
|
+
clearTimeout(timer);
|
|
243
|
+
resolveReady();
|
|
244
|
+
});
|
|
245
|
+
socket.addEventListener("close", fail);
|
|
246
|
+
socket.addEventListener("error", fail);
|
|
247
|
+
try {
|
|
248
|
+
socket.accept();
|
|
249
|
+
socket.send(JSON.stringify({ type: "sync" }));
|
|
250
|
+
await connected;
|
|
251
|
+
} catch (error) {
|
|
252
|
+
close();
|
|
253
|
+
throw error;
|
|
254
|
+
}
|
|
255
|
+
return { signal: controller.signal, close };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// src/chat-routes/stale-turn-lock.ts
|
|
259
|
+
var DEFAULT_STALE_TURN_LOCK_GRACE_MS = 5 * 60 * 1e3;
|
|
260
|
+
var DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS = 60 * 1e3;
|
|
261
|
+
function messageOf(err) {
|
|
262
|
+
return err instanceof Error ? err.message : String(err);
|
|
263
|
+
}
|
|
264
|
+
async function reconcileStaleTurnLock(options) {
|
|
265
|
+
if (options.owner) return reconcileTurnOwner(options);
|
|
266
|
+
let sandbox;
|
|
267
|
+
try {
|
|
268
|
+
sandbox = await options.probeSandbox();
|
|
269
|
+
} catch (err) {
|
|
270
|
+
return forceReleaseUnreachable(options, {
|
|
271
|
+
unreachableReason: "SANDBOX_PROBE_FAILED",
|
|
272
|
+
unreachableDetail: messageOf(err)
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
if (sandbox.status !== "running") {
|
|
276
|
+
return forceReleaseUnreachable(options, {
|
|
277
|
+
unreachableReason: sandbox.status === "absent" ? "SANDBOX_ABSENT" : "SANDBOX_NOT_RUNNING",
|
|
278
|
+
...sandbox.status === "not-running" && sandbox.state !== void 0 ? { sandboxState: sandbox.state } : {}
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
const observedAt = (options.now ?? Date.now)();
|
|
282
|
+
let session;
|
|
283
|
+
try {
|
|
284
|
+
session = await options.probeSession();
|
|
285
|
+
} catch (err) {
|
|
286
|
+
return forceReleaseUnreachable(options, {
|
|
287
|
+
unreachableReason: "SESSION_PROBE_FAILED",
|
|
288
|
+
unreachableDetail: messageOf(err)
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
if (!session.reachable) {
|
|
292
|
+
return forceReleaseUnreachable(options, {
|
|
293
|
+
unreachableReason: "SESSION_UNREACHABLE",
|
|
294
|
+
...session.reason !== void 0 ? { sessionProbeError: session.reason } : {}
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
const diagnostics = {
|
|
298
|
+
sandboxReachable: true,
|
|
299
|
+
sessionTerminal: session.terminal,
|
|
300
|
+
...session.diagnostics ?? {}
|
|
301
|
+
};
|
|
302
|
+
if (!session.terminal) return { released: false, diagnostics };
|
|
303
|
+
const terminalGraceMs = options.terminalGraceMs ?? DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS;
|
|
304
|
+
const lockAgeMs = observedAt - options.lockStartedAt;
|
|
305
|
+
if (lockAgeMs < terminalGraceMs) {
|
|
306
|
+
const log = options.log ?? ((message, meta) => console.warn(message, meta));
|
|
307
|
+
const withheld = {
|
|
308
|
+
...diagnostics,
|
|
309
|
+
lockAgeMs,
|
|
310
|
+
terminalReleaseGraceMs: terminalGraceMs,
|
|
311
|
+
terminalReleaseWithheld: "LOCK_WITHIN_TERMINAL_GRACE_PERIOD"
|
|
312
|
+
};
|
|
313
|
+
log("[chat-routes] stale turn lock held: terminal verdict but lock is younger than the registration window", {
|
|
314
|
+
...options.context ?? {},
|
|
315
|
+
...withheld
|
|
316
|
+
});
|
|
317
|
+
return { released: false, diagnostics: withheld };
|
|
318
|
+
}
|
|
319
|
+
const released = await options.release({ observedAt });
|
|
320
|
+
return {
|
|
321
|
+
released,
|
|
322
|
+
diagnostics: { ...diagnostics, lockAgeMs, terminalReleaseGraceMs: terminalGraceMs, released, observedAt }
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
async function reconcileTurnOwner(options) {
|
|
326
|
+
const owner = options.owner;
|
|
327
|
+
const observedAt = (options.now ?? Date.now)();
|
|
328
|
+
let evidence;
|
|
329
|
+
try {
|
|
330
|
+
evidence = await owner.probe(owner.identity);
|
|
331
|
+
} catch (error) {
|
|
332
|
+
return { released: false, diagnostics: { ownerStatus: "unknown", ownerProbeError: messageOf(error) } };
|
|
333
|
+
}
|
|
334
|
+
const keys = ["workspaceId", "threadId", "scope", "executionId", "lockId", "turnStreamId"];
|
|
335
|
+
if (!evidence?.identity || keys.some((key) => evidence.identity[key] !== owner.identity[key])) {
|
|
336
|
+
return { released: false, diagnostics: { ownerStatus: "unknown", releaseWithheld: "OWNER_IDENTITY_MISMATCH" } };
|
|
337
|
+
}
|
|
338
|
+
const status = evidence.status === "abandoned" || evidence.status === "live" ? evidence.status : "unknown";
|
|
339
|
+
const diagnostics = { ...evidence.diagnostics ?? {}, ownerStatus: status };
|
|
340
|
+
if (status !== "abandoned") return { released: false, diagnostics };
|
|
341
|
+
const terminalGraceMs = options.terminalGraceMs ?? DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS;
|
|
342
|
+
const lockAgeMs = observedAt - options.lockStartedAt;
|
|
343
|
+
if (lockAgeMs < terminalGraceMs) return { released: false, diagnostics: {
|
|
344
|
+
...diagnostics,
|
|
345
|
+
lockAgeMs,
|
|
346
|
+
terminalReleaseGraceMs: terminalGraceMs,
|
|
347
|
+
releaseWithheld: "LOCK_WITHIN_TERMINAL_GRACE_PERIOD"
|
|
348
|
+
} };
|
|
349
|
+
const released = await options.release({ observedAt });
|
|
350
|
+
return { released, diagnostics: { ...diagnostics, lockAgeMs, terminalReleaseGraceMs: terminalGraceMs, observedAt, released } };
|
|
351
|
+
}
|
|
352
|
+
async function forceReleaseUnreachable(options, reason) {
|
|
353
|
+
const log = options.log ?? ((message, meta) => console.warn(message, meta));
|
|
354
|
+
const graceMs = options.graceMs ?? DEFAULT_STALE_TURN_LOCK_GRACE_MS;
|
|
355
|
+
const at = (options.now ?? Date.now)();
|
|
356
|
+
const lockAgeMs = at - options.lockStartedAt;
|
|
357
|
+
const diagnostics = {
|
|
358
|
+
...reason,
|
|
359
|
+
sandboxReachable: false,
|
|
360
|
+
lockAgeMs,
|
|
361
|
+
forceReleaseGraceMs: graceMs
|
|
362
|
+
};
|
|
363
|
+
if (lockAgeMs < graceMs) {
|
|
364
|
+
log("[chat-routes] stale turn lock held: sandbox unreachable but lock is inside the grace period", {
|
|
365
|
+
...options.context ?? {},
|
|
366
|
+
...diagnostics
|
|
367
|
+
});
|
|
368
|
+
return { released: false, diagnostics: { ...diagnostics, forceReleaseWithheld: "LOCK_WITHIN_GRACE_PERIOD" } };
|
|
369
|
+
}
|
|
370
|
+
const released = await options.release({ observedAt: at });
|
|
371
|
+
log("[chat-routes] force-released stale turn lock: sandbox unreachable, no turn can be executing", {
|
|
372
|
+
...options.context ?? {},
|
|
373
|
+
...diagnostics,
|
|
374
|
+
released
|
|
375
|
+
});
|
|
376
|
+
return { released, diagnostics: { ...diagnostics, forceReleased: released } };
|
|
377
|
+
}
|
|
378
|
+
|
|
210
379
|
// src/turn-stream/adapters.ts
|
|
211
380
|
var DEFAULT_TURN_LOCK_LOCATION_HINT = "enam";
|
|
212
381
|
var INTERNAL_ORIGIN = "https://turn-stream.internal";
|
|
213
|
-
async function postJson(namespace, channelKey, path, body, auth) {
|
|
382
|
+
async function postJson(namespace, channelKey, path, body, auth, claims) {
|
|
214
383
|
const stub = namespace.get(namespace.idFromName(channelKey));
|
|
215
384
|
const response = await stub.fetch(`${INTERNAL_ORIGIN}${path}`, {
|
|
216
385
|
method: "POST",
|
|
217
386
|
headers: {
|
|
218
387
|
"Content-Type": "application/json",
|
|
219
|
-
[TURN_STREAM_TOKEN_HEADER]: await mintTurnStreamToken(channelKey, auth.secret)
|
|
388
|
+
[TURN_STREAM_TOKEN_HEADER]: await mintTurnStreamToken(channelKey, auth.secret, void 0, void 0, claims)
|
|
220
389
|
},
|
|
221
390
|
body: JSON.stringify(body)
|
|
222
391
|
});
|
|
223
392
|
return { status: response.status, body: await response.json() };
|
|
224
393
|
}
|
|
225
|
-
async function postJsonOk(namespace, channelKey, path, body, auth) {
|
|
226
|
-
const result = await postJson(namespace, channelKey, path, body, auth);
|
|
394
|
+
async function postJsonOk(namespace, channelKey, path, body, auth, claims) {
|
|
395
|
+
const result = await postJson(namespace, channelKey, path, body, auth, claims);
|
|
227
396
|
if (result.status !== 200) {
|
|
228
397
|
throw new Error(`turn-stream ${path} failed with status ${result.status}`);
|
|
229
398
|
}
|
|
@@ -312,23 +481,37 @@ async function releaseInterruptedDurableTurnLock(namespace, input, auth) {
|
|
|
312
481
|
scope,
|
|
313
482
|
threadId: input.threadId,
|
|
314
483
|
interruptedAt: input.interruptedAt,
|
|
315
|
-
...input.turnId ? { turnId: input.turnId } : {}
|
|
316
|
-
|
|
484
|
+
...input.turnId ? { turnId: input.turnId } : {},
|
|
485
|
+
...input.recovery ? { recovery: input.recovery } : {}
|
|
486
|
+
}, auth, input.recovery ? { purpose: "turn-recovery", identity: input.recovery } : void 0);
|
|
317
487
|
if (result.released) return true;
|
|
318
488
|
}
|
|
319
489
|
return false;
|
|
320
490
|
}
|
|
321
491
|
async function reconcileStaleDurableTurnLock(options) {
|
|
322
|
-
const { namespace, auth, workspaceId, threadId, active, ...policy } = options;
|
|
492
|
+
const { namespace, auth, workspaceId, threadId, active, probeOwner, ...policy } = options;
|
|
493
|
+
const owner = active.cancellation ? {
|
|
494
|
+
workspaceId: active.workspaceId,
|
|
495
|
+
threadId: active.threadId,
|
|
496
|
+
scope: active.scope,
|
|
497
|
+
executionId: active.executionId,
|
|
498
|
+
lockId: active.lockId,
|
|
499
|
+
turnStreamId: active.cancellation.turnStreamId
|
|
500
|
+
} : void 0;
|
|
323
501
|
return reconcileStaleTurnLock({
|
|
324
502
|
...policy,
|
|
325
503
|
lockStartedAt: active.startedAt,
|
|
504
|
+
...owner ? { owner: {
|
|
505
|
+
identity: owner,
|
|
506
|
+
probe: probeOwner ?? (async (identity) => ({ identity, status: "unknown" }))
|
|
507
|
+
} } : {},
|
|
326
508
|
release: (fence) => releaseInterruptedDurableTurnLock(namespace, {
|
|
327
509
|
workspaceId,
|
|
328
510
|
threadId: active.threadId,
|
|
329
511
|
scope: active.scope,
|
|
330
512
|
interruptedAt: fence.observedAt,
|
|
331
|
-
...active.turnId ? { turnId: active.turnId } : {}
|
|
513
|
+
...active.turnId ? { turnId: active.turnId } : {},
|
|
514
|
+
...owner ? { recovery: owner } : {}
|
|
332
515
|
}, auth)
|
|
333
516
|
});
|
|
334
517
|
}
|
|
@@ -370,12 +553,15 @@ function createDurableTurnLock(options) {
|
|
|
370
553
|
const scope = options.scopeOf(args);
|
|
371
554
|
const executionId = options.lockExecutionIdOf?.(args) ?? args.executionId;
|
|
372
555
|
const turnId = options.clientTurnIdOf ? options.clientTurnIdOf(args) : args.body.turnId;
|
|
556
|
+
const cancellable = options.cancellationOf?.(args) === true;
|
|
557
|
+
if (cancellable && !args.turnStreamId) throw new Error("Durable cancellation requires the route turnStreamId");
|
|
373
558
|
const input = {
|
|
374
559
|
workspaceId,
|
|
375
560
|
threadId,
|
|
376
561
|
scope,
|
|
377
562
|
executionId,
|
|
378
|
-
...turnId ? { turnId } : {}
|
|
563
|
+
...turnId ? { turnId } : {},
|
|
564
|
+
...cancellable ? { cancellation: { turnStreamId: args.turnStreamId } } : {}
|
|
379
565
|
};
|
|
380
566
|
let acquired = await acquireDurableTurnLock(namespace, input, options.auth);
|
|
381
567
|
let diagnostics;
|
|
@@ -399,7 +585,18 @@ function createDurableTurnLock(options) {
|
|
|
399
585
|
executionId,
|
|
400
586
|
lockId: acquired.lock.lockId
|
|
401
587
|
};
|
|
402
|
-
return { acquired: true, handle };
|
|
588
|
+
if (!cancellable) return { acquired: true, handle };
|
|
589
|
+
try {
|
|
590
|
+
const cancellation = await connectDurableTurnCancellation(
|
|
591
|
+
options.namespace,
|
|
592
|
+
{ ...handle, turnStreamId: args.turnStreamId },
|
|
593
|
+
options.auth
|
|
594
|
+
);
|
|
595
|
+
return { acquired: true, handle, cancellation };
|
|
596
|
+
} catch (error) {
|
|
597
|
+
await releaseDurableTurnLock(options.namespace, handle, options.auth);
|
|
598
|
+
throw error;
|
|
599
|
+
}
|
|
403
600
|
},
|
|
404
601
|
async release(handle) {
|
|
405
602
|
if (!handle) return;
|
|
@@ -455,9 +652,6 @@ function createTurnStreamUpgradeHandler(options) {
|
|
|
455
652
|
}
|
|
456
653
|
|
|
457
654
|
export {
|
|
458
|
-
DEFAULT_STALE_TURN_LOCK_GRACE_MS,
|
|
459
|
-
DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS,
|
|
460
|
-
reconcileStaleTurnLock,
|
|
461
655
|
TURN_STREAM_TOKEN_TTL_MS,
|
|
462
656
|
TURN_STREAM_TOKEN_HEADER,
|
|
463
657
|
mintTurnStreamToken,
|
|
@@ -479,6 +673,13 @@ export {
|
|
|
479
673
|
TURN_STREAM_PATHS,
|
|
480
674
|
TURN_STREAM_STORAGE_KEYS,
|
|
481
675
|
turnEventStorageKey,
|
|
676
|
+
TurnCancellationRequestedError,
|
|
677
|
+
TurnControlUnavailableError,
|
|
678
|
+
requestDurableTurnCancellation,
|
|
679
|
+
connectDurableTurnCancellation,
|
|
680
|
+
DEFAULT_STALE_TURN_LOCK_GRACE_MS,
|
|
681
|
+
DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS,
|
|
682
|
+
reconcileStaleTurnLock,
|
|
482
683
|
DEFAULT_TURN_LOCK_LOCATION_HINT,
|
|
483
684
|
createDurableObjectTurnEventStore,
|
|
484
685
|
acquireDurableTurnLock,
|
|
@@ -491,4 +692,4 @@ export {
|
|
|
491
692
|
broadcastThreadCreated,
|
|
492
693
|
createTurnStreamUpgradeHandler
|
|
493
694
|
};
|
|
494
|
-
//# sourceMappingURL=chunk-
|
|
695
|
+
//# sourceMappingURL=chunk-MT6ZTQGD.js.map
|