@tangle-network/agent-app 0.63.2 → 0.63.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-profiles/d1.js +3 -3
- package/dist/agent-profiles/index.js +3 -3
- package/dist/chat-routes/cancellation.d.ts +12 -1
- package/dist/chat-routes/index.d.ts +1 -0
- package/dist/chat-routes/index.js +15 -15
- package/dist/chat-routes/route-storage.d.ts +21 -1
- package/dist/chat-routes/stale-turn-lock.d.ts +17 -1
- package/dist/chat-routes/turn-ownership.d.ts +1 -0
- package/dist/chat-routes/turn-route-contracts.d.ts +15 -12
- package/dist/{chunk-ND5XWLRU.js → chunk-72ASTUPN.js} +83 -24
- package/dist/chunk-72ASTUPN.js.map +1 -0
- package/dist/{chunk-XEAZWJ3W.js → chunk-7Y7U7U7X.js} +2 -2
- package/dist/{chunk-5BJHKQ33.js → chunk-ACB7KOQ6.js} +4 -4
- package/dist/{chunk-7TZA2TAM.js → chunk-FCYHDENJ.js} +2 -2
- package/dist/{chunk-WSQFNMIA.js → chunk-HRBF4B5L.js} +5 -5
- package/dist/{chunk-WSQFNMIA.js.map → chunk-HRBF4B5L.js.map} +1 -1
- package/dist/{chunk-SRWQCWLR.js → chunk-IX6UVHXY.js} +3 -3
- package/dist/{chunk-A2NQMVMF.js → chunk-MT6ZTQGD.js} +328 -118
- package/dist/chunk-MT6ZTQGD.js.map +1 -0
- package/dist/{chunk-HG2WVONR.js → chunk-UXK5EBQT.js} +42 -26
- package/dist/chunk-UXK5EBQT.js.map +1 -0
- package/dist/preset-cloudflare/index.js +8 -8
- package/dist/public-consultation/index.d.ts +1 -1
- package/dist/public-consultation/index.js +8 -8
- package/dist/sandbox/index.js +7 -5
- package/dist/sandbox/prewarm-route.d.ts +7 -0
- package/dist/stream/index.js +17 -17
- package/dist/tools/index.js +4 -4
- package/dist/turn-stream/adapters.d.ts +19 -11
- package/dist/turn-stream/cancellation-do.d.ts +31 -0
- package/dist/turn-stream/cancellation.d.ts +23 -0
- package/dist/turn-stream/core.d.ts +23 -4
- package/dist/turn-stream/do.d.ts +4 -2
- package/dist/turn-stream/index.d.ts +1 -0
- package/dist/turn-stream/index.js +188 -14
- package/dist/turn-stream/index.js.map +1 -1
- package/dist/work-product/index.js +2 -2
- package/package.json +3 -2
- package/dist/chunk-A2NQMVMF.js.map +0 -1
- package/dist/chunk-HG2WVONR.js.map +0 -1
- package/dist/chunk-ND5XWLRU.js.map +0 -1
- /package/dist/{chunk-XEAZWJ3W.js.map → chunk-7Y7U7U7X.js.map} +0 -0
- /package/dist/{chunk-5BJHKQ33.js.map → chunk-ACB7KOQ6.js.map} +0 -0
- /package/dist/{chunk-7TZA2TAM.js.map → chunk-FCYHDENJ.js.map} +0 -0
- /package/dist/{chunk-SRWQCWLR.js.map → chunk-IX6UVHXY.js.map} +0 -0
|
@@ -4,10 +4,10 @@ import {
|
|
|
4
4
|
} from "./chunk-I6BZVIRH.js";
|
|
5
5
|
import {
|
|
6
6
|
defineAppTool
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-FCYHDENJ.js";
|
|
8
8
|
import {
|
|
9
9
|
ToolInputError
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-HRBF4B5L.js";
|
|
11
11
|
|
|
12
12
|
// src/agent-profiles/index.ts
|
|
13
13
|
import {
|
|
@@ -672,4 +672,4 @@ export {
|
|
|
672
672
|
PROFILE_MEMORY_PREFIX,
|
|
673
673
|
profileTools
|
|
674
674
|
};
|
|
675
|
-
//# sourceMappingURL=chunk-
|
|
675
|
+
//# sourceMappingURL=chunk-IX6UVHXY.js.map
|
|
@@ -4,118 +4,48 @@ import {
|
|
|
4
4
|
hmacSha256Base64Url
|
|
5
5
|
} from "./chunk-S5SRJJQG.js";
|
|
6
6
|
|
|
7
|
-
// src/chat-routes/stale-turn-lock.ts
|
|
8
|
-
var DEFAULT_STALE_TURN_LOCK_GRACE_MS = 5 * 60 * 1e3;
|
|
9
|
-
var DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS = 60 * 1e3;
|
|
10
|
-
function messageOf(err) {
|
|
11
|
-
return err instanceof Error ? err.message : String(err);
|
|
12
|
-
}
|
|
13
|
-
async function reconcileStaleTurnLock(options) {
|
|
14
|
-
let sandbox;
|
|
15
|
-
try {
|
|
16
|
-
sandbox = await options.probeSandbox();
|
|
17
|
-
} catch (err) {
|
|
18
|
-
return forceReleaseUnreachable(options, {
|
|
19
|
-
unreachableReason: "SANDBOX_PROBE_FAILED",
|
|
20
|
-
unreachableDetail: messageOf(err)
|
|
21
|
-
});
|
|
22
|
-
}
|
|
23
|
-
if (sandbox.status !== "running") {
|
|
24
|
-
return forceReleaseUnreachable(options, {
|
|
25
|
-
unreachableReason: sandbox.status === "absent" ? "SANDBOX_ABSENT" : "SANDBOX_NOT_RUNNING",
|
|
26
|
-
...sandbox.status === "not-running" && sandbox.state !== void 0 ? { sandboxState: sandbox.state } : {}
|
|
27
|
-
});
|
|
28
|
-
}
|
|
29
|
-
const observedAt = (options.now ?? Date.now)();
|
|
30
|
-
let session;
|
|
31
|
-
try {
|
|
32
|
-
session = await options.probeSession();
|
|
33
|
-
} catch (err) {
|
|
34
|
-
return forceReleaseUnreachable(options, {
|
|
35
|
-
unreachableReason: "SESSION_PROBE_FAILED",
|
|
36
|
-
unreachableDetail: messageOf(err)
|
|
37
|
-
});
|
|
38
|
-
}
|
|
39
|
-
if (!session.reachable) {
|
|
40
|
-
return forceReleaseUnreachable(options, {
|
|
41
|
-
unreachableReason: "SESSION_UNREACHABLE",
|
|
42
|
-
...session.reason !== void 0 ? { sessionProbeError: session.reason } : {}
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
const diagnostics = {
|
|
46
|
-
sandboxReachable: true,
|
|
47
|
-
sessionTerminal: session.terminal,
|
|
48
|
-
...session.diagnostics ?? {}
|
|
49
|
-
};
|
|
50
|
-
if (!session.terminal) return { released: false, diagnostics };
|
|
51
|
-
const terminalGraceMs = options.terminalGraceMs ?? DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS;
|
|
52
|
-
const lockAgeMs = observedAt - options.lockStartedAt;
|
|
53
|
-
if (lockAgeMs < terminalGraceMs) {
|
|
54
|
-
const log = options.log ?? ((message, meta) => console.warn(message, meta));
|
|
55
|
-
const withheld = {
|
|
56
|
-
...diagnostics,
|
|
57
|
-
lockAgeMs,
|
|
58
|
-
terminalReleaseGraceMs: terminalGraceMs,
|
|
59
|
-
terminalReleaseWithheld: "LOCK_WITHIN_TERMINAL_GRACE_PERIOD"
|
|
60
|
-
};
|
|
61
|
-
log("[chat-routes] stale turn lock held: terminal verdict but lock is younger than the registration window", {
|
|
62
|
-
...options.context ?? {},
|
|
63
|
-
...withheld
|
|
64
|
-
});
|
|
65
|
-
return { released: false, diagnostics: withheld };
|
|
66
|
-
}
|
|
67
|
-
const released = await options.release({ observedAt });
|
|
68
|
-
return {
|
|
69
|
-
released,
|
|
70
|
-
diagnostics: { ...diagnostics, lockAgeMs, terminalReleaseGraceMs: terminalGraceMs, released, observedAt }
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
async function forceReleaseUnreachable(options, reason) {
|
|
74
|
-
const log = options.log ?? ((message, meta) => console.warn(message, meta));
|
|
75
|
-
const graceMs = options.graceMs ?? DEFAULT_STALE_TURN_LOCK_GRACE_MS;
|
|
76
|
-
const at = (options.now ?? Date.now)();
|
|
77
|
-
const lockAgeMs = at - options.lockStartedAt;
|
|
78
|
-
const diagnostics = {
|
|
79
|
-
...reason,
|
|
80
|
-
sandboxReachable: false,
|
|
81
|
-
lockAgeMs,
|
|
82
|
-
forceReleaseGraceMs: graceMs
|
|
83
|
-
};
|
|
84
|
-
if (lockAgeMs < graceMs) {
|
|
85
|
-
log("[chat-routes] stale turn lock held: sandbox unreachable but lock is inside the grace period", {
|
|
86
|
-
...options.context ?? {},
|
|
87
|
-
...diagnostics
|
|
88
|
-
});
|
|
89
|
-
return { released: false, diagnostics: { ...diagnostics, forceReleaseWithheld: "LOCK_WITHIN_GRACE_PERIOD" } };
|
|
90
|
-
}
|
|
91
|
-
const released = await options.release({ observedAt: at });
|
|
92
|
-
log("[chat-routes] force-released stale turn lock: sandbox unreachable, no turn can be executing", {
|
|
93
|
-
...options.context ?? {},
|
|
94
|
-
...diagnostics,
|
|
95
|
-
released
|
|
96
|
-
});
|
|
97
|
-
return { released, diagnostics: { ...diagnostics, forceReleased: released } };
|
|
98
|
-
}
|
|
99
|
-
|
|
100
7
|
// src/turn-stream/core.ts
|
|
101
8
|
var TURN_STREAM_TOKEN_TTL_MS = 6e4;
|
|
102
9
|
var TURN_STREAM_TOKEN_HEADER = "x-turn-stream-token";
|
|
103
|
-
|
|
10
|
+
function encodeTurnStreamClaims(claims) {
|
|
11
|
+
const { identity, purpose } = claims;
|
|
12
|
+
if (!["turn-control", "turn-cancel", "turn-recovery"].includes(purpose) || !identity || !["thread", "workspace"].includes(identity.scope) || [identity.workspaceId, identity.threadId, identity.executionId, identity.lockId, identity.turnStreamId].some((value) => typeof value !== "string" || !value.trim())) throw new Error("Invalid turn-stream control claims");
|
|
13
|
+
return base64UrlEncodeText(JSON.stringify([
|
|
14
|
+
purpose,
|
|
15
|
+
identity.workspaceId,
|
|
16
|
+
identity.threadId,
|
|
17
|
+
identity.scope,
|
|
18
|
+
identity.executionId,
|
|
19
|
+
identity.lockId,
|
|
20
|
+
identity.turnStreamId
|
|
21
|
+
]));
|
|
22
|
+
}
|
|
23
|
+
async function mintTurnStreamToken(channelName, secret, now = Date.now, ttlMs = TURN_STREAM_TOKEN_TTL_MS, claims) {
|
|
104
24
|
if (!secret || secret.length < 32) throw new Error("turn-stream token secret must be at least 32 characters");
|
|
105
25
|
const expiresAt = now() + ttlMs;
|
|
106
|
-
const payload =
|
|
26
|
+
const payload = `${claims ? "v2" : "v1"}.${base64UrlEncodeText(channelName)}.${expiresAt.toString(36)}${claims ? `.${encodeTurnStreamClaims(claims)}` : ""}`;
|
|
107
27
|
return `${payload}.${await hmacSha256Base64Url(payload, secret)}`;
|
|
108
28
|
}
|
|
109
|
-
async function verifyTurnStreamToken(channelName, token, secret, now = Date.now) {
|
|
29
|
+
async function verifyTurnStreamToken(channelName, token, secret, now = Date.now, claims) {
|
|
110
30
|
if (!secret || secret.length < 32) return false;
|
|
111
31
|
const parts = token.split(".");
|
|
112
|
-
|
|
113
|
-
|
|
32
|
+
const version = parts[0];
|
|
33
|
+
if (!(version === "v1" && parts.length === 4 || version === "v2" && parts.length === 5)) return false;
|
|
34
|
+
const name = parts[1];
|
|
35
|
+
const expires36 = parts[2];
|
|
36
|
+
const mac = parts.at(-1);
|
|
37
|
+
if (claims) {
|
|
38
|
+
try {
|
|
39
|
+
if (version !== "v2" || parts[3] !== encodeTurnStreamClaims(claims)) return false;
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
114
44
|
if (name !== base64UrlEncodeText(channelName)) return false;
|
|
115
45
|
if (!/^[0-9a-z]+$/.test(expires36)) return false;
|
|
116
46
|
const expiresAt = parseInt(expires36, 36);
|
|
117
47
|
if (!Number.isSafeInteger(expiresAt) || expiresAt <= now()) return false;
|
|
118
|
-
const expected = await hmacSha256Base64Url(
|
|
48
|
+
const expected = await hmacSha256Base64Url(parts.slice(0, -1).join("."), secret);
|
|
119
49
|
return constantTimeEqual(mac, expected);
|
|
120
50
|
}
|
|
121
51
|
function threadChannelKey(workspaceId, threadId) {
|
|
@@ -163,7 +93,8 @@ function createTurnLock(input, now, ttlMs = TURN_LOCK_TTL_MS) {
|
|
|
163
93
|
lockId: input.lockId,
|
|
164
94
|
startedAt: now,
|
|
165
95
|
expiresAt: now + ttlMs,
|
|
166
|
-
...input.turnId ? { turnId: input.turnId } : {}
|
|
96
|
+
...input.turnId ? { turnId: input.turnId } : {},
|
|
97
|
+
...input.cancellation ? { cancellation: { turnStreamId: input.cancellation.turnStreamId } } : {}
|
|
167
98
|
};
|
|
168
99
|
}
|
|
169
100
|
function turnLockMatchesRelease(active, input) {
|
|
@@ -207,22 +138,261 @@ function turnEventStorageKey(seq) {
|
|
|
207
138
|
return `${TURN_STREAM_STORAGE_KEYS.turnEventPrefix}${String(seq).padStart(10, "0")}`;
|
|
208
139
|
}
|
|
209
140
|
|
|
141
|
+
// src/turn-stream/cancellation.ts
|
|
142
|
+
var TurnCancellationRequestedError = class extends Error {
|
|
143
|
+
code = "turn_cancel_requested";
|
|
144
|
+
constructor() {
|
|
145
|
+
super("Cancellation was requested for this turn");
|
|
146
|
+
this.name = "TurnCancellationRequestedError";
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
var TurnControlUnavailableError = class extends Error {
|
|
150
|
+
code = "turn_control_unavailable";
|
|
151
|
+
constructor(message = "The durable turn control connection was lost") {
|
|
152
|
+
super(message);
|
|
153
|
+
this.name = "TurnControlUnavailableError";
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
async function cancellationRequest(namespace, identity, auth, upgrade) {
|
|
157
|
+
const key = turnLockChannelKey(identity.workspaceId, identity.threadId, identity.scope);
|
|
158
|
+
const url = new URL(`https://turn-stream.internal${upgrade ? "/" : TURN_STREAM_PATHS.lockReleaseInterrupted}`);
|
|
159
|
+
const headers = new Headers({ [TURN_STREAM_TOKEN_HEADER]: await mintTurnStreamToken(
|
|
160
|
+
key,
|
|
161
|
+
auth.secret,
|
|
162
|
+
void 0,
|
|
163
|
+
void 0,
|
|
164
|
+
{ purpose: upgrade ? "turn-control" : "turn-cancel", identity }
|
|
165
|
+
) });
|
|
166
|
+
if (upgrade) {
|
|
167
|
+
for (const [name, value] of Object.entries(identity)) url.searchParams.set(name, value);
|
|
168
|
+
url.searchParams.set("sessionId", identity.threadId);
|
|
169
|
+
url.searchParams.set("cancellation", "1");
|
|
170
|
+
headers.set("Upgrade", "websocket");
|
|
171
|
+
} else headers.set("Content-Type", "application/json");
|
|
172
|
+
return namespace.get(namespace.idFromName(key)).fetch(url.href, {
|
|
173
|
+
method: upgrade ? "GET" : "POST",
|
|
174
|
+
headers,
|
|
175
|
+
...upgrade ? {} : { body: JSON.stringify({ ...identity, cancel: true }) }
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
async function requestDurableTurnCancellation(namespace, identity, auth) {
|
|
179
|
+
const response = await cancellationRequest(namespace, identity, auth, false);
|
|
180
|
+
if (!response.ok) throw new Error(`Turn cancellation failed (HTTP ${response.status})`);
|
|
181
|
+
return response.json();
|
|
182
|
+
}
|
|
183
|
+
async function connectDurableTurnCancellation(namespace, identity, auth) {
|
|
184
|
+
const response = await cancellationRequest(namespace, identity, auth, true);
|
|
185
|
+
if (response.status !== 101 || !response.webSocket) throw new TurnControlUnavailableError(`Turn control connection failed (HTTP ${response.status})`);
|
|
186
|
+
const socket = response.webSocket;
|
|
187
|
+
const controller = new AbortController();
|
|
188
|
+
let closing = false;
|
|
189
|
+
let ready = false;
|
|
190
|
+
let resolveReady;
|
|
191
|
+
let rejectReady;
|
|
192
|
+
const connected = new Promise((resolve, reject) => {
|
|
193
|
+
resolveReady = resolve;
|
|
194
|
+
rejectReady = reject;
|
|
195
|
+
});
|
|
196
|
+
const fail = () => {
|
|
197
|
+
if (closing) return;
|
|
198
|
+
const error = new TurnControlUnavailableError();
|
|
199
|
+
if (!ready) rejectReady(error);
|
|
200
|
+
controller.abort(error);
|
|
201
|
+
};
|
|
202
|
+
const close = () => {
|
|
203
|
+
closing = true;
|
|
204
|
+
clearTimeout(timer);
|
|
205
|
+
try {
|
|
206
|
+
socket.close(1e3, "Owner observation ended");
|
|
207
|
+
} catch {
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
const timer = setTimeout(() => {
|
|
211
|
+
if (!ready) {
|
|
212
|
+
fail();
|
|
213
|
+
close();
|
|
214
|
+
}
|
|
215
|
+
}, 1e4);
|
|
216
|
+
socket.addEventListener("message", (event) => {
|
|
217
|
+
let message;
|
|
218
|
+
try {
|
|
219
|
+
message = JSON.parse(String(event.data));
|
|
220
|
+
} catch {
|
|
221
|
+
fail();
|
|
222
|
+
close();
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
if (message.type === "turn.control.released") {
|
|
226
|
+
fail();
|
|
227
|
+
close();
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
if (message.type === "turn.control.ownership-lost") {
|
|
231
|
+
fail();
|
|
232
|
+
close();
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
if (message.type !== "turn.control" || typeof message.cancellationRequested !== "boolean") {
|
|
236
|
+
fail();
|
|
237
|
+
close();
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
if (message.cancellationRequested) controller.abort(new TurnCancellationRequestedError());
|
|
241
|
+
ready = true;
|
|
242
|
+
clearTimeout(timer);
|
|
243
|
+
resolveReady();
|
|
244
|
+
});
|
|
245
|
+
socket.addEventListener("close", fail);
|
|
246
|
+
socket.addEventListener("error", fail);
|
|
247
|
+
try {
|
|
248
|
+
socket.accept();
|
|
249
|
+
socket.send(JSON.stringify({ type: "sync" }));
|
|
250
|
+
await connected;
|
|
251
|
+
} catch (error) {
|
|
252
|
+
close();
|
|
253
|
+
throw error;
|
|
254
|
+
}
|
|
255
|
+
return { signal: controller.signal, close };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// src/chat-routes/stale-turn-lock.ts
|
|
259
|
+
var DEFAULT_STALE_TURN_LOCK_GRACE_MS = 5 * 60 * 1e3;
|
|
260
|
+
var DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS = 60 * 1e3;
|
|
261
|
+
function messageOf(err) {
|
|
262
|
+
return err instanceof Error ? err.message : String(err);
|
|
263
|
+
}
|
|
264
|
+
async function reconcileStaleTurnLock(options) {
|
|
265
|
+
if (options.owner) return reconcileTurnOwner(options);
|
|
266
|
+
let sandbox;
|
|
267
|
+
try {
|
|
268
|
+
sandbox = await options.probeSandbox();
|
|
269
|
+
} catch (err) {
|
|
270
|
+
return forceReleaseUnreachable(options, {
|
|
271
|
+
unreachableReason: "SANDBOX_PROBE_FAILED",
|
|
272
|
+
unreachableDetail: messageOf(err)
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
if (sandbox.status !== "running") {
|
|
276
|
+
return forceReleaseUnreachable(options, {
|
|
277
|
+
unreachableReason: sandbox.status === "absent" ? "SANDBOX_ABSENT" : "SANDBOX_NOT_RUNNING",
|
|
278
|
+
...sandbox.status === "not-running" && sandbox.state !== void 0 ? { sandboxState: sandbox.state } : {}
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
const observedAt = (options.now ?? Date.now)();
|
|
282
|
+
let session;
|
|
283
|
+
try {
|
|
284
|
+
session = await options.probeSession();
|
|
285
|
+
} catch (err) {
|
|
286
|
+
return forceReleaseUnreachable(options, {
|
|
287
|
+
unreachableReason: "SESSION_PROBE_FAILED",
|
|
288
|
+
unreachableDetail: messageOf(err)
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
if (!session.reachable) {
|
|
292
|
+
return forceReleaseUnreachable(options, {
|
|
293
|
+
unreachableReason: "SESSION_UNREACHABLE",
|
|
294
|
+
...session.reason !== void 0 ? { sessionProbeError: session.reason } : {}
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
const diagnostics = {
|
|
298
|
+
sandboxReachable: true,
|
|
299
|
+
sessionTerminal: session.terminal,
|
|
300
|
+
...session.diagnostics ?? {}
|
|
301
|
+
};
|
|
302
|
+
if (!session.terminal) return { released: false, diagnostics };
|
|
303
|
+
const terminalGraceMs = options.terminalGraceMs ?? DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS;
|
|
304
|
+
const lockAgeMs = observedAt - options.lockStartedAt;
|
|
305
|
+
if (lockAgeMs < terminalGraceMs) {
|
|
306
|
+
const log = options.log ?? ((message, meta) => console.warn(message, meta));
|
|
307
|
+
const withheld = {
|
|
308
|
+
...diagnostics,
|
|
309
|
+
lockAgeMs,
|
|
310
|
+
terminalReleaseGraceMs: terminalGraceMs,
|
|
311
|
+
terminalReleaseWithheld: "LOCK_WITHIN_TERMINAL_GRACE_PERIOD"
|
|
312
|
+
};
|
|
313
|
+
log("[chat-routes] stale turn lock held: terminal verdict but lock is younger than the registration window", {
|
|
314
|
+
...options.context ?? {},
|
|
315
|
+
...withheld
|
|
316
|
+
});
|
|
317
|
+
return { released: false, diagnostics: withheld };
|
|
318
|
+
}
|
|
319
|
+
const released = await options.release({ observedAt });
|
|
320
|
+
return {
|
|
321
|
+
released,
|
|
322
|
+
diagnostics: { ...diagnostics, lockAgeMs, terminalReleaseGraceMs: terminalGraceMs, released, observedAt }
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
async function reconcileTurnOwner(options) {
|
|
326
|
+
const owner = options.owner;
|
|
327
|
+
const observedAt = (options.now ?? Date.now)();
|
|
328
|
+
let evidence;
|
|
329
|
+
try {
|
|
330
|
+
evidence = await owner.probe(owner.identity);
|
|
331
|
+
} catch (error) {
|
|
332
|
+
return { released: false, diagnostics: { ownerStatus: "unknown", ownerProbeError: messageOf(error) } };
|
|
333
|
+
}
|
|
334
|
+
const keys = ["workspaceId", "threadId", "scope", "executionId", "lockId", "turnStreamId"];
|
|
335
|
+
if (!evidence?.identity || keys.some((key) => evidence.identity[key] !== owner.identity[key])) {
|
|
336
|
+
return { released: false, diagnostics: { ownerStatus: "unknown", releaseWithheld: "OWNER_IDENTITY_MISMATCH" } };
|
|
337
|
+
}
|
|
338
|
+
const status = evidence.status === "abandoned" || evidence.status === "live" ? evidence.status : "unknown";
|
|
339
|
+
const diagnostics = { ...evidence.diagnostics ?? {}, ownerStatus: status };
|
|
340
|
+
if (status !== "abandoned") return { released: false, diagnostics };
|
|
341
|
+
const terminalGraceMs = options.terminalGraceMs ?? DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS;
|
|
342
|
+
const lockAgeMs = observedAt - options.lockStartedAt;
|
|
343
|
+
if (lockAgeMs < terminalGraceMs) return { released: false, diagnostics: {
|
|
344
|
+
...diagnostics,
|
|
345
|
+
lockAgeMs,
|
|
346
|
+
terminalReleaseGraceMs: terminalGraceMs,
|
|
347
|
+
releaseWithheld: "LOCK_WITHIN_TERMINAL_GRACE_PERIOD"
|
|
348
|
+
} };
|
|
349
|
+
const released = await options.release({ observedAt });
|
|
350
|
+
return { released, diagnostics: { ...diagnostics, lockAgeMs, terminalReleaseGraceMs: terminalGraceMs, observedAt, released } };
|
|
351
|
+
}
|
|
352
|
+
async function forceReleaseUnreachable(options, reason) {
|
|
353
|
+
const log = options.log ?? ((message, meta) => console.warn(message, meta));
|
|
354
|
+
const graceMs = options.graceMs ?? DEFAULT_STALE_TURN_LOCK_GRACE_MS;
|
|
355
|
+
const at = (options.now ?? Date.now)();
|
|
356
|
+
const lockAgeMs = at - options.lockStartedAt;
|
|
357
|
+
const diagnostics = {
|
|
358
|
+
...reason,
|
|
359
|
+
sandboxReachable: false,
|
|
360
|
+
lockAgeMs,
|
|
361
|
+
forceReleaseGraceMs: graceMs
|
|
362
|
+
};
|
|
363
|
+
if (lockAgeMs < graceMs) {
|
|
364
|
+
log("[chat-routes] stale turn lock held: sandbox unreachable but lock is inside the grace period", {
|
|
365
|
+
...options.context ?? {},
|
|
366
|
+
...diagnostics
|
|
367
|
+
});
|
|
368
|
+
return { released: false, diagnostics: { ...diagnostics, forceReleaseWithheld: "LOCK_WITHIN_GRACE_PERIOD" } };
|
|
369
|
+
}
|
|
370
|
+
const released = await options.release({ observedAt: at });
|
|
371
|
+
log("[chat-routes] force-released stale turn lock: sandbox unreachable, no turn can be executing", {
|
|
372
|
+
...options.context ?? {},
|
|
373
|
+
...diagnostics,
|
|
374
|
+
released
|
|
375
|
+
});
|
|
376
|
+
return { released, diagnostics: { ...diagnostics, forceReleased: released } };
|
|
377
|
+
}
|
|
378
|
+
|
|
210
379
|
// src/turn-stream/adapters.ts
|
|
380
|
+
var DEFAULT_TURN_LOCK_LOCATION_HINT = "enam";
|
|
211
381
|
var INTERNAL_ORIGIN = "https://turn-stream.internal";
|
|
212
|
-
async function postJson(namespace, channelKey, path, body, auth) {
|
|
382
|
+
async function postJson(namespace, channelKey, path, body, auth, claims) {
|
|
213
383
|
const stub = namespace.get(namespace.idFromName(channelKey));
|
|
214
384
|
const response = await stub.fetch(`${INTERNAL_ORIGIN}${path}`, {
|
|
215
385
|
method: "POST",
|
|
216
386
|
headers: {
|
|
217
387
|
"Content-Type": "application/json",
|
|
218
|
-
[TURN_STREAM_TOKEN_HEADER]: await mintTurnStreamToken(channelKey, auth.secret)
|
|
388
|
+
[TURN_STREAM_TOKEN_HEADER]: await mintTurnStreamToken(channelKey, auth.secret, void 0, void 0, claims)
|
|
219
389
|
},
|
|
220
390
|
body: JSON.stringify(body)
|
|
221
391
|
});
|
|
222
392
|
return { status: response.status, body: await response.json() };
|
|
223
393
|
}
|
|
224
|
-
async function postJsonOk(namespace, channelKey, path, body, auth) {
|
|
225
|
-
const result = await postJson(namespace, channelKey, path, body, auth);
|
|
394
|
+
async function postJsonOk(namespace, channelKey, path, body, auth, claims) {
|
|
395
|
+
const result = await postJson(namespace, channelKey, path, body, auth, claims);
|
|
226
396
|
if (result.status !== 200) {
|
|
227
397
|
throw new Error(`turn-stream ${path} failed with status ${result.status}`);
|
|
228
398
|
}
|
|
@@ -311,23 +481,37 @@ async function releaseInterruptedDurableTurnLock(namespace, input, auth) {
|
|
|
311
481
|
scope,
|
|
312
482
|
threadId: input.threadId,
|
|
313
483
|
interruptedAt: input.interruptedAt,
|
|
314
|
-
...input.turnId ? { turnId: input.turnId } : {}
|
|
315
|
-
|
|
484
|
+
...input.turnId ? { turnId: input.turnId } : {},
|
|
485
|
+
...input.recovery ? { recovery: input.recovery } : {}
|
|
486
|
+
}, auth, input.recovery ? { purpose: "turn-recovery", identity: input.recovery } : void 0);
|
|
316
487
|
if (result.released) return true;
|
|
317
488
|
}
|
|
318
489
|
return false;
|
|
319
490
|
}
|
|
320
491
|
async function reconcileStaleDurableTurnLock(options) {
|
|
321
|
-
const { namespace, auth, workspaceId, threadId, active, ...policy } = options;
|
|
492
|
+
const { namespace, auth, workspaceId, threadId, active, probeOwner, ...policy } = options;
|
|
493
|
+
const owner = active.cancellation ? {
|
|
494
|
+
workspaceId: active.workspaceId,
|
|
495
|
+
threadId: active.threadId,
|
|
496
|
+
scope: active.scope,
|
|
497
|
+
executionId: active.executionId,
|
|
498
|
+
lockId: active.lockId,
|
|
499
|
+
turnStreamId: active.cancellation.turnStreamId
|
|
500
|
+
} : void 0;
|
|
322
501
|
return reconcileStaleTurnLock({
|
|
323
502
|
...policy,
|
|
324
503
|
lockStartedAt: active.startedAt,
|
|
504
|
+
...owner ? { owner: {
|
|
505
|
+
identity: owner,
|
|
506
|
+
probe: probeOwner ?? (async (identity) => ({ identity, status: "unknown" }))
|
|
507
|
+
} } : {},
|
|
325
508
|
release: (fence) => releaseInterruptedDurableTurnLock(namespace, {
|
|
326
509
|
workspaceId,
|
|
327
510
|
threadId: active.threadId,
|
|
328
511
|
scope: active.scope,
|
|
329
512
|
interruptedAt: fence.observedAt,
|
|
330
|
-
...active.turnId ? { turnId: active.turnId } : {}
|
|
513
|
+
...active.turnId ? { turnId: active.turnId } : {},
|
|
514
|
+
...owner ? { recovery: owner } : {}
|
|
331
515
|
}, auth)
|
|
332
516
|
});
|
|
333
517
|
}
|
|
@@ -350,11 +534,18 @@ function defaultRefusalResponse(active, diagnostics) {
|
|
|
350
534
|
);
|
|
351
535
|
}
|
|
352
536
|
function createDurableTurnLock(options) {
|
|
537
|
+
const namespace = {
|
|
538
|
+
idFromName: (name) => options.namespace.idFromName(name),
|
|
539
|
+
get: (id, init) => options.namespace.get(id, {
|
|
540
|
+
...init,
|
|
541
|
+
locationHint: options.locationHint ?? DEFAULT_TURN_LOCK_LOCATION_HINT
|
|
542
|
+
})
|
|
543
|
+
};
|
|
353
544
|
return {
|
|
354
545
|
// Every lock lives on the workspace channel, so this wakes that object
|
|
355
546
|
// (evicted after idle) before the turn's acquire reaches it.
|
|
356
547
|
async warm(target) {
|
|
357
|
-
await peekDurableTurnLock(
|
|
548
|
+
await peekDurableTurnLock(namespace, target.threadId ? { workspaceId: target.tenantId, threadId: target.threadId, scope: "thread" } : { workspaceId: target.tenantId, scope: "workspace" }, options.auth);
|
|
358
549
|
},
|
|
359
550
|
async acquire(args) {
|
|
360
551
|
const workspaceId = args.identity.tenantId;
|
|
@@ -362,20 +553,23 @@ function createDurableTurnLock(options) {
|
|
|
362
553
|
const scope = options.scopeOf(args);
|
|
363
554
|
const executionId = options.lockExecutionIdOf?.(args) ?? args.executionId;
|
|
364
555
|
const turnId = options.clientTurnIdOf ? options.clientTurnIdOf(args) : args.body.turnId;
|
|
556
|
+
const cancellable = options.cancellationOf?.(args) === true;
|
|
557
|
+
if (cancellable && !args.turnStreamId) throw new Error("Durable cancellation requires the route turnStreamId");
|
|
365
558
|
const input = {
|
|
366
559
|
workspaceId,
|
|
367
560
|
threadId,
|
|
368
561
|
scope,
|
|
369
562
|
executionId,
|
|
370
|
-
...turnId ? { turnId } : {}
|
|
563
|
+
...turnId ? { turnId } : {},
|
|
564
|
+
...cancellable ? { cancellation: { turnStreamId: args.turnStreamId } } : {}
|
|
371
565
|
};
|
|
372
|
-
let acquired = await acquireDurableTurnLock(
|
|
566
|
+
let acquired = await acquireDurableTurnLock(namespace, input, options.auth);
|
|
373
567
|
let diagnostics;
|
|
374
568
|
if (!acquired.acquired && options.reconcile) {
|
|
375
569
|
const reconciled = await options.reconcile(args, acquired.active);
|
|
376
570
|
diagnostics = reconciled.diagnostics;
|
|
377
571
|
if (reconciled.released) {
|
|
378
|
-
acquired = await acquireDurableTurnLock(
|
|
572
|
+
acquired = await acquireDurableTurnLock(namespace, input, options.auth);
|
|
379
573
|
}
|
|
380
574
|
}
|
|
381
575
|
if (!acquired.acquired) {
|
|
@@ -391,11 +585,22 @@ function createDurableTurnLock(options) {
|
|
|
391
585
|
executionId,
|
|
392
586
|
lockId: acquired.lock.lockId
|
|
393
587
|
};
|
|
394
|
-
return { acquired: true, handle };
|
|
588
|
+
if (!cancellable) return { acquired: true, handle };
|
|
589
|
+
try {
|
|
590
|
+
const cancellation = await connectDurableTurnCancellation(
|
|
591
|
+
options.namespace,
|
|
592
|
+
{ ...handle, turnStreamId: args.turnStreamId },
|
|
593
|
+
options.auth
|
|
594
|
+
);
|
|
595
|
+
return { acquired: true, handle, cancellation };
|
|
596
|
+
} catch (error) {
|
|
597
|
+
await releaseDurableTurnLock(options.namespace, handle, options.auth);
|
|
598
|
+
throw error;
|
|
599
|
+
}
|
|
395
600
|
},
|
|
396
601
|
async release(handle) {
|
|
397
602
|
if (!handle) return;
|
|
398
|
-
await releaseDurableTurnLock(
|
|
603
|
+
await releaseDurableTurnLock(namespace, handle, options.auth);
|
|
399
604
|
}
|
|
400
605
|
};
|
|
401
606
|
}
|
|
@@ -447,9 +652,6 @@ function createTurnStreamUpgradeHandler(options) {
|
|
|
447
652
|
}
|
|
448
653
|
|
|
449
654
|
export {
|
|
450
|
-
DEFAULT_STALE_TURN_LOCK_GRACE_MS,
|
|
451
|
-
DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS,
|
|
452
|
-
reconcileStaleTurnLock,
|
|
453
655
|
TURN_STREAM_TOKEN_TTL_MS,
|
|
454
656
|
TURN_STREAM_TOKEN_HEADER,
|
|
455
657
|
mintTurnStreamToken,
|
|
@@ -471,6 +673,14 @@ export {
|
|
|
471
673
|
TURN_STREAM_PATHS,
|
|
472
674
|
TURN_STREAM_STORAGE_KEYS,
|
|
473
675
|
turnEventStorageKey,
|
|
676
|
+
TurnCancellationRequestedError,
|
|
677
|
+
TurnControlUnavailableError,
|
|
678
|
+
requestDurableTurnCancellation,
|
|
679
|
+
connectDurableTurnCancellation,
|
|
680
|
+
DEFAULT_STALE_TURN_LOCK_GRACE_MS,
|
|
681
|
+
DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS,
|
|
682
|
+
reconcileStaleTurnLock,
|
|
683
|
+
DEFAULT_TURN_LOCK_LOCATION_HINT,
|
|
474
684
|
createDurableObjectTurnEventStore,
|
|
475
685
|
acquireDurableTurnLock,
|
|
476
686
|
peekDurableTurnLock,
|
|
@@ -482,4 +692,4 @@ export {
|
|
|
482
692
|
broadcastThreadCreated,
|
|
483
693
|
createTurnStreamUpgradeHandler
|
|
484
694
|
};
|
|
485
|
-
//# sourceMappingURL=chunk-
|
|
695
|
+
//# sourceMappingURL=chunk-MT6ZTQGD.js.map
|